Annotation of ZSQLExtend/importFMPXML.py, revision 1.14
1.1 casties 1: #!/usr/local/bin/python
2: #
3:
4: import string
5: import logging
6: import sys
1.7 dwinter 7: import types
1.8 casties 8: import time
1.1 casties 9:
1.5 casties 10: from xml import sax
11: from amara import saxtools
12:
1.2 casties 13: try:
14: import psycopg2 as psycopg
15: psyco = 2
16: except:
17: import psycopg
18: psyco = 1
19:
1.5 casties 20: fm_ns = 'http://www.filemaker.com/fmpxmlresult'
1.1 casties 21:
1.8 casties 22: version_string = "V0.4 ROC 29.3.2007"
23:
1.1 casties 24: def getTextFromNode(nodename):
25: """get the cdata content of a node"""
26: if nodename is None:
27: return ""
28: nodelist=nodename.childNodes
29: rc = ""
30: for node in nodelist:
31: if node.nodeType == node.TEXT_NODE:
32: rc = rc + node.data
33: return rc
34:
35: def sql_quote(v):
36: # quote dictionary
37: quote_dict = {"\'": "''", "\\": "\\\\"}
38: for dkey in quote_dict.keys():
39: if string.find(v, dkey) >= 0:
40: v=string.join(string.split(v,dkey),quote_dict[dkey])
1.5 casties 41: return "'%s'"%v
1.1 casties 42:
1.8 casties 43: def SimpleSearch(curs,query, args=None, ascii=False):
1.1 casties 44: """execute sql query and return data"""
1.8 casties 45: #logging.debug("executing: "+query)
46: if ascii:
47: # encode all in UTF-8
1.2 casties 48: query = query.encode("UTF-8")
1.8 casties 49: if args is not None:
50: encargs = []
51: for a in args:
52: if a is not None:
53: a = a.encode("UTF-8")
54: encargs.append(a)
55:
56: args = encargs
1.7 dwinter 57:
1.1 casties 58: curs.execute(query, args)
1.8 casties 59: #logging.debug("sql done")
1.4 casties 60: try:
61: return curs.fetchall()
62: except:
63: return None
1.1 casties 64:
65:
1.8 casties 66: class TableColumn:
67: """simple type for storing sql column name and type"""
68:
69: def __init__(self, name, type=None):
70: #print "new tablecolumn(%s,%s)"%(name, type)
71: self.name = name
72: self.type = type
73:
74: def getName(self):
75: return self.name
76:
77: def getType(self):
78: if self.type is not None:
79: return self.type
80: else:
81: return "text"
1.5 casties 82:
1.8 casties 83: def __str__(self):
84: return self.name
85:
86:
1.5 casties 87: class xml_handler:
1.8 casties 88: def __init__(self,options):
1.9 dwinter 89: """SAX handler to import FileMaker XML file (FMPXMLRESULT format) into the table.
1.8 casties 90: @param options: dict of options
91: @param options.dsn: database connection string
92: @param options.table: name of the table the xml shall be imported into
93: @param options.filename: xmlfile filename
94: @param options.update_fields: (optional) list of fields to update; default is to create all fields
95: @param options.id_field: (optional) field which uniquely identifies an entry for updating purposes.
96: @param options.sync_mode: (optional) really synchronise, i.e. delete entries not in XML file
97: @param options.lc_names: (optional) lower case and clean up field names from XML
98: @param options.keep_fields: (optional) don't add fields to SQL database
99: @param options.ascii_db: (optional) assume ascii encoding in db
100: @param options.replace_table: (optional) delete and re-insert data
1.14 ! casties 101: @param options.backup_table: (optional) create backup of old table (breaks indices)
1.9 dwinter 102: """
103:
1.5 casties 104: # set up parser
105: self.event = None
106: self.top_dispatcher = {
107: (saxtools.START_ELEMENT, fm_ns, u'METADATA'):
108: self.handle_meta_fields,
109: (saxtools.START_ELEMENT, fm_ns, u'RESULTSET'):
1.8 casties 110: self.handle_data_fields,
1.5 casties 111: }
112:
113: # connect database
1.8 casties 114: self.dbCon = psycopg.connect(options.dsn)
1.5 casties 115: self.db = self.dbCon.cursor()
1.8 casties 116: assert self.db, "AIIEE no db cursor for %s!!"%options.dsn
1.1 casties 117:
1.9 dwinter 118: self.table = getattr(options,"table",None)
119: self.update_fields = getattr(options,"update_fields",None)
120: self.id_field = getattr(options,"id_field",None)
121: self.sync_mode = getattr(options,"sync_mode",None)
122: self.lc_names = getattr(options,"lc_names",None)
123: self.keep_fields = getattr(options,"keep_fields",None)
124: self.ascii_db = getattr(options,"ascii_db",None)
125: self.replace_table = getattr(options,"replace_table",None)
126: self.backup_table = getattr(options,"backup_table",None)
1.8 casties 127:
1.9 dwinter 128: logging.debug("dsn: "+repr(getattr(options,"dsn",None)))
1.8 casties 129: logging.debug("table: "+repr(self.table))
130: logging.debug("update_fields: "+repr(self.update_fields))
131: logging.debug("id_field: "+repr(self.id_field))
132: logging.debug("sync_mode: "+repr(self.sync_mode))
133: logging.debug("lc_names: "+repr(self.lc_names))
134: logging.debug("keep_fields: "+repr(self.keep_fields))
135: logging.debug("ascii_db: "+repr(self.ascii_db))
136: logging.debug("replace_table: "+repr(self.replace_table))
1.14 ! casties 137: logging.debug("backup_table: "+repr(self.backup_table))
1.5 casties 138:
139: self.dbIDs = {}
140: self.rowcnt = 0
1.8 casties 141:
142: if self.id_field is not None:
1.5 casties 143: # prepare a list of ids for sync mode
1.8 casties 144: qstr="select %s from %s"%(self.id_field,self.table)
1.5 casties 145: for id in SimpleSearch(self.db, qstr):
146: # value 0: not updated
147: self.dbIDs[id[0]] = 0;
148: self.rowcnt += 1
149:
150: logging.info("%d entries in DB to sync"%self.rowcnt)
151:
1.8 casties 152: # names of fields in XML file
153: self.xml_field_names = []
154: # map XML field names to SQL field names
155: self.xml_field_map = {}
156: # and vice versa
157: self.sql_field_map = {}
1.5 casties 158:
159: return
160:
161: def handle_meta_fields(self, end_condition):
162: dispatcher = {
163: (saxtools.START_ELEMENT, fm_ns, u'FIELD'):
164: self.handle_meta_field,
165: }
166: #First round through the generator corresponds to the
167: #start element event
168: logging.debug("START METADATA")
169: yield None
1.1 casties 170:
1.5 casties 171: #delegate is a generator that handles all the events "within"
172: #this element
173: delegate = None
174: while not self.event == end_condition:
175: delegate = saxtools.tenorsax.event_loop_body(
176: dispatcher, delegate, self.event)
177: yield None
178:
179: #Element closed. Wrap up
180: logging.debug("END METADATA")
1.8 casties 181:
182: # rename table for backup
183: if self.backup_table:
184: self.orig_table = self.table
185: self.table = self.table + "_tmp"
186: # remove old temp table
187: qstr = "DROP TABLE %s"%(self.table)
188: try:
189: self.db.execute(qstr)
190: except:
191: pass
192:
193: self.dbCon.commit()
194:
195: if self.id_field:
196: # sync mode -- copy table
197: logging.info("copy table %s to %s"%(self.orig_table,self.table))
198: qstr = "CREATE TABLE %s AS (SELECT * FROM %s)"%(self.table,self.orig_table)
199:
200: else:
201: # rename table and create empty new one
202: logging.info("create empty table %s"%(self.table))
203: qstr = "CREATE TABLE %s AS (SELECT * FROM %s WHERE 1=0)"%(self.table,self.orig_table)
204:
205: self.db.execute(qstr)
206: self.dbCon.commit()
207:
208: # delete data from table for replace
209: if self.replace_table:
210: logging.info("delete data from table %s"%(self.table))
211: qstr = "TRUNCATE TABLE %s"%(self.table)
212: self.db.execute(qstr)
213: self.dbCon.commit()
214:
215: # try to match date style with XML
216: self.db.execute("set datestyle to 'german'")
217:
218: # translate id_field (SQL-name) to XML-name
219: self.xml_id = self.sql_field_map.get(self.id_field, None)
220:
221: #logging.debug("xml-fieldnames:"+repr(self.xml_field_names))
222: # get list of fields and types of db table
223: qstr="select attname, format_type(pg_attribute.atttypid, pg_attribute.atttypmod) from pg_attribute, pg_class where attrelid = pg_class.oid and pg_attribute.attnum > 0 and relname = '%s'"
224: self.sql_fields={}
225: for f in SimpleSearch(self.db, qstr%self.table):
226: n = f[0]
227: t = f[1]
228: #print "SQL fields: %s (%s)"%(n,t)
229: self.sql_fields[n] = TableColumn(n,t)
230:
231: # check fields to update
1.5 casties 232: if self.update_fields is None:
1.8 casties 233: if self.keep_fields:
1.13 casties 234: # update all existing fields from sql (when they are in the xml file)
235: self.update_fields = {}
236: for f in self.sql_fields.keys():
237: if self.sql_field_map.has_key(f):
238: xf = self.sql_field_map[f]
239: self.update_fields[f] = self.xml_field_map[xf]
240:
1.8 casties 241: else:
242: # update all fields
243: if self.lc_names:
244: # create dict with sql names
245: self.update_fields = {}
246: for f in self.xml_field_map.values():
247: self.update_fields[f.getName()] = f
248:
249: else:
250: self.update_fields = self.xml_field_map
251:
252: # and translate to list of xml fields
253: if self.lc_names:
254: self.xml_update_list = [self.sql_field_map[x] for x in self.update_fields]
255: else:
256: self.xml_update_list = self.update_fields.keys()
1.5 casties 257:
1.8 casties 258: if not self.keep_fields:
259: # adjust db table to fields in XML and update_fields
260: for f in self.xml_field_map.values():
261: logging.debug("sync-fieldname: %s"%f.getName())
262: sf = self.sql_fields.get(f.getName(), None)
263: uf = self.update_fields.get(f.getName(), None)
264: if sf is not None:
265: # name in db -- check type
266: if f.getType() != sf.getType():
267: logging.debug("field %s has different type (%s vs %s)"%(f,f.getType(),sf.getType()))
268: elif uf is not None:
269: # add field to table
270: qstr="alter table %s add %s %s"%(self.table,uf.getName(),uf.getType())
271: logging.info("db add field:"+qstr)
272:
273: if self.ascii_db and type(qstr)==types.UnicodeType:
274: qstr=qstr.encode('utf-8')
275:
276: self.db.execute(qstr)
277: self.dbCon.commit()
1.7 dwinter 278:
1.5 casties 279: # prepare sql statements for update
1.8 casties 280: setStr=string.join(["%s = %%s"%self.xml_field_map[f] for f in self.xml_update_list], ', ')
1.5 casties 281: self.updQuery="UPDATE %s SET %s WHERE %s = %%s"%(self.table,setStr,self.id_field)
282: # and insert
1.8 casties 283: fields=string.join([self.xml_field_map[x].getName() for x in self.xml_update_list], ',')
284: values=string.join(['%s' for f in self.xml_update_list], ',')
1.5 casties 285: self.addQuery="INSERT INTO %s (%s) VALUES (%s)"%(self.table,fields,values)
1.8 casties 286: logging.debug("update-query: "+self.updQuery)
287: logging.debug("add-query: "+self.addQuery)
1.5 casties 288: return
289:
290: def handle_meta_field(self, end_condition):
291: name = self.params.get((None, u'NAME'))
292: yield None
293: #Element closed. Wrap up
1.8 casties 294: if self.lc_names:
295: # clean name
296: sqlname = name.replace(" ","_").lower()
297: else:
298: sqlname = name
299: self.xml_field_names.append(name)
300: # map to sql name and default text type
301: self.xml_field_map[name] = TableColumn(sqlname, 'text')
302: self.sql_field_map[sqlname] = name
1.5 casties 303: logging.debug("FIELD name: "+name)
304: return
305:
1.8 casties 306: def handle_data_fields(self, end_condition):
1.5 casties 307: dispatcher = {
308: (saxtools.START_ELEMENT, fm_ns, u'ROW'):
309: self.handle_row,
310: }
311: #First round through the generator corresponds to the
312: #start element event
313: logging.debug("START RESULTSET")
314: self.rowcnt = 0
315: yield None
1.1 casties 316:
1.5 casties 317: #delegate is a generator that handles all the events "within"
318: #this element
319: delegate = None
320: while not self.event == end_condition:
321: delegate = saxtools.tenorsax.event_loop_body(
322: dispatcher, delegate, self.event)
323: yield None
324:
325: #Element closed. Wrap up
326: logging.debug("END RESULTSET")
327: self.dbCon.commit()
1.1 casties 328:
1.5 casties 329: if self.sync_mode:
330: # delete unmatched entries in db
1.8 casties 331: logging.info("deleting unmatched rows from db")
1.6 casties 332: delQuery = "DELETE FROM %s WHERE %s = %%s"%(self.table,self.id_field)
1.5 casties 333: for id in self.dbIDs.keys():
334: # find all not-updated fields
335: if self.dbIDs[id] == 0:
336: logging.info(" delete:"+id)
1.8 casties 337: SimpleSearch(self.db, delQuery, [id], ascii=self.ascii_db)
1.6 casties 338: sys.exit(1)
1.1 casties 339:
1.5 casties 340: elif self.dbIDs[id] > 1:
1.8 casties 341: logging.info(" sync: ID %s used more than once?"%id)
342:
343: self.dbCon.commit()
1.1 casties 344:
1.8 casties 345: # reinstate backup tables
346: if self.backup_table:
347: backup_name = "%s_%s"%(self.orig_table,time.strftime('%Y_%m_%d_%H_%M_%S'))
348: logging.info("rename backup table %s to %s"%(self.orig_table,backup_name))
349: qstr = "ALTER TABLE %s RENAME TO %s"%(self.orig_table,backup_name)
350: self.db.execute(qstr)
351: logging.info("rename working table %s to %s"%(self.table,self.orig_table))
352: qstr = "ALTER TABLE %s RENAME TO %s"%(self.table,self.orig_table)
353: self.db.execute(qstr)
1.5 casties 354: self.dbCon.commit()
1.1 casties 355:
1.5 casties 356: return
357:
358: def handle_row(self, end_condition):
359: dispatcher = {
360: (saxtools.START_ELEMENT, fm_ns, u'COL'):
361: self.handle_col,
362: }
363: logging.debug("START ROW")
1.8 casties 364: self.xml_data = {}
1.5 casties 365: self.colIdx = 0
366: yield None
1.1 casties 367:
1.5 casties 368: #delegate is a generator that handles all the events "within"
369: #this element
370: delegate = None
371: while not self.event == end_condition:
372: delegate = saxtools.tenorsax.event_loop_body(
373: dispatcher, delegate, self.event)
374: yield None
375:
376: #Element closed. Wrap up
377: logging.debug("END ROW")
378: self.rowcnt += 1
379: # process collected row data
380: update=False
381: id_val=''
382: # synchronize by id_field
383: if self.id_field:
1.8 casties 384: id_val = self.xml_data[self.xml_id]
1.5 casties 385: if id_val in self.dbIDs:
386: self.dbIDs[id_val] += 1
387: update=True
1.8 casties 388:
389: # collect all values
390: args = []
391: for fn in self.xml_update_list:
392: f = self.xml_field_map[fn]
393: val = self.xml_data[fn]
394: type = self.sql_fields[f.getName()].getType()
395: if type == "date" and len(val) == 0:
396: # empty date field
397: val = None
398:
399: elif type == "integer" and len(val) == 0:
400: # empty int field
401: val = None
402:
403: args.append(val)
404:
1.5 casties 405: if update:
406: # update existing row (by id_field)
1.8 casties 407: # last argument is ID match
1.5 casties 408: args.append(id_val)
1.8 casties 409: logging.debug("update: %s = %s"%(id_val, args))
410: SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db)
411:
1.5 casties 412: else:
413: # create new row
1.8 casties 414: logging.debug("insert: %s"%args)
415: SimpleSearch(self.db, self.addQuery, args, ascii=self.ascii_db)
1.5 casties 416:
417: #logging.info(" row:"+"%d (%s)"%(self.rowcnt,id_val))
418: if (self.rowcnt % 10) == 0:
419: logging.info(" row:"+"%d (%s)"%(self.rowcnt,id_val))
420: self.dbCon.commit()
421:
422: return
423:
424: def handle_col(self, end_condition):
425: dispatcher = {
426: (saxtools.START_ELEMENT, fm_ns, u'DATA'):
427: self.handle_data_tag,
428: }
429: #print "START COL"
430: yield None
431: #delegate is a generator that handles all the events "within"
432: #this element
433: delegate = None
434: while not self.event == end_condition:
435: delegate = saxtools.tenorsax.event_loop_body(
436: dispatcher, delegate, self.event)
437: yield None
438: #Element closed. Wrap up
439: #print "END COL"
440: self.colIdx += 1
441: return
442:
443: def handle_data_tag(self, end_condition):
444: #print "START DATA"
445: content = u''
446: yield None
447: # gather child elements
448: while not self.event == end_condition:
449: if self.event[0] == saxtools.CHARACTER_DATA:
450: content += self.params
451: yield None
452: #Element closed. Wrap up
1.8 casties 453: fn = self.xml_field_names[self.colIdx]
454: self.xml_data[fn] = content
1.5 casties 455: return
456:
457:
1.11 dwinter 458: def importFMPXML(options):
1.14 ! casties 459: """import FileMaker XML file (FMPXMLRESULT format) into the table.
1.11 dwinter 460: @param options: dict of options
461: @param options.dsn: database connection string
462: @param options.table: name of the table the xml shall be imported into
463: @param options.filename: xmlfile filename
464: @param options.update_fields: (optional) list of fields to update; default is to create all fields
465: @param options.id_field: (optional) field which uniquely identifies an entry for updating purposes.
466: @param options.sync_mode: (optional) really synchronise, i.e. delete entries not in XML file
467: @param options.lc_names: (optional) lower case and clean up field names from XML
468: @param options.keep_fields: (optional) don't add fields to SQL database
469: @param options.ascii_db: (optional) assume ascii encoding in db
470: @param options.replace_table: (optional) delete and re-insert data
1.14 ! casties 471: @param options.backup_table: (optional) create backup of old table (breaks indices)
1.11 dwinter 472: """
473:
474:
475: if getattr(options,'update_fields',None):
476: uf = {}
477: for f in options.update_fields.split(','):
1.12 casties 478: if f.find(':') > 0:
479: (n,t) = f.split(':')
480: else:
481: n = f
482: t = None
1.11 dwinter 483: uf[n] = TableColumn(n,t)
484:
485: options.update_fields = uf
486:
487: if getattr(options,'id_field',None) and getattr(options,'replace_table',None):
488: logging.error("ABORT: sorry, you can't do both sync (id_field) and replace")
489: sys.exit(1)
490:
491: parser = sax.make_parser()
492: #The "consumer" is our own handler
493: consumer = xml_handler(options)
494: #Initialize Tenorsax with handler
495: handler = saxtools.tenorsax(consumer)
496: #Resulting tenorsax instance is the SAX handler
497: parser.setContentHandler(handler)
498: parser.setFeature(sax.handler.feature_namespaces, 1)
499: parser.parse(options.filename)
500:
1.1 casties 501:
1.9 dwinter 502: if __name__ == "__main__":
503: from optparse import OptionParser
1.5 casties 504:
1.9 dwinter 505: opars = OptionParser()
506: opars.add_option("-f", "--file",
507: dest="filename",
508: help="FMPXML file name", metavar="FILE")
509: opars.add_option("-c", "--dsn",
510: dest="dsn",
511: help="database connection string")
512: opars.add_option("-t", "--table",
513: dest="table",
514: help="database table name")
515: opars.add_option("--fields", default=None,
516: dest="update_fields",
517: help="list of fields to update (comma separated, sql-names)", metavar="LIST")
518: opars.add_option("--id-field", default=None,
519: dest="id_field",
520: help="name of id field for synchronisation (only appends data otherwise, sql-name)", metavar="NAME")
521: opars.add_option("--sync", "--sync-mode", default=False, action="store_true",
522: dest="sync_mode",
523: help="do full sync based on id field (remove unmatched fields from db)")
524: opars.add_option("--lc-names", default=False, action="store_true",
525: dest="lc_names",
526: help="clean and lower case field names from XML")
527: opars.add_option("--keep-fields", default=False, action="store_true",
528: dest="keep_fields",
529: help="don't add fields from XML to SQL table")
530: opars.add_option("--ascii-db", default=False, action="store_true",
531: dest="ascii_db",
532: help="the SQL database stores ASCII instead of unicode")
533: opars.add_option("--replace", default=False, action="store_true",
534: dest="replace_table",
535: help="replace table i.e. delete and re-insert data")
536: opars.add_option("--backup", default=False, action="store_true",
537: dest="backup_table",
538: help="create backup of old table (breaks indices)")
539: opars.add_option("-d", "--debug", default=False, action="store_true",
540: dest="debug",
541: help="debug mode (more output)")
542:
543: (options, args) = opars.parse_args()
544:
545: if len(sys.argv) < 2 or options.filename is None or options.dsn is None:
546: print "importFMPXML "+version_string
547: opars.print_help()
548: sys.exit(1)
549:
550: if options.debug:
551: loglevel = logging.DEBUG
552: else:
553: loglevel = logging.INFO
554:
555: logging.basicConfig(level=loglevel,
556: format='%(asctime)s %(levelname)s %(message)s',
557: datefmt='%H:%M:%S')
558:
1.11 dwinter 559: importFMPXML(options)
560:
1.9 dwinter 561:
562:
1.1 casties 563:
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/cvs/cvsweb/icons/back.gif){kind=icon}
Return to importFMPXML.py CVS log ![[TXT]](/cvs/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvs/cvsweb/icons/dir.gif){kind=icon}
Up to [Repository] / ZSQLExtend