--- ZSQLExtend/importFMPXML.py 2007/02/20 16:00:03 1.7 +++ ZSQLExtend/importFMPXML.py 2007/03/29 18:31:32 1.8 @@ -5,6 +5,7 @@ import string import logging import sys import types +import time from xml import sax from amara import saxtools @@ -18,6 +19,8 @@ except: fm_ns = 'http://www.filemaker.com/fmpxmlresult' +version_string = "V0.4 ROC 29.3.2007" + def getTextFromNode(nodename): """get the cdata content of a node""" if nodename is None: @@ -37,36 +40,66 @@ def sql_quote(v): v=string.join(string.split(v,dkey),quote_dict[dkey]) return "'%s'"%v -def SimpleSearch(curs,query, args=None): +def SimpleSearch(curs,query, args=None, ascii=False): """execute sql query and return data""" - logging.debug("executing: "+query) - if psyco == 1: + #logging.debug("executing: "+query) + if ascii: + # encode all in UTF-8 query = query.encode("UTF-8") - #if args is not None: - # args = [ sql_quote(a) for a in args ] - #logging.debug(query) - #logging.debug(args) + if args is not None: + encargs = [] + for a in args: + if a is not None: + a = a.encode("UTF-8") + encargs.append(a) + + args = encargs curs.execute(query, args) - logging.debug("sql done") + #logging.debug("sql done") try: return curs.fetchall() except: return None +class TableColumn: + """simple type for storing sql column name and type""" + + def __init__(self, name, type=None): + #print "new tablecolumn(%s,%s)"%(name, type) + self.name = name + self.type = type + + def getName(self): + return self.name + + def getType(self): + if self.type is not None: + return self.type + else: + return "text" + def __str__(self): + return self.name + + class xml_handler: - def __init__(self,dsn,table,update_fields=None,id_field=None,sync_mode=False): + def __init__(self,options): ''' SAX handler to import FileMaker XML file (FMPXMLRESULT format) into the table. - @param dsn: database connection string - @param table: name of the table the xml shall be imported into - @param filename: xmlfile filename - @param update_fields: (optional) list of fields to update; default is to create all fields - @param id_field: (optional) field which uniquely identifies an entry for updating purposes. - @param sync_mode: (optional) really synchronise, i.e. delete entries not in XML file + @param options: dict of options + @param options.dsn: database connection string + @param options.table: name of the table the xml shall be imported into + @param options.filename: xmlfile filename + @param options.update_fields: (optional) list of fields to update; default is to create all fields + @param options.id_field: (optional) field which uniquely identifies an entry for updating purposes. + @param options.sync_mode: (optional) really synchronise, i.e. delete entries not in XML file + @param options.lc_names: (optional) lower case and clean up field names from XML + @param options.keep_fields: (optional) don't add fields to SQL database + @param options.ascii_db: (optional) assume ascii encoding in db + @param options.replace_table: (optional) delete and re-insert data ''' # set up parser self.event = None @@ -74,32 +107,40 @@ class xml_handler: (saxtools.START_ELEMENT, fm_ns, u'METADATA'): self.handle_meta_fields, (saxtools.START_ELEMENT, fm_ns, u'RESULTSET'): - self.handle_data, + self.handle_data_fields, } # connect database - self.dbCon = psycopg.connect(dsn) + self.dbCon = psycopg.connect(options.dsn) self.db = self.dbCon.cursor() - assert self.db, "AIIEE no db cursor for %s!!"%dsn + assert self.db, "AIIEE no db cursor for %s!!"%options.dsn - logging.debug("dsn: "+repr(dsn)) - logging.debug("table: "+repr(table)) - logging.debug("update_fields: "+repr(update_fields)) - logging.debug("id_field: "+repr(id_field)) - logging.debug("sync_mode: "+repr(sync_mode)) - - self.table = table - self.update_fields = update_fields - self.id_field = id_field - self.sync_mode = sync_mode + self.table = options.table + self.update_fields = options.update_fields + self.id_field = options.id_field + self.sync_mode = options.sync_mode + self.lc_names = options.lc_names + self.keep_fields = options.keep_fields + self.ascii_db = options.ascii_db + self.replace_table = options.replace_table + self.backup_table = options.backup_table + + logging.debug("dsn: "+repr(options.dsn)) + logging.debug("table: "+repr(self.table)) + logging.debug("update_fields: "+repr(self.update_fields)) + logging.debug("id_field: "+repr(self.id_field)) + logging.debug("sync_mode: "+repr(self.sync_mode)) + logging.debug("lc_names: "+repr(self.lc_names)) + logging.debug("keep_fields: "+repr(self.keep_fields)) + logging.debug("ascii_db: "+repr(self.ascii_db)) + logging.debug("replace_table: "+repr(self.replace_table)) self.dbIDs = {} self.rowcnt = 0 - - self.db.execute("set datestyle to 'german'") - if id_field is not None: + + if self.id_field is not None: # prepare a list of ids for sync mode - qstr="select %s from %s"%(id_field,table) + qstr="select %s from %s"%(self.id_field,self.table) for id in SimpleSearch(self.db, qstr): # value 0: not updated self.dbIDs[id[0]] = 0; @@ -107,7 +148,12 @@ class xml_handler: logging.info("%d entries in DB to sync"%self.rowcnt) - self.fieldNames = [] + # names of fields in XML file + self.xml_field_names = [] + # map XML field names to SQL field names + self.xml_field_map = {} + # and vice versa + self.sql_field_map = {} return @@ -131,50 +177,128 @@ class xml_handler: #Element closed. Wrap up logging.debug("END METADATA") + + # rename table for backup + if self.backup_table: + self.orig_table = self.table + self.table = self.table + "_tmp" + # remove old temp table + qstr = "DROP TABLE %s"%(self.table) + try: + self.db.execute(qstr) + except: + pass + + self.dbCon.commit() + + if self.id_field: + # sync mode -- copy table + logging.info("copy table %s to %s"%(self.orig_table,self.table)) + qstr = "CREATE TABLE %s AS (SELECT * FROM %s)"%(self.table,self.orig_table) + + else: + # rename table and create empty new one + logging.info("create empty table %s"%(self.table)) + qstr = "CREATE TABLE %s AS (SELECT * FROM %s WHERE 1=0)"%(self.table,self.orig_table) + + self.db.execute(qstr) + self.dbCon.commit() + + # delete data from table for replace + if self.replace_table: + logging.info("delete data from table %s"%(self.table)) + qstr = "TRUNCATE TABLE %s"%(self.table) + self.db.execute(qstr) + self.dbCon.commit() + + # try to match date style with XML + self.db.execute("set datestyle to 'german'") + + # translate id_field (SQL-name) to XML-name + self.xml_id = self.sql_field_map.get(self.id_field, None) + + #logging.debug("xml-fieldnames:"+repr(self.xml_field_names)) + # get list of fields and types of db table + qstr="select attname, format_type(pg_attribute.atttypid, pg_attribute.atttypmod) from pg_attribute, pg_class where attrelid = pg_class.oid and pg_attribute.attnum > 0 and relname = '%s'" + self.sql_fields={} + for f in SimpleSearch(self.db, qstr%self.table): + n = f[0] + t = f[1] + #print "SQL fields: %s (%s)"%(n,t) + self.sql_fields[n] = TableColumn(n,t) + + # check fields to update if self.update_fields is None: - # update all fields - self.update_fields = self.fieldNames + if self.keep_fields: + # update existing fields + self.update_fields = self.sql_fields + + else: + # update all fields + if self.lc_names: + # create dict with sql names + self.update_fields = {} + for f in self.xml_field_map.values(): + self.update_fields[f.getName()] = f + + else: + self.update_fields = self.xml_field_map + + # and translate to list of xml fields + if self.lc_names: + self.xml_update_list = [self.sql_field_map[x] for x in self.update_fields] + else: + self.xml_update_list = self.update_fields.keys() - logging.debug("xml-fieldnames:"+repr(self.fieldNames)) - # get list of fields in db table - qstr="""select attname from pg_attribute, pg_class where attrelid = pg_class.oid and relname = '%s'""" - columns=[x[0] for x in SimpleSearch(self.db, qstr%self.table)] - - # adjust db table to fields in XML and fieldlist - for fieldName in self.fieldNames: - logging.debug("db-fieldname:"+repr(fieldName)) - fieldName=fieldName.replace(" ","_") # repair _ - if (fieldName.lower() not in columns) and (fieldName in self.update_fields): - qstr="alter table %s add %s %s"%(self.table,fieldName,'text') - logging.info("db add field:"+qstr) + if not self.keep_fields: + # adjust db table to fields in XML and update_fields + for f in self.xml_field_map.values(): + logging.debug("sync-fieldname: %s"%f.getName()) + sf = self.sql_fields.get(f.getName(), None) + uf = self.update_fields.get(f.getName(), None) + if sf is not None: + # name in db -- check type + if f.getType() != sf.getType(): + logging.debug("field %s has different type (%s vs %s)"%(f,f.getType(),sf.getType())) + elif uf is not None: + # add field to table + qstr="alter table %s add %s %s"%(self.table,uf.getName(),uf.getType()) + logging.info("db add field:"+qstr) + + if self.ascii_db and type(qstr)==types.UnicodeType: + qstr=qstr.encode('utf-8') + + self.db.execute(qstr) + self.dbCon.commit() - if type(qstr)==types.UnicodeType: - qstr=qstr.encode('utf-8') - self.db.execute(qstr) - self.dbCon.commit() - # prepare sql statements for update - setStr=string.join(["%s = %%s"%f for f in self.update_fields], ', ') + setStr=string.join(["%s = %%s"%self.xml_field_map[f] for f in self.xml_update_list], ', ') self.updQuery="UPDATE %s SET %s WHERE %s = %%s"%(self.table,setStr,self.id_field) # and insert - fields=string.join(self.update_fields, ',') - values=string.join(['%s' for f in self.update_fields], ',') + fields=string.join([self.xml_field_map[x].getName() for x in self.xml_update_list], ',') + values=string.join(['%s' for f in self.xml_update_list], ',') self.addQuery="INSERT INTO %s (%s) VALUES (%s)"%(self.table,fields,values) - #print "upQ: ", self.updQuery - #print "adQ: ", self.addQuery - + logging.debug("update-query: "+self.updQuery) + logging.debug("add-query: "+self.addQuery) return def handle_meta_field(self, end_condition): name = self.params.get((None, u'NAME')) yield None #Element closed. Wrap up - name=name.replace(" ","_")# make sure no spaces - self.fieldNames.append(name) + if self.lc_names: + # clean name + sqlname = name.replace(" ","_").lower() + else: + sqlname = name + self.xml_field_names.append(name) + # map to sql name and default text type + self.xml_field_map[name] = TableColumn(sqlname, 'text') + self.sql_field_map[sqlname] = name logging.debug("FIELD name: "+name) return - def handle_data(self, end_condition): + def handle_data_fields(self, end_condition): dispatcher = { (saxtools.START_ELEMENT, fm_ns, u'ROW'): self.handle_row, @@ -199,17 +323,29 @@ class xml_handler: if self.sync_mode: # delete unmatched entries in db + logging.info("deleting unmatched rows from db") delQuery = "DELETE FROM %s WHERE %s = %%s"%(self.table,self.id_field) for id in self.dbIDs.keys(): # find all not-updated fields if self.dbIDs[id] == 0: logging.info(" delete:"+id) - SimpleSearch(self.db, delQuery, [id]) + SimpleSearch(self.db, delQuery, [id], ascii=self.ascii_db) sys.exit(1) elif self.dbIDs[id] > 1: - logging.info(" sync:"+"id used more than once?"+id) + logging.info(" sync: ID %s used more than once?"%id) + + self.dbCon.commit() + # reinstate backup tables + if self.backup_table: + backup_name = "%s_%s"%(self.orig_table,time.strftime('%Y_%m_%d_%H_%M_%S')) + logging.info("rename backup table %s to %s"%(self.orig_table,backup_name)) + qstr = "ALTER TABLE %s RENAME TO %s"%(self.orig_table,backup_name) + self.db.execute(qstr) + logging.info("rename working table %s to %s"%(self.table,self.orig_table)) + qstr = "ALTER TABLE %s RENAME TO %s"%(self.table,self.orig_table) + self.db.execute(qstr) self.dbCon.commit() return @@ -220,7 +356,7 @@ class xml_handler: self.handle_col, } logging.debug("START ROW") - self.dataSet = {} + self.xml_data = {} self.colIdx = 0 yield None @@ -240,40 +376,38 @@ class xml_handler: id_val='' # synchronize by id_field if self.id_field: - id_val=self.dataSet[self.id_field.lower()] + id_val = self.xml_data[self.xml_id] if id_val in self.dbIDs: self.dbIDs[id_val] += 1 update=True - + + # collect all values + args = [] + for fn in self.xml_update_list: + f = self.xml_field_map[fn] + val = self.xml_data[fn] + type = self.sql_fields[f.getName()].getType() + if type == "date" and len(val) == 0: + # empty date field + val = None + + elif type == "integer" and len(val) == 0: + # empty int field + val = None + + args.append(val) + if update: # update existing row (by id_field) - #setvals=[] - #for fieldName in self.update_fields: - # setvals.append("%s = %s"%(fieldName,sql_quote(self.dataSet[fieldName]))) - #setStr=string.join(setvals, ',') - id_val=self.dataSet[self.id_field.lower()] - #qstr="UPDATE %s SET %s WHERE %s = '%s' "%(self.table,setStr,self.id_field,id_val) - args = [self.dataSet[f.lower()] for f in self.update_fields] + # last argument is ID match args.append(id_val) - SimpleSearch(self.db, self.updQuery, args) - logging.debug("update: %s"%id_val) + logging.debug("update: %s = %s"%(id_val, args)) + SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db) + else: # create new row - #fields=string.join(update_fields, ',') - #values=string.join([" %s "%sql_quote(self.dataSet[x]) for x in self.update_fields], ',') - #qstr="INSERT INTO %s (%s) VALUES (%s)"%(self.table,fields,self.values) - args=[] - for f in self.update_fields: - value=self.dataSet[f.lower()].encode('utf-8') - if value=="": #hack DW - value=None - - args.append(value) - - #args = [self.dataSet[f.lower()].encode('utf-8') for f in self.update_fields] - logging.debug(args) - SimpleSearch(self.db, self.addQuery, args) - logging.debug("add: %s"%self.dataSet.get(self.id_field, self.rowcnt)) + logging.debug("insert: %s"%args) + SimpleSearch(self.db, self.addQuery, args, ascii=self.ascii_db) #logging.info(" row:"+"%d (%s)"%(self.rowcnt,id_val)) if (self.rowcnt % 10) == 0: @@ -311,9 +445,8 @@ class xml_handler: content += self.params yield None #Element closed. Wrap up - field = self.fieldNames[self.colIdx] - self.dataSet[field.lower()] = content - #print " DATA(", field, ") ", repr(content) + fn = self.xml_field_names[self.colIdx] + self.xml_data[fn] = content return @@ -338,13 +471,28 @@ opars.add_option("-t", "--table", help="database table name") opars.add_option("--fields", default=None, dest="update_fields", - help="list of fields to update (comma separated)", metavar="LIST") + help="list of fields to update (comma separated, sql-names)", metavar="LIST") opars.add_option("--id-field", default=None, dest="id_field", - help="name of id field for synchronisation (only appends data otherwise)", metavar="NAME") -opars.add_option("--sync-mode", default=False, action="store_true", + help="name of id field for synchronisation (only appends data otherwise, sql-name)", metavar="NAME") +opars.add_option("--sync", "--sync-mode", default=False, action="store_true", dest="sync_mode", help="do full sync based on id field (remove unmatched fields from db)") +opars.add_option("--lc-names", default=False, action="store_true", + dest="lc_names", + help="clean and lower case field names from XML") +opars.add_option("--keep-fields", default=False, action="store_true", + dest="keep_fields", + help="don't add fields from XML to SQL table") +opars.add_option("--ascii-db", default=False, action="store_true", + dest="ascii_db", + help="the SQL database stores ASCII instead of unicode") +opars.add_option("--replace", default=False, action="store_true", + dest="replace_table", + help="replace table i.e. delete and re-insert data") +opars.add_option("--backup", default=False, action="store_true", + dest="backup_table", + help="create backup of old table (breaks indices)") opars.add_option("-d", "--debug", default=False, action="store_true", dest="debug", help="debug mode (more output)") @@ -352,6 +500,7 @@ opars.add_option("-d", "--debug", defaul (options, args) = opars.parse_args() if len(sys.argv) < 2 or options.filename is None or options.dsn is None: + print "importFMPXML "+version_string opars.print_help() sys.exit(1) @@ -367,13 +516,20 @@ logging.basicConfig(level=loglevel, update_fields = None if options.update_fields: - update_fields = [string.strip(s) for s in options.update_fields.split(',')] + uf = {} + for f in options.update_fields.split(','): + (n,t) = f.split(':') + uf[n] = TableColumn(n,t) + + options.update_fields = uf +if options.id_field and options.replace_table: + logging.error("ABORT: sorry, you can't do both sync (id_field) and replace") + sys.exit(1) + parser = sax.make_parser() #The "consumer" is our own handler -consumer = xml_handler(dsn=options.dsn,table=options.table, - update_fields=update_fields,id_field=options.id_field, - sync_mode=options.sync_mode) +consumer = xml_handler(options) #Initialize Tenorsax with handler handler = saxtools.tenorsax(consumer) #Resulting tenorsax instance is the SAX handler