--- ZSQLExtend/importFMPXML.py 2007/04/02 09:48:13 1.9 +++ ZSQLExtend/importFMPXML.py 2008/01/09 14:23:26 1.20 @@ -19,7 +19,32 @@ except: fm_ns = 'http://www.filemaker.com/fmpxmlresult' -version_string = "V0.4 ROC 29.3.2007" +version_string = "V0.5.1 ROC 9.1.2008" + +def unicodify(str, withNone=False): + """decode str (utf-8 or latin-1 representation) into unicode object""" + if withNone and str is None: + return None + if not str: + return u"" + if type(str) is StringType: + try: + return str.decode('utf-8') + except: + return str.decode('latin-1') + else: + return str + +def utf8ify(str, withNone=False): + """encode unicode object or string into byte string in utf-8 representation""" + if withNone and str is None: + return None + if not str: + return "" + if type(str) is StringType: + return str + else: + return str.encode('utf-8') def getTextFromNode(nodename): """get the cdata content of a node""" @@ -42,21 +67,19 @@ def sql_quote(v): def SimpleSearch(curs,query, args=None, ascii=False): """execute sql query and return data""" - #logging.debug("executing: "+query) + #logger.debug("executing: "+query) if ascii: # encode all in UTF-8 - query = query.encode("UTF-8") + query = utf8ify(query) if args is not None: encargs = [] for a in args: - if a is not None: - a = a.encode("UTF-8") - encargs.append(a) + encargs.append(utf8ify(a, withNone=True)) args = encargs curs.execute(query, args) - #logging.debug("sql done") + #logger.debug("sql done") try: return curs.fetchall() except: @@ -98,8 +121,17 @@ class xml_handler: @param options.keep_fields: (optional) don't add fields to SQL database @param options.ascii_db: (optional) assume ascii encoding in db @param options.replace_table: (optional) delete and re-insert data + @param options.backup_table: (optional) create backup of old table (breaks indices) + @param options.use_logger_instance: (optional) use this instance of a logger """ + # set up logger + if hasattr(options, 'use_logger_instance'): + self.logger = options.use_logger_instance + else: + self.logger = logging.getLogger('db.import.fmpxml') + + # set up parser self.event = None self.top_dispatcher = { @@ -124,15 +156,16 @@ class xml_handler: self.replace_table = getattr(options,"replace_table",None) self.backup_table = getattr(options,"backup_table",None) - logging.debug("dsn: "+repr(getattr(options,"dsn",None))) - logging.debug("table: "+repr(self.table)) - logging.debug("update_fields: "+repr(self.update_fields)) - logging.debug("id_field: "+repr(self.id_field)) - logging.debug("sync_mode: "+repr(self.sync_mode)) - logging.debug("lc_names: "+repr(self.lc_names)) - logging.debug("keep_fields: "+repr(self.keep_fields)) - logging.debug("ascii_db: "+repr(self.ascii_db)) - logging.debug("replace_table: "+repr(self.replace_table)) + self.logger.debug("dsn: "+repr(getattr(options,"dsn",None))) + self.logger.debug("table: "+repr(self.table)) + self.logger.debug("update_fields: "+repr(self.update_fields)) + self.logger.debug("id_field: "+repr(self.id_field)) + self.logger.debug("sync_mode: "+repr(self.sync_mode)) + self.logger.debug("lc_names: "+repr(self.lc_names)) + self.logger.debug("keep_fields: "+repr(self.keep_fields)) + self.logger.debug("ascii_db: "+repr(self.ascii_db)) + self.logger.debug("replace_table: "+repr(self.replace_table)) + self.logger.debug("backup_table: "+repr(self.backup_table)) self.dbIDs = {} self.rowcnt = 0 @@ -145,7 +178,7 @@ class xml_handler: self.dbIDs[id[0]] = 0; self.rowcnt += 1 - logging.info("%d entries in DB to sync"%self.rowcnt) + self.logger.info("%d entries in DB to sync"%self.rowcnt) # names of fields in XML file self.xml_field_names = [] @@ -163,7 +196,8 @@ class xml_handler: } #First round through the generator corresponds to the #start element event - logging.debug("START METADATA") + self.logger.info("reading metadata...") + self.logger.debug("START METADATA") yield None #delegate is a generator that handles all the events "within" @@ -175,14 +209,16 @@ class xml_handler: yield None #Element closed. Wrap up - logging.debug("END METADATA") + self.logger.debug("END METADATA") # rename table for backup if self.backup_table: self.orig_table = self.table - self.table = self.table + "_tmp" + self.tmp_table = self.table + "_tmp" + backup_name = "%s_%s"%(self.table,time.strftime('%Y_%m_%d_%H_%M_%S')) + # remove old temp table - qstr = "DROP TABLE %s"%(self.table) + qstr = "DROP TABLE %s"%(self.tmp_table) try: self.db.execute(qstr) except: @@ -191,13 +227,14 @@ class xml_handler: self.dbCon.commit() if self.id_field: - # sync mode -- copy table - logging.info("copy table %s to %s"%(self.orig_table,self.table)) - qstr = "CREATE TABLE %s AS (SELECT * FROM %s)"%(self.table,self.orig_table) + # sync mode -- copy backup table, update current table + self.logger.info("copy table %s to %s"%(self.table,backup_name)) + qstr = "CREATE TABLE %s AS (SELECT * FROM %s)"%(backup_name,self.table) else: - # rename table and create empty new one - logging.info("create empty table %s"%(self.table)) + # replace mode -- create empty tmp table, insert into tmp table + self.table = self.tmp_table + self.logger.info("create empty table %s"%(self.table)) qstr = "CREATE TABLE %s AS (SELECT * FROM %s WHERE 1=0)"%(self.table,self.orig_table) self.db.execute(qstr) @@ -205,7 +242,7 @@ class xml_handler: # delete data from table for replace if self.replace_table: - logging.info("delete data from table %s"%(self.table)) + self.logger.info("delete data from table %s"%(self.table)) qstr = "TRUNCATE TABLE %s"%(self.table) self.db.execute(qstr) self.dbCon.commit() @@ -213,10 +250,7 @@ class xml_handler: # try to match date style with XML self.db.execute("set datestyle to 'german'") - # translate id_field (SQL-name) to XML-name - self.xml_id = self.sql_field_map.get(self.id_field, None) - - #logging.debug("xml-fieldnames:"+repr(self.xml_field_names)) + #self.logger.debug("xml-fieldnames:"+repr(self.xml_field_names)) # get list of fields and types of db table qstr="select attname, format_type(pg_attribute.atttypid, pg_attribute.atttypmod) from pg_attribute, pg_class where attrelid = pg_class.oid and pg_attribute.attnum > 0 and relname = '%s'" self.sql_fields={} @@ -226,13 +260,24 @@ class xml_handler: #print "SQL fields: %s (%s)"%(n,t) self.sql_fields[n] = TableColumn(n,t) + # translate id_field (SQL-name) to XML-name + self.xml_id = self.sql_field_map.get(self.id_field, None) + # get type of id_field + if self.id_field: + self.id_type = self.sql_fields[self.id_field].getType() + else: + self.id_type = None + # check fields to update if self.update_fields is None: if self.keep_fields: - # update existing fields - self.update_fields = self.sql_fields - - + # update all existing fields from sql (when they are in the xml file) + self.update_fields = {} + for f in self.sql_fields.keys(): + if self.sql_field_map.has_key(f): + xf = self.sql_field_map[f] + self.update_fields[f] = self.xml_field_map[xf] + else: # update all fields if self.lc_names: @@ -243,27 +288,27 @@ class xml_handler: else: self.update_fields = self.xml_field_map - + # and translate to list of xml fields if self.lc_names: self.xml_update_list = [self.sql_field_map[x] for x in self.update_fields] else: self.xml_update_list = self.update_fields.keys() - + if not self.keep_fields: # adjust db table to fields in XML and update_fields for f in self.xml_field_map.values(): - logging.debug("sync-fieldname: %s"%f.getName()) + self.logger.debug("sync-fieldname: %s"%f.getName()) sf = self.sql_fields.get(f.getName(), None) uf = self.update_fields.get(f.getName(), None) if sf is not None: # name in db -- check type if f.getType() != sf.getType(): - logging.debug("field %s has different type (%s vs %s)"%(f,f.getType(),sf.getType())) + self.logger.debug("field %s has different type (%s vs %s)"%(f,f.getType(),sf.getType())) elif uf is not None: # add field to table qstr="alter table %s add %s %s"%(self.table,uf.getName(),uf.getType()) - logging.info("db add field:"+qstr) + self.logger.info("db add field:"+qstr) if self.ascii_db and type(qstr)==types.UnicodeType: qstr=qstr.encode('utf-8') @@ -271,15 +316,15 @@ class xml_handler: self.db.execute(qstr) self.dbCon.commit() - # prepare sql statements for update - setStr=string.join(["%s = %%s"%self.xml_field_map[f] for f in self.xml_update_list], ', ') + # prepare sql statements for update (do not update id_field) + setStr=string.join(["%s = %%s"%self.xml_field_map[f] for f in self.xml_update_list if f != self.xml_id], ', ') self.updQuery="UPDATE %s SET %s WHERE %s = %%s"%(self.table,setStr,self.id_field) # and insert fields=string.join([self.xml_field_map[x].getName() for x in self.xml_update_list], ',') values=string.join(['%s' for f in self.xml_update_list], ',') self.addQuery="INSERT INTO %s (%s) VALUES (%s)"%(self.table,fields,values) - logging.debug("update-query: "+self.updQuery) - logging.debug("add-query: "+self.addQuery) + self.logger.debug("update-query: "+self.updQuery) + self.logger.debug("add-query: "+self.addQuery) return def handle_meta_field(self, end_condition): @@ -295,7 +340,7 @@ class xml_handler: # map to sql name and default text type self.xml_field_map[name] = TableColumn(sqlname, 'text') self.sql_field_map[sqlname] = name - logging.debug("FIELD name: "+name) + self.logger.debug("FIELD name: "+name) return def handle_data_fields(self, end_condition): @@ -305,7 +350,8 @@ class xml_handler: } #First round through the generator corresponds to the #start element event - logging.debug("START RESULTSET") + self.logger.info("reading data...") + self.logger.debug("START RESULTSET") self.rowcnt = 0 yield None @@ -318,32 +364,32 @@ class xml_handler: yield None #Element closed. Wrap up - logging.debug("END RESULTSET") + self.logger.debug("END RESULTSET") self.dbCon.commit() if self.sync_mode: # delete unmatched entries in db - logging.info("deleting unmatched rows from db") + self.logger.info("deleting unmatched rows from db") delQuery = "DELETE FROM %s WHERE %s = %%s"%(self.table,self.id_field) for id in self.dbIDs.keys(): # find all not-updated fields if self.dbIDs[id] == 0: - logging.info(" delete:"+id) + self.logger.info(" delete:"+id) SimpleSearch(self.db, delQuery, [id], ascii=self.ascii_db) sys.exit(1) elif self.dbIDs[id] > 1: - logging.info(" sync: ID %s used more than once?"%id) + self.logger.info(" sync: ID %s used more than once?"%id) self.dbCon.commit() # reinstate backup tables - if self.backup_table: + if self.backup_table and not self.id_field: backup_name = "%s_%s"%(self.orig_table,time.strftime('%Y_%m_%d_%H_%M_%S')) - logging.info("rename backup table %s to %s"%(self.orig_table,backup_name)) + self.logger.info("rename backup table %s to %s"%(self.orig_table,backup_name)) qstr = "ALTER TABLE %s RENAME TO %s"%(self.orig_table,backup_name) self.db.execute(qstr) - logging.info("rename working table %s to %s"%(self.table,self.orig_table)) + self.logger.info("rename working table %s to %s"%(self.table,self.orig_table)) qstr = "ALTER TABLE %s RENAME TO %s"%(self.table,self.orig_table) self.db.execute(qstr) self.dbCon.commit() @@ -355,7 +401,7 @@ class xml_handler: (saxtools.START_ELEMENT, fm_ns, u'COL'): self.handle_col, } - logging.debug("START ROW") + self.logger.debug("START ROW") self.xml_data = {} self.colIdx = 0 yield None @@ -369,14 +415,18 @@ class xml_handler: yield None #Element closed. Wrap up - logging.debug("END ROW") + self.logger.debug("END ROW") self.rowcnt += 1 # process collected row data update=False id_val='' # synchronize by id_field if self.id_field: - id_val = self.xml_data[self.xml_id] + if self.id_type == 'integer': + id_val = int(self.xml_data[self.xml_id]) + else: + id_val = self.xml_data[self.xml_id] + if id_val in self.dbIDs: self.dbIDs[id_val] += 1 update=True @@ -384,6 +434,10 @@ class xml_handler: # collect all values args = [] for fn in self.xml_update_list: + # do not update id_field + if update and fn == self.xml_id: + continue + f = self.xml_field_map[fn] val = self.xml_data[fn] type = self.sql_fields[f.getName()].getType() @@ -401,17 +455,17 @@ class xml_handler: # update existing row (by id_field) # last argument is ID match args.append(id_val) - logging.debug("update: %s = %s"%(id_val, args)) + self.logger.debug("update: %s = %s"%(id_val, args)) SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db) else: # create new row - logging.debug("insert: %s"%args) + self.logger.debug("insert: %s"%args) SimpleSearch(self.db, self.addQuery, args, ascii=self.ascii_db) - #logging.info(" row:"+"%d (%s)"%(self.rowcnt,id_val)) - if (self.rowcnt % 10) == 0: - logging.info(" row:"+"%d (%s)"%(self.rowcnt,id_val)) + #self.logger.info(" row:"+"%d (%s)"%(self.rowcnt,id_val)) + if (self.rowcnt % 100) == 0: + self.logger.info(" row:"+"%d (id:%s)"%(self.rowcnt,id_val)) self.dbCon.commit() return @@ -450,8 +504,48 @@ class xml_handler: return - - +def importFMPXML(options): + """import FileMaker XML file (FMPXMLRESULT format) into the table. + @param options: dict of options + @param options.dsn: database connection string + @param options.table: name of the table the xml shall be imported into + @param options.filename: xmlfile filename + @param options.update_fields: (optional) list of fields to update; default is to create all fields + @param options.id_field: (optional) field which uniquely identifies an entry for updating purposes. + @param options.sync_mode: (optional) really synchronise, i.e. delete entries not in XML file + @param options.lc_names: (optional) lower case and clean up field names from XML + @param options.keep_fields: (optional) don't add fields to SQL database + @param options.ascii_db: (optional) assume ascii encoding in db + @param options.replace_table: (optional) delete and re-insert data + @param options.backup_table: (optional) create backup of old table + """ + + if getattr(options,'update_fields',None): + uf = {} + for f in options.update_fields.split(','): + if f.find(':') > 0: + (n,t) = f.split(':') + else: + n = f + t = None + uf[n] = TableColumn(n,t) + + options.update_fields = uf + + if getattr(options,'id_field',None) and getattr(options,'replace_table',None): + logging.error("ABORT: sorry, you can't do both sync (id_field) and replace") + return + + parser = sax.make_parser() + #The "consumer" is our own handler + consumer = xml_handler(options) + #Initialize Tenorsax with handler + handler = saxtools.tenorsax(consumer) + #Resulting tenorsax instance is the SAX handler + parser.setContentHandler(handler) + parser.setFeature(sax.handler.feature_namespaces, 1) + parser.parse(options.filename) + if __name__ == "__main__": from optparse import OptionParser @@ -489,7 +583,7 @@ if __name__ == "__main__": help="replace table i.e. delete and re-insert data") opars.add_option("--backup", default=False, action="store_true", dest="backup_table", - help="create backup of old table (breaks indices)") + help="create backup of old table") opars.add_option("-d", "--debug", default=False, action="store_true", dest="debug", help="debug mode (more output)") @@ -512,43 +606,6 @@ if __name__ == "__main__": importFMPXML(options) -def importFMPXML(options): - """SAX handler to import FileMaker XML file (FMPXMLRESULT format) into the table. - @param options: dict of options - @param options.dsn: database connection string - @param options.table: name of the table the xml shall be imported into - @param options.filename: xmlfile filename - @param options.update_fields: (optional) list of fields to update; default is to create all fields - @param options.id_field: (optional) field which uniquely identifies an entry for updating purposes. - @param options.sync_mode: (optional) really synchronise, i.e. delete entries not in XML file - @param options.lc_names: (optional) lower case and clean up field names from XML - @param options.keep_fields: (optional) don't add fields to SQL database - @param options.ascii_db: (optional) assume ascii encoding in db - @param options.replace_table: (optional) delete and re-insert data - """ - update_fields = None - - if getattr(options,'update_fields',None): - uf = {} - for f in options.update_fields.split(','): - (n,t) = f.split(':') - uf[n] = TableColumn(n,t) - - options.update_fields = uf - - if getattr(options,'id_field',None) and getattr(options,'replace_table',None): - logging.error("ABORT: sorry, you can't do both sync (id_field) and replace") - sys.exit(1) - - parser = sax.make_parser() - #The "consumer" is our own handler - consumer = xml_handler(options) - #Initialize Tenorsax with handler - handler = saxtools.tenorsax(consumer) - #Resulting tenorsax instance is the SAX handler - parser.setContentHandler(handler) - parser.setFeature(sax.handler.feature_namespaces, 1) - parser.parse(options.filename) - +