--- ZSQLExtend/importFMPXML.py 2007/04/18 16:18:58 1.12 +++ ZSQLExtend/importFMPXML.py 2009/02/10 17:54:00 1.29 @@ -12,6 +12,9 @@ from amara import saxtools try: import psycopg2 as psycopg + import psycopg2.extensions + # switch to unicode + psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) psyco = 2 except: import psycopg @@ -19,7 +22,32 @@ except: fm_ns = 'http://www.filemaker.com/fmpxmlresult' -version_string = "V0.4 ROC 29.3.2007" +version_string = "V0.6.3 ROC 10.2.2009" + +def unicodify(text, withNone=False): + """decode str (utf-8 or latin-1 representation) into unicode object""" + if withNone and text is None: + return None + if not text: + return u"" + if isinstance(text, str): + try: + return text.decode('utf-8') + except: + return text.decode('latin-1') + else: + return text + +def utf8ify(text, withNone=False): + """encode unicode object or string into byte string in utf-8 representation""" + if withNone and text is None: + return None + if not text: + return "" + if isinstance(text, unicode): + return text.encode('utf-8') + else: + return text def getTextFromNode(nodename): """get the cdata content of a node""" @@ -42,21 +70,19 @@ def sql_quote(v): def SimpleSearch(curs,query, args=None, ascii=False): """execute sql query and return data""" - #logging.debug("executing: "+query) + #logger.debug("executing: "+query) if ascii: # encode all in UTF-8 - query = query.encode("UTF-8") + query = utf8ify(query) if args is not None: encargs = [] for a in args: - if a is not None: - a = a.encode("UTF-8") - encargs.append(a) + encargs.append(utf8ify(a, withNone=True)) args = encargs curs.execute(query, args) - #logging.debug("sql done") + #logger.debug("sql done") try: return curs.fetchall() except: @@ -98,8 +124,17 @@ class xml_handler: @param options.keep_fields: (optional) don't add fields to SQL database @param options.ascii_db: (optional) assume ascii encoding in db @param options.replace_table: (optional) delete and re-insert data + @param options.backup_table: (optional) create backup of old table (breaks indices) + @param options.use_logger_instance: (optional) use this instance of a logger """ + # set up logger + if hasattr(options, 'use_logger_instance'): + self.logger = options.use_logger_instance + else: + self.logger = logging.getLogger('db.import.fmpxml') + + # set up parser self.event = None self.top_dispatcher = { @@ -111,6 +146,7 @@ class xml_handler: # connect database self.dbCon = psycopg.connect(options.dsn) + logging.debug("DB encoding: %s"%getattr(self.dbCon, 'encoding', 'UNKNOWN')) self.db = self.dbCon.cursor() assert self.db, "AIIEE no db cursor for %s!!"%options.dsn @@ -123,16 +159,21 @@ class xml_handler: self.ascii_db = getattr(options,"ascii_db",None) self.replace_table = getattr(options,"replace_table",None) self.backup_table = getattr(options,"backup_table",None) + self.read_before_update = getattr(options,"read_before_update",None) + self.debug_data = getattr(options,"debug_data",None) - logging.debug("dsn: "+repr(getattr(options,"dsn",None))) - logging.debug("table: "+repr(self.table)) - logging.debug("update_fields: "+repr(self.update_fields)) - logging.debug("id_field: "+repr(self.id_field)) - logging.debug("sync_mode: "+repr(self.sync_mode)) - logging.debug("lc_names: "+repr(self.lc_names)) - logging.debug("keep_fields: "+repr(self.keep_fields)) - logging.debug("ascii_db: "+repr(self.ascii_db)) - logging.debug("replace_table: "+repr(self.replace_table)) + self.logger.debug("dsn: "+repr(getattr(options,"dsn",None))) + self.logger.debug("table: "+repr(self.table)) + self.logger.debug("update_fields: "+repr(self.update_fields)) + self.logger.debug("id_field: "+repr(self.id_field)) + self.logger.debug("sync_mode: "+repr(self.sync_mode)) + self.logger.debug("lc_names: "+repr(self.lc_names)) + self.logger.debug("keep_fields: "+repr(self.keep_fields)) + self.logger.debug("ascii_db: "+repr(self.ascii_db)) + self.logger.debug("replace_table: "+repr(self.replace_table)) + self.logger.debug("backup_table: "+repr(self.backup_table)) + self.logger.debug("read_before_update: "+repr(self.read_before_update)) + self.logger.debug("debug_data: "+repr(self.debug_data)) self.dbIDs = {} self.rowcnt = 0 @@ -145,7 +186,7 @@ class xml_handler: self.dbIDs[id[0]] = 0; self.rowcnt += 1 - logging.info("%d entries in DB to sync"%self.rowcnt) + self.logger.info("%d entries in DB to sync"%self.rowcnt) # names of fields in XML file self.xml_field_names = [] @@ -163,7 +204,8 @@ class xml_handler: } #First round through the generator corresponds to the #start element event - logging.debug("START METADATA") + self.logger.info("reading metadata...") + self.logger.debug("START METADATA") yield None #delegate is a generator that handles all the events "within" @@ -175,14 +217,16 @@ class xml_handler: yield None #Element closed. Wrap up - logging.debug("END METADATA") + self.logger.debug("END METADATA") # rename table for backup if self.backup_table: self.orig_table = self.table - self.table = self.table + "_tmp" + self.tmp_table = self.table + "_tmp" + backup_name = "%s_%s"%(self.table,time.strftime('%Y_%m_%d_%H_%M_%S')) + # remove old temp table - qstr = "DROP TABLE %s"%(self.table) + qstr = "DROP TABLE %s"%(self.tmp_table) try: self.db.execute(qstr) except: @@ -191,13 +235,14 @@ class xml_handler: self.dbCon.commit() if self.id_field: - # sync mode -- copy table - logging.info("copy table %s to %s"%(self.orig_table,self.table)) - qstr = "CREATE TABLE %s AS (SELECT * FROM %s)"%(self.table,self.orig_table) + # sync mode -- copy backup table, update current table + self.logger.info("copy table %s to %s"%(self.table,backup_name)) + qstr = "CREATE TABLE %s AS (SELECT * FROM %s)"%(backup_name,self.table) else: - # rename table and create empty new one - logging.info("create empty table %s"%(self.table)) + # replace mode -- create empty tmp table, insert into tmp table + self.table = self.tmp_table + self.logger.info("create empty table %s"%(self.table)) qstr = "CREATE TABLE %s AS (SELECT * FROM %s WHERE 1=0)"%(self.table,self.orig_table) self.db.execute(qstr) @@ -205,7 +250,7 @@ class xml_handler: # delete data from table for replace if self.replace_table: - logging.info("delete data from table %s"%(self.table)) + self.logger.info("delete data from table %s"%(self.table)) qstr = "TRUNCATE TABLE %s"%(self.table) self.db.execute(qstr) self.dbCon.commit() @@ -213,26 +258,34 @@ class xml_handler: # try to match date style with XML self.db.execute("set datestyle to 'german'") - # translate id_field (SQL-name) to XML-name - self.xml_id = self.sql_field_map.get(self.id_field, None) - - #logging.debug("xml-fieldnames:"+repr(self.xml_field_names)) + #self.logger.debug("xml-fieldnames:"+repr(self.xml_field_names)) # get list of fields and types of db table qstr="select attname, format_type(pg_attribute.atttypid, pg_attribute.atttypmod) from pg_attribute, pg_class where attrelid = pg_class.oid and pg_attribute.attnum > 0 and relname = '%s'" self.sql_fields={} for f in SimpleSearch(self.db, qstr%self.table): - n = f[0] - t = f[1] + fn = f[0] + ft = f[1] #print "SQL fields: %s (%s)"%(n,t) - self.sql_fields[n] = TableColumn(n,t) + self.sql_fields[fn] = TableColumn(fn,ft) + + # translate id_field (SQL-name) to XML-name + self.xml_id = self.sql_field_map.get(self.id_field, None) + # get type of id_field + if self.id_field: + self.id_type = self.sql_fields[self.id_field].getType() + else: + self.id_type = None # check fields to update if self.update_fields is None: if self.keep_fields: - # update existing fields - self.update_fields = self.sql_fields - - + # update all existing fields from sql (when they are in the xml file) + self.update_fields = {} + for f in self.sql_fields.keys(): + if self.sql_field_map.has_key(f): + xf = self.sql_field_map[f] + self.update_fields[f] = self.xml_field_map[xf] + else: # update all fields if self.lc_names: @@ -243,43 +296,51 @@ class xml_handler: else: self.update_fields = self.xml_field_map - + # and translate to list of xml fields if self.lc_names: self.xml_update_list = [self.sql_field_map[x] for x in self.update_fields] else: self.xml_update_list = self.update_fields.keys() - + if not self.keep_fields: # adjust db table to fields in XML and update_fields for f in self.xml_field_map.values(): - logging.debug("sync-fieldname: %s"%f.getName()) + self.logger.debug("sync-fieldname: %s"%f.getName()) sf = self.sql_fields.get(f.getName(), None) uf = self.update_fields.get(f.getName(), None) if sf is not None: # name in db -- check type if f.getType() != sf.getType(): - logging.debug("field %s has different type (%s vs %s)"%(f,f.getType(),sf.getType())) + self.logger.debug("field %s has different type (%s vs %s)"%(f,f.getType(),sf.getType())) elif uf is not None: # add field to table - qstr="alter table %s add %s %s"%(self.table,uf.getName(),uf.getType()) - logging.info("db add field:"+qstr) + fn = uf.getName() + ft = uf.getType() + qstr="alter table %s add \"%s\" %s"%(self.table,fn,ft) + self.logger.info("db add field:"+qstr) if self.ascii_db and type(qstr)==types.UnicodeType: qstr=qstr.encode('utf-8') self.db.execute(qstr) self.dbCon.commit() + # add field to field list + self.sql_fields[fn] = TableColumn(fn, ft) - # prepare sql statements for update - setStr=string.join(["%s = %%s"%self.xml_field_map[f] for f in self.xml_update_list], ', ') - self.updQuery="UPDATE %s SET %s WHERE %s = %%s"%(self.table,setStr,self.id_field) + # prepare sql statements for update (do not update id_field) + setStr=string.join(["\"%s\" = %%s"%self.xml_field_map[f] for f in self.xml_update_list if f != self.xml_id], ', ') + self.updQuery="UPDATE %s SET %s WHERE \"%s\" = %%s"%(self.table,setStr,self.id_field) + # and select (for update check) + selStr=string.join([self.xml_field_map[f].getName() for f in self.xml_update_list if f != self.xml_id], ', ') + self.selQuery="SELECT %s FROM %s WHERE \"%s\" = %%s"%(selStr,self.table,self.id_field) # and insert - fields=string.join([self.xml_field_map[x].getName() for x in self.xml_update_list], ',') + fields=string.join(["\"%s\""%self.xml_field_map[x].getName() for x in self.xml_update_list], ',') values=string.join(['%s' for f in self.xml_update_list], ',') self.addQuery="INSERT INTO %s (%s) VALUES (%s)"%(self.table,fields,values) - logging.debug("update-query: "+self.updQuery) - logging.debug("add-query: "+self.addQuery) + self.logger.debug("update-query: "+self.updQuery) + self.logger.debug("sel-query: "+self.selQuery) + self.logger.debug("add-query: "+self.addQuery) return def handle_meta_field(self, end_condition): @@ -295,7 +356,7 @@ class xml_handler: # map to sql name and default text type self.xml_field_map[name] = TableColumn(sqlname, 'text') self.sql_field_map[sqlname] = name - logging.debug("FIELD name: "+name) + self.logger.debug("FIELD name: "+name) return def handle_data_fields(self, end_condition): @@ -305,7 +366,8 @@ class xml_handler: } #First round through the generator corresponds to the #start element event - logging.debug("START RESULTSET") + self.logger.info("reading data...") + self.logger.debug("START RESULTSET") self.rowcnt = 0 yield None @@ -318,32 +380,31 @@ class xml_handler: yield None #Element closed. Wrap up - logging.debug("END RESULTSET") + self.logger.debug("END RESULTSET") self.dbCon.commit() if self.sync_mode: # delete unmatched entries in db - logging.info("deleting unmatched rows from db") - delQuery = "DELETE FROM %s WHERE %s = %%s"%(self.table,self.id_field) + self.logger.info("deleting unmatched rows from db") + delQuery = "DELETE FROM %s WHERE \"%s\" = %%s"%(self.table,self.id_field) for id in self.dbIDs.keys(): # find all not-updated fields if self.dbIDs[id] == 0: - logging.info(" delete:"+id) + self.logger.info(" delete: %s"%id) SimpleSearch(self.db, delQuery, [id], ascii=self.ascii_db) - sys.exit(1) elif self.dbIDs[id] > 1: - logging.info(" sync: ID %s used more than once?"%id) + self.logger.info(" sync: ID %s used more than once?"%id) self.dbCon.commit() # reinstate backup tables - if self.backup_table: + if self.backup_table and not self.id_field: backup_name = "%s_%s"%(self.orig_table,time.strftime('%Y_%m_%d_%H_%M_%S')) - logging.info("rename backup table %s to %s"%(self.orig_table,backup_name)) + self.logger.info("rename backup table %s to %s"%(self.orig_table,backup_name)) qstr = "ALTER TABLE %s RENAME TO %s"%(self.orig_table,backup_name) self.db.execute(qstr) - logging.info("rename working table %s to %s"%(self.table,self.orig_table)) + self.logger.info("rename working table %s to %s"%(self.table,self.orig_table)) qstr = "ALTER TABLE %s RENAME TO %s"%(self.table,self.orig_table) self.db.execute(qstr) self.dbCon.commit() @@ -355,7 +416,7 @@ class xml_handler: (saxtools.START_ELEMENT, fm_ns, u'COL'): self.handle_col, } - logging.debug("START ROW") + self.logger.debug("START ROW") self.xml_data = {} self.colIdx = 0 yield None @@ -369,14 +430,26 @@ class xml_handler: yield None #Element closed. Wrap up - logging.debug("END ROW") + self.logger.debug("END ROW") self.rowcnt += 1 # process collected row data update=False id_val='' # synchronize by id_field if self.id_field: - id_val = self.xml_data[self.xml_id] + if self.id_type == 'integer': + try: + id_val = int(self.xml_data[self.xml_id]) + except: + pass + else: + id_val = self.xml_data[self.xml_id] + + if not id_val: + # abort update + self.logger.error("ERROR: unable to sync! emtpy id in row %s"%self.rowcnt) + return + if id_val in self.dbIDs: self.dbIDs[id_val] += 1 update=True @@ -384,6 +457,10 @@ class xml_handler: # collect all values args = [] for fn in self.xml_update_list: + # do not update id_field + if update and fn == self.xml_id: + continue + f = self.xml_field_map[fn] val = self.xml_data[fn] type = self.sql_fields[f.getName()].getType() @@ -399,19 +476,38 @@ class xml_handler: if update: # update existing row (by id_field) - # last argument is ID match - args.append(id_val) - logging.debug("update: %s = %s"%(id_val, args)) - SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db) + if self.read_before_update: + # read data + if self.debug_data: + self.logger.debug("update check: %s = %s"%(id_val, args)) + oldrow = SimpleSearch(self.db, self.selQuery, [id_val], ascii=self.ascii_db) + #i = 0 + #for v in oldrow[0]: + # logging.debug("v: %s = %s (%s)"%(v,args[i],v==args[i])) + # i += 1 + if tuple(oldrow[0]) != tuple(args): + # data has changed -- update + if self.debug_data: + self.logger.debug("really update: %s = %s"%(id_val, args)) + args.append(id_val) # last arg is id + SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db) + + else: + # always update + if self.debug_data: + self.logger.debug("update: %s = %s"%(id_val, args)) + args.append(id_val) # last arg is id + SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db) else: # create new row - logging.debug("insert: %s"%args) + if self.debug_data: + self.logger.debug("insert: %s"%args) SimpleSearch(self.db, self.addQuery, args, ascii=self.ascii_db) - #logging.info(" row:"+"%d (%s)"%(self.rowcnt,id_val)) - if (self.rowcnt % 10) == 0: - logging.info(" row:"+"%d (%s)"%(self.rowcnt,id_val)) + #self.logger.info(" row:"+"%d (%s)"%(self.rowcnt,id_val)) + if (self.rowcnt % 100) == 0: + self.logger.info(" row:"+"%d (id:%s)"%(self.rowcnt,id_val)) self.dbCon.commit() return @@ -451,7 +547,7 @@ class xml_handler: def importFMPXML(options): - """SAX handler to import FileMaker XML file (FMPXMLRESULT format) into the table. + """import FileMaker XML file (FMPXMLRESULT format) into the table. @param options: dict of options @param options.dsn: database connection string @param options.table: name of the table the xml shall be imported into @@ -463,9 +559,9 @@ def importFMPXML(options): @param options.keep_fields: (optional) don't add fields to SQL database @param options.ascii_db: (optional) assume ascii encoding in db @param options.replace_table: (optional) delete and re-insert data + @param options.backup_table: (optional) create backup of old table """ - - + if getattr(options,'update_fields',None): uf = {} for f in options.update_fields.split(','): @@ -480,7 +576,7 @@ def importFMPXML(options): if getattr(options,'id_field',None) and getattr(options,'replace_table',None): logging.error("ABORT: sorry, you can't do both sync (id_field) and replace") - sys.exit(1) + return parser = sax.make_parser() #The "consumer" is our own handler @@ -529,10 +625,16 @@ if __name__ == "__main__": help="replace table i.e. delete and re-insert data") opars.add_option("--backup", default=False, action="store_true", dest="backup_table", - help="create backup of old table (breaks indices)") + help="create backup of old table") + opars.add_option("--read-before-update", default=False, action="store_true", + dest="read_before_update", + help="read all data to check if it really changed") opars.add_option("-d", "--debug", default=False, action="store_true", dest="debug", help="debug mode (more output)") + opars.add_option("--debug-data", default=False, action="store_true", + dest="debug_data", + help="debug mode for data (even more output)") (options, args) = opars.parse_args()