--- ZSQLExtend/importFMPXML.py 2007/07/31 11:28:48 1.14 +++ ZSQLExtend/importFMPXML.py 2007/08/09 15:09:27 1.15 @@ -19,7 +19,7 @@ except: fm_ns = 'http://www.filemaker.com/fmpxmlresult' -version_string = "V0.4 ROC 29.3.2007" +version_string = "V0.4.1 ROC 9.8.2007" def getTextFromNode(nodename): """get the cdata content of a node""" @@ -42,7 +42,7 @@ def sql_quote(v): def SimpleSearch(curs,query, args=None, ascii=False): """execute sql query and return data""" - #logging.debug("executing: "+query) + #logger.debug("executing: "+query) if ascii: # encode all in UTF-8 query = query.encode("UTF-8") @@ -56,7 +56,7 @@ def SimpleSearch(curs,query, args=None, args = encargs curs.execute(query, args) - #logging.debug("sql done") + #logger.debug("sql done") try: return curs.fetchall() except: @@ -99,8 +99,16 @@ class xml_handler: @param options.ascii_db: (optional) assume ascii encoding in db @param options.replace_table: (optional) delete and re-insert data @param options.backup_table: (optional) create backup of old table (breaks indices) + @param options.use_logger_instance: (optional) use this instance of a logger """ + # set up logger + if hasattr(options, 'use_logger_instance'): + self.logger = options.use_logger_instance + else: + self.logger = logging.getLogger('db.import.fmpxml') + + # set up parser self.event = None self.top_dispatcher = { @@ -125,16 +133,16 @@ class xml_handler: self.replace_table = getattr(options,"replace_table",None) self.backup_table = getattr(options,"backup_table",None) - logging.debug("dsn: "+repr(getattr(options,"dsn",None))) - logging.debug("table: "+repr(self.table)) - logging.debug("update_fields: "+repr(self.update_fields)) - logging.debug("id_field: "+repr(self.id_field)) - logging.debug("sync_mode: "+repr(self.sync_mode)) - logging.debug("lc_names: "+repr(self.lc_names)) - logging.debug("keep_fields: "+repr(self.keep_fields)) - logging.debug("ascii_db: "+repr(self.ascii_db)) - logging.debug("replace_table: "+repr(self.replace_table)) - logging.debug("backup_table: "+repr(self.backup_table)) + self.logger.debug("dsn: "+repr(getattr(options,"dsn",None))) + self.logger.debug("table: "+repr(self.table)) + self.logger.debug("update_fields: "+repr(self.update_fields)) + self.logger.debug("id_field: "+repr(self.id_field)) + self.logger.debug("sync_mode: "+repr(self.sync_mode)) + self.logger.debug("lc_names: "+repr(self.lc_names)) + self.logger.debug("keep_fields: "+repr(self.keep_fields)) + self.logger.debug("ascii_db: "+repr(self.ascii_db)) + self.logger.debug("replace_table: "+repr(self.replace_table)) + self.logger.debug("backup_table: "+repr(self.backup_table)) self.dbIDs = {} self.rowcnt = 0 @@ -147,7 +155,7 @@ class xml_handler: self.dbIDs[id[0]] = 0; self.rowcnt += 1 - logging.info("%d entries in DB to sync"%self.rowcnt) + self.logger.info("%d entries in DB to sync"%self.rowcnt) # names of fields in XML file self.xml_field_names = [] @@ -165,7 +173,8 @@ class xml_handler: } #First round through the generator corresponds to the #start element event - logging.debug("START METADATA") + self.logger.info("reading metadata...") + self.logger.debug("START METADATA") yield None #delegate is a generator that handles all the events "within" @@ -177,7 +186,7 @@ class xml_handler: yield None #Element closed. Wrap up - logging.debug("END METADATA") + self.logger.debug("END METADATA") # rename table for backup if self.backup_table: @@ -194,12 +203,12 @@ class xml_handler: if self.id_field: # sync mode -- copy table - logging.info("copy table %s to %s"%(self.orig_table,self.table)) + self.logger.info("copy table %s to %s"%(self.orig_table,self.table)) qstr = "CREATE TABLE %s AS (SELECT * FROM %s)"%(self.table,self.orig_table) else: # rename table and create empty new one - logging.info("create empty table %s"%(self.table)) + self.logger.info("create empty table %s"%(self.table)) qstr = "CREATE TABLE %s AS (SELECT * FROM %s WHERE 1=0)"%(self.table,self.orig_table) self.db.execute(qstr) @@ -207,7 +216,7 @@ class xml_handler: # delete data from table for replace if self.replace_table: - logging.info("delete data from table %s"%(self.table)) + self.logger.info("delete data from table %s"%(self.table)) qstr = "TRUNCATE TABLE %s"%(self.table) self.db.execute(qstr) self.dbCon.commit() @@ -218,7 +227,7 @@ class xml_handler: # translate id_field (SQL-name) to XML-name self.xml_id = self.sql_field_map.get(self.id_field, None) - #logging.debug("xml-fieldnames:"+repr(self.xml_field_names)) + #self.logger.debug("xml-fieldnames:"+repr(self.xml_field_names)) # get list of fields and types of db table qstr="select attname, format_type(pg_attribute.atttypid, pg_attribute.atttypmod) from pg_attribute, pg_class where attrelid = pg_class.oid and pg_attribute.attnum > 0 and relname = '%s'" self.sql_fields={} @@ -258,17 +267,17 @@ class xml_handler: if not self.keep_fields: # adjust db table to fields in XML and update_fields for f in self.xml_field_map.values(): - logging.debug("sync-fieldname: %s"%f.getName()) + self.logger.debug("sync-fieldname: %s"%f.getName()) sf = self.sql_fields.get(f.getName(), None) uf = self.update_fields.get(f.getName(), None) if sf is not None: # name in db -- check type if f.getType() != sf.getType(): - logging.debug("field %s has different type (%s vs %s)"%(f,f.getType(),sf.getType())) + self.logger.debug("field %s has different type (%s vs %s)"%(f,f.getType(),sf.getType())) elif uf is not None: # add field to table qstr="alter table %s add %s %s"%(self.table,uf.getName(),uf.getType()) - logging.info("db add field:"+qstr) + self.logger.info("db add field:"+qstr) if self.ascii_db and type(qstr)==types.UnicodeType: qstr=qstr.encode('utf-8') @@ -283,8 +292,8 @@ class xml_handler: fields=string.join([self.xml_field_map[x].getName() for x in self.xml_update_list], ',') values=string.join(['%s' for f in self.xml_update_list], ',') self.addQuery="INSERT INTO %s (%s) VALUES (%s)"%(self.table,fields,values) - logging.debug("update-query: "+self.updQuery) - logging.debug("add-query: "+self.addQuery) + self.logger.debug("update-query: "+self.updQuery) + self.logger.debug("add-query: "+self.addQuery) return def handle_meta_field(self, end_condition): @@ -300,7 +309,7 @@ class xml_handler: # map to sql name and default text type self.xml_field_map[name] = TableColumn(sqlname, 'text') self.sql_field_map[sqlname] = name - logging.debug("FIELD name: "+name) + self.logger.debug("FIELD name: "+name) return def handle_data_fields(self, end_condition): @@ -310,7 +319,8 @@ class xml_handler: } #First round through the generator corresponds to the #start element event - logging.debug("START RESULTSET") + self.logger.info("reading data...") + self.logger.debug("START RESULTSET") self.rowcnt = 0 yield None @@ -323,32 +333,32 @@ class xml_handler: yield None #Element closed. Wrap up - logging.debug("END RESULTSET") + self.logger.debug("END RESULTSET") self.dbCon.commit() if self.sync_mode: # delete unmatched entries in db - logging.info("deleting unmatched rows from db") + self.logger.info("deleting unmatched rows from db") delQuery = "DELETE FROM %s WHERE %s = %%s"%(self.table,self.id_field) for id in self.dbIDs.keys(): # find all not-updated fields if self.dbIDs[id] == 0: - logging.info(" delete:"+id) + self.logger.info(" delete:"+id) SimpleSearch(self.db, delQuery, [id], ascii=self.ascii_db) sys.exit(1) elif self.dbIDs[id] > 1: - logging.info(" sync: ID %s used more than once?"%id) + self.logger.info(" sync: ID %s used more than once?"%id) self.dbCon.commit() # reinstate backup tables if self.backup_table: backup_name = "%s_%s"%(self.orig_table,time.strftime('%Y_%m_%d_%H_%M_%S')) - logging.info("rename backup table %s to %s"%(self.orig_table,backup_name)) + self.logger.info("rename backup table %s to %s"%(self.orig_table,backup_name)) qstr = "ALTER TABLE %s RENAME TO %s"%(self.orig_table,backup_name) self.db.execute(qstr) - logging.info("rename working table %s to %s"%(self.table,self.orig_table)) + self.logger.info("rename working table %s to %s"%(self.table,self.orig_table)) qstr = "ALTER TABLE %s RENAME TO %s"%(self.table,self.orig_table) self.db.execute(qstr) self.dbCon.commit() @@ -360,7 +370,7 @@ class xml_handler: (saxtools.START_ELEMENT, fm_ns, u'COL'): self.handle_col, } - logging.debug("START ROW") + self.logger.debug("START ROW") self.xml_data = {} self.colIdx = 0 yield None @@ -374,7 +384,7 @@ class xml_handler: yield None #Element closed. Wrap up - logging.debug("END ROW") + self.logger.debug("END ROW") self.rowcnt += 1 # process collected row data update=False @@ -406,17 +416,17 @@ class xml_handler: # update existing row (by id_field) # last argument is ID match args.append(id_val) - logging.debug("update: %s = %s"%(id_val, args)) + self.logger.debug("update: %s = %s"%(id_val, args)) SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db) else: # create new row - logging.debug("insert: %s"%args) + self.logger.debug("insert: %s"%args) SimpleSearch(self.db, self.addQuery, args, ascii=self.ascii_db) - #logging.info(" row:"+"%d (%s)"%(self.rowcnt,id_val)) - if (self.rowcnt % 10) == 0: - logging.info(" row:"+"%d (%s)"%(self.rowcnt,id_val)) + #self.logger.info(" row:"+"%d (%s)"%(self.rowcnt,id_val)) + if (self.rowcnt % 100) == 0: + self.logger.info(" row:"+"%d (id:%s)"%(self.rowcnt,id_val)) self.dbCon.commit() return @@ -470,8 +480,7 @@ def importFMPXML(options): @param options.replace_table: (optional) delete and re-insert data @param options.backup_table: (optional) create backup of old table (breaks indices) """ - - + if getattr(options,'update_fields',None): uf = {} for f in options.update_fields.split(','): @@ -486,7 +495,7 @@ def importFMPXML(options): if getattr(options,'id_field',None) and getattr(options,'replace_table',None): logging.error("ABORT: sorry, you can't do both sync (id_field) and replace") - sys.exit(1) + return parser = sax.make_parser() #The "consumer" is our own handler