--- ZSQLExtend/importFMPXML.py 2008/02/04 19:06:34 1.22 +++ ZSQLExtend/importFMPXML.py 2008/09/05 19:05:57 1.27 @@ -12,6 +12,9 @@ from amara import saxtools try: import psycopg2 as psycopg + import psycopg2.extensions + # switch to unicode + psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) psyco = 2 except: import psycopg @@ -19,7 +22,7 @@ except: fm_ns = 'http://www.filemaker.com/fmpxmlresult' -version_string = "V0.5.2 ROC 4.2.2008" +version_string = "V0.6.1 ROC 2.7.2008" def unicodify(text, withNone=False): """decode str (utf-8 or latin-1 representation) into unicode object""" @@ -143,6 +146,7 @@ class xml_handler: # connect database self.dbCon = psycopg.connect(options.dsn) + logging.debug("DB encoding: %s"%getattr(self.dbCon, 'encoding', 'UNKNOWN')) self.db = self.dbCon.cursor() assert self.db, "AIIEE no db cursor for %s!!"%options.dsn @@ -155,6 +159,7 @@ class xml_handler: self.ascii_db = getattr(options,"ascii_db",None) self.replace_table = getattr(options,"replace_table",None) self.backup_table = getattr(options,"backup_table",None) + self.read_before_update = getattr(options,"read_before_update",None) self.logger.debug("dsn: "+repr(getattr(options,"dsn",None))) self.logger.debug("table: "+repr(self.table)) @@ -166,6 +171,7 @@ class xml_handler: self.logger.debug("ascii_db: "+repr(self.ascii_db)) self.logger.debug("replace_table: "+repr(self.replace_table)) self.logger.debug("backup_table: "+repr(self.backup_table)) + self.logger.debug("read_before_update: "+repr(self.read_before_update)) self.dbIDs = {} self.rowcnt = 0 @@ -255,10 +261,10 @@ class xml_handler: qstr="select attname, format_type(pg_attribute.atttypid, pg_attribute.atttypmod) from pg_attribute, pg_class where attrelid = pg_class.oid and pg_attribute.attnum > 0 and relname = '%s'" self.sql_fields={} for f in SimpleSearch(self.db, qstr%self.table): - n = f[0] - t = f[1] + fn = f[0] + ft = f[1] #print "SQL fields: %s (%s)"%(n,t) - self.sql_fields[n] = TableColumn(n,t) + self.sql_fields[fn] = TableColumn(fn,ft) # translate id_field (SQL-name) to XML-name self.xml_id = self.sql_field_map.get(self.id_field, None) @@ -307,7 +313,9 @@ class xml_handler: self.logger.debug("field %s has different type (%s vs %s)"%(f,f.getType(),sf.getType())) elif uf is not None: # add field to table - qstr="alter table %s add %s %s"%(self.table,uf.getName(),uf.getType()) + fn = uf.getName() + ft = uf.getType() + qstr="alter table %s add \"%s\" %s"%(self.table,fn,ft) self.logger.info("db add field:"+qstr) if self.ascii_db and type(qstr)==types.UnicodeType: @@ -315,15 +323,21 @@ class xml_handler: self.db.execute(qstr) self.dbCon.commit() + # add field to field list + self.sql_fields[fn] = TableColumn(fn, ft) # prepare sql statements for update (do not update id_field) - setStr=string.join(["%s = %%s"%self.xml_field_map[f] for f in self.xml_update_list if f != self.xml_id], ', ') - self.updQuery="UPDATE %s SET %s WHERE %s = %%s"%(self.table,setStr,self.id_field) + setStr=string.join(["\"%s\" = %%s"%self.xml_field_map[f] for f in self.xml_update_list if f != self.xml_id], ', ') + self.updQuery="UPDATE %s SET %s WHERE \"%s\" = %%s"%(self.table,setStr,self.id_field) + # and select (for update check) + selStr=string.join([self.xml_field_map[f].getName() for f in self.xml_update_list if f != self.xml_id], ', ') + self.selQuery="SELECT %s FROM %s WHERE \"%s\" = %%s"%(selStr,self.table,self.id_field) # and insert - fields=string.join([self.xml_field_map[x].getName() for x in self.xml_update_list], ',') + fields=string.join(["\"%s\""%self.xml_field_map[x].getName() for x in self.xml_update_list], ',') values=string.join(['%s' for f in self.xml_update_list], ',') self.addQuery="INSERT INTO %s (%s) VALUES (%s)"%(self.table,fields,values) self.logger.debug("update-query: "+self.updQuery) + self.logger.debug("sel-query: "+self.selQuery) self.logger.debug("add-query: "+self.addQuery) return @@ -370,13 +384,12 @@ class xml_handler: if self.sync_mode: # delete unmatched entries in db self.logger.info("deleting unmatched rows from db") - delQuery = "DELETE FROM %s WHERE %s = %%s"%(self.table,self.id_field) + delQuery = "DELETE FROM %s WHERE \"%s\" = %%s"%(self.table,self.id_field) for id in self.dbIDs.keys(): # find all not-updated fields if self.dbIDs[id] == 0: - self.logger.info(" delete:"+id) + self.logger.info(" delete: %s"%id) SimpleSearch(self.db, delQuery, [id], ascii=self.ascii_db) - sys.exit(1) elif self.dbIDs[id] > 1: self.logger.info(" sync: ID %s used more than once?"%id) @@ -453,10 +466,25 @@ class xml_handler: if update: # update existing row (by id_field) - # last argument is ID match - args.append(id_val) - self.logger.debug("update: %s = %s"%(id_val, args)) - SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db) + if self.read_before_update: + # read data + self.logger.debug("update check: %s = %s"%(id_val, args)) + oldrow = SimpleSearch(self.db, self.selQuery, [id_val], ascii=self.ascii_db) + #i = 0 + #for v in oldrow[0]: + # logging.debug("v: %s = %s (%s)"%(v,args[i],v==args[i])) + # i += 1 + if tuple(oldrow[0]) != tuple(args): + # data has changed -- update + self.logger.debug("really update: %s = %s"%(id_val, args)) + args.append(id_val) # last arg is id + SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db) + + else: + # always update + self.logger.debug("update: %s = %s"%(id_val, args)) + args.append(id_val) # last arg is id + SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db) else: # create new row @@ -584,6 +612,9 @@ if __name__ == "__main__": opars.add_option("--backup", default=False, action="store_true", dest="backup_table", help="create backup of old table") + opars.add_option("--read-before-update", default=False, action="store_true", + dest="read_before_update", + help="read all data to check if it really changed") opars.add_option("-d", "--debug", default=False, action="store_true", dest="debug", help="debug mode (more output)")