--- ZSQLExtend/importFMPXML.py 2007/08/09 15:09:27 1.15 +++ ZSQLExtend/importFMPXML.py 2008/01/09 14:23:26 1.20 @@ -19,7 +19,32 @@ except: fm_ns = 'http://www.filemaker.com/fmpxmlresult' -version_string = "V0.4.1 ROC 9.8.2007" +version_string = "V0.5.1 ROC 9.1.2008" + +def unicodify(str, withNone=False): + """decode str (utf-8 or latin-1 representation) into unicode object""" + if withNone and str is None: + return None + if not str: + return u"" + if type(str) is StringType: + try: + return str.decode('utf-8') + except: + return str.decode('latin-1') + else: + return str + +def utf8ify(str, withNone=False): + """encode unicode object or string into byte string in utf-8 representation""" + if withNone and str is None: + return None + if not str: + return "" + if type(str) is StringType: + return str + else: + return str.encode('utf-8') def getTextFromNode(nodename): """get the cdata content of a node""" @@ -45,13 +70,11 @@ def SimpleSearch(curs,query, args=None, #logger.debug("executing: "+query) if ascii: # encode all in UTF-8 - query = query.encode("UTF-8") + query = utf8ify(query) if args is not None: encargs = [] for a in args: - if a is not None: - a = a.encode("UTF-8") - encargs.append(a) + encargs.append(utf8ify(a, withNone=True)) args = encargs @@ -191,9 +214,11 @@ class xml_handler: # rename table for backup if self.backup_table: self.orig_table = self.table - self.table = self.table + "_tmp" + self.tmp_table = self.table + "_tmp" + backup_name = "%s_%s"%(self.table,time.strftime('%Y_%m_%d_%H_%M_%S')) + # remove old temp table - qstr = "DROP TABLE %s"%(self.table) + qstr = "DROP TABLE %s"%(self.tmp_table) try: self.db.execute(qstr) except: @@ -202,12 +227,13 @@ class xml_handler: self.dbCon.commit() if self.id_field: - # sync mode -- copy table - self.logger.info("copy table %s to %s"%(self.orig_table,self.table)) - qstr = "CREATE TABLE %s AS (SELECT * FROM %s)"%(self.table,self.orig_table) + # sync mode -- copy backup table, update current table + self.logger.info("copy table %s to %s"%(self.table,backup_name)) + qstr = "CREATE TABLE %s AS (SELECT * FROM %s)"%(backup_name,self.table) else: - # rename table and create empty new one + # replace mode -- create empty tmp table, insert into tmp table + self.table = self.tmp_table self.logger.info("create empty table %s"%(self.table)) qstr = "CREATE TABLE %s AS (SELECT * FROM %s WHERE 1=0)"%(self.table,self.orig_table) @@ -224,9 +250,6 @@ class xml_handler: # try to match date style with XML self.db.execute("set datestyle to 'german'") - # translate id_field (SQL-name) to XML-name - self.xml_id = self.sql_field_map.get(self.id_field, None) - #self.logger.debug("xml-fieldnames:"+repr(self.xml_field_names)) # get list of fields and types of db table qstr="select attname, format_type(pg_attribute.atttypid, pg_attribute.atttypmod) from pg_attribute, pg_class where attrelid = pg_class.oid and pg_attribute.attnum > 0 and relname = '%s'" @@ -237,6 +260,14 @@ class xml_handler: #print "SQL fields: %s (%s)"%(n,t) self.sql_fields[n] = TableColumn(n,t) + # translate id_field (SQL-name) to XML-name + self.xml_id = self.sql_field_map.get(self.id_field, None) + # get type of id_field + if self.id_field: + self.id_type = self.sql_fields[self.id_field].getType() + else: + self.id_type = None + # check fields to update if self.update_fields is None: if self.keep_fields: @@ -257,13 +288,13 @@ class xml_handler: else: self.update_fields = self.xml_field_map - + # and translate to list of xml fields if self.lc_names: self.xml_update_list = [self.sql_field_map[x] for x in self.update_fields] else: self.xml_update_list = self.update_fields.keys() - + if not self.keep_fields: # adjust db table to fields in XML and update_fields for f in self.xml_field_map.values(): @@ -285,8 +316,8 @@ class xml_handler: self.db.execute(qstr) self.dbCon.commit() - # prepare sql statements for update - setStr=string.join(["%s = %%s"%self.xml_field_map[f] for f in self.xml_update_list], ', ') + # prepare sql statements for update (do not update id_field) + setStr=string.join(["%s = %%s"%self.xml_field_map[f] for f in self.xml_update_list if f != self.xml_id], ', ') self.updQuery="UPDATE %s SET %s WHERE %s = %%s"%(self.table,setStr,self.id_field) # and insert fields=string.join([self.xml_field_map[x].getName() for x in self.xml_update_list], ',') @@ -353,7 +384,7 @@ class xml_handler: self.dbCon.commit() # reinstate backup tables - if self.backup_table: + if self.backup_table and not self.id_field: backup_name = "%s_%s"%(self.orig_table,time.strftime('%Y_%m_%d_%H_%M_%S')) self.logger.info("rename backup table %s to %s"%(self.orig_table,backup_name)) qstr = "ALTER TABLE %s RENAME TO %s"%(self.orig_table,backup_name) @@ -391,7 +422,11 @@ class xml_handler: id_val='' # synchronize by id_field if self.id_field: - id_val = self.xml_data[self.xml_id] + if self.id_type == 'integer': + id_val = int(self.xml_data[self.xml_id]) + else: + id_val = self.xml_data[self.xml_id] + if id_val in self.dbIDs: self.dbIDs[id_val] += 1 update=True @@ -399,6 +434,10 @@ class xml_handler: # collect all values args = [] for fn in self.xml_update_list: + # do not update id_field + if update and fn == self.xml_id: + continue + f = self.xml_field_map[fn] val = self.xml_data[fn] type = self.sql_fields[f.getName()].getType() @@ -478,7 +517,7 @@ def importFMPXML(options): @param options.keep_fields: (optional) don't add fields to SQL database @param options.ascii_db: (optional) assume ascii encoding in db @param options.replace_table: (optional) delete and re-insert data - @param options.backup_table: (optional) create backup of old table (breaks indices) + @param options.backup_table: (optional) create backup of old table """ if getattr(options,'update_fields',None): @@ -544,7 +583,7 @@ if __name__ == "__main__": help="replace table i.e. delete and re-insert data") opars.add_option("--backup", default=False, action="store_true", dest="backup_table", - help="create backup of old table (breaks indices)") + help="create backup of old table") opars.add_option("-d", "--debug", default=False, action="store_true", dest="debug", help="debug mode (more output)")