Annotation of ZSQLExtend/importFMPXML.py, revision 1.21

1.1       casties     1: #!/usr/local/bin/python
                      2: #
                      3: 
                      4: import string
                      5: import logging
                      6: import sys
1.7       dwinter     7: import types
1.8       casties     8: import time
1.1       casties     9: 
1.5       casties    10: from xml import sax
                     11: from amara import saxtools
                     12: 
1.2       casties    13: try:
                     14:     import psycopg2 as psycopg
                     15:     psyco = 2
                     16: except:
                     17:     import psycopg
                     18:     psyco = 1
                     19: 
1.5       casties    20: fm_ns = 'http://www.filemaker.com/fmpxmlresult'
1.1       casties    21: 
1.20      casties    22: version_string = "V0.5.1 ROC 9.1.2008"
                     23: 
1.21    ! casties    24: def unicodify(text, withNone=False):
1.20      casties    25:     """decode str (utf-8 or latin-1 representation) into unicode object"""
1.21    ! casties    26:     if withNone and text is None:
1.20      casties    27:         return None
1.21    ! casties    28:     if not text:
1.20      casties    29:         return u""
1.21    ! casties    30:     if isinstance(text, str):
1.20      casties    31:         try:
1.21    ! casties    32:             return text.decode('utf-8')
1.20      casties    33:         except:
1.21    ! casties    34:             return text.decode('latin-1')
1.20      casties    35:     else:
1.21    ! casties    36:         return text
1.20      casties    37: 
1.21    ! casties    38: def utf8ify(text, withNone=False):
1.20      casties    39:     """encode unicode object or string into byte string in utf-8 representation"""
1.21    ! casties    40:     if withNone and text is None:
1.20      casties    41:         return None
1.21    ! casties    42:     if not text:
1.20      casties    43:         return ""
1.21    ! casties    44:     if isinstance(text, str):
        !            45:         return text
1.20      casties    46:     else:
1.21    ! casties    47:         return text.encode('utf-8')
1.8       casties    48: 
1.1       casties    49: def getTextFromNode(nodename):
                     50:     """get the cdata content of a node"""
                     51:     if nodename is None:
                     52:         return ""
                     53:     nodelist=nodename.childNodes
                     54:     rc = ""
                     55:     for node in nodelist:
                     56:         if node.nodeType == node.TEXT_NODE:
                     57:            rc = rc + node.data
                     58:     return rc
                     59: 
                     60: def sql_quote(v):
                     61:     # quote dictionary
                     62:     quote_dict = {"\'": "''", "\\": "\\\\"}
                     63:     for dkey in quote_dict.keys():
                     64:         if string.find(v, dkey) >= 0:
                     65:             v=string.join(string.split(v,dkey),quote_dict[dkey])
1.5       casties    66:     return "'%s'"%v
1.1       casties    67: 
1.8       casties    68: def SimpleSearch(curs,query, args=None, ascii=False):
1.1       casties    69:     """execute sql query and return data"""
1.15      casties    70:     #logger.debug("executing: "+query)
1.8       casties    71:     if ascii:
                     72:         # encode all in UTF-8
1.20      casties    73:         query = utf8ify(query)
1.8       casties    74:         if args is not None:
                     75:             encargs = []
                     76:             for a in args:
1.20      casties    77:                 encargs.append(utf8ify(a, withNone=True))
1.8       casties    78:             
                     79:             args = encargs
1.7       dwinter    80: 
1.1       casties    81:     curs.execute(query, args)
1.15      casties    82:     #logger.debug("sql done")
1.4       casties    83:     try:
                     84:         return curs.fetchall()
                     85:     except:
                     86:         return None
1.1       casties    87: 
                     88: 
1.8       casties    89: class TableColumn:
                     90:     """simple type for storing sql column name and type"""
                     91:     
                     92:     def __init__(self, name, type=None):
                     93:         #print "new tablecolumn(%s,%s)"%(name, type)
                     94:         self.name = name
                     95:         self.type = type
                     96:         
                     97:     def getName(self):
                     98:         return self.name
                     99:     
                    100:     def getType(self):
                    101:         if self.type is not None:
                    102:             return self.type
                    103:         else:
                    104:             return "text"
1.5       casties   105: 
1.8       casties   106:     def __str__(self):
                    107:         return self.name
                    108:     
                    109:     
1.5       casties   110: class xml_handler:
1.8       casties   111:     def __init__(self,options):
1.9       dwinter   112:         """SAX handler to import FileMaker XML file (FMPXMLRESULT format) into the table.
1.8       casties   113:         @param options: dict of options
                    114:         @param options.dsn: database connection string
                    115:         @param options.table: name of the table the xml shall be imported into
                    116:         @param options.filename: xmlfile filename
                    117:         @param options.update_fields: (optional) list of fields to update; default is to create all fields
                    118:         @param options.id_field: (optional) field which uniquely identifies an entry for updating purposes.
                    119:         @param options.sync_mode: (optional) really synchronise, i.e. delete entries not in XML file
                    120:         @param options.lc_names: (optional) lower case and clean up field names from XML
                    121:         @param options.keep_fields: (optional) don't add fields to SQL database
                    122:         @param options.ascii_db: (optional) assume ascii encoding in db
                    123:         @param options.replace_table: (optional) delete and re-insert data
1.14      casties   124:         @param options.backup_table: (optional) create backup of old table (breaks indices)
1.15      casties   125:         @param options.use_logger_instance: (optional) use this instance of a logger
1.9       dwinter   126:         """
                    127:         
1.15      casties   128:         # set up logger
                    129:         if hasattr(options, 'use_logger_instance'):
                    130:             self.logger = options.use_logger_instance
                    131:         else:
                    132:             self.logger = logging.getLogger('db.import.fmpxml')
                    133: 
                    134:         
1.5       casties   135:         # set up parser
                    136:         self.event = None
                    137:         self.top_dispatcher = { 
                    138:             (saxtools.START_ELEMENT, fm_ns, u'METADATA'): 
                    139:             self.handle_meta_fields,
                    140:             (saxtools.START_ELEMENT, fm_ns, u'RESULTSET'): 
1.8       casties   141:             self.handle_data_fields,
1.5       casties   142:             }
                    143:         
                    144:         # connect database
1.8       casties   145:         self.dbCon = psycopg.connect(options.dsn)
1.5       casties   146:         self.db = self.dbCon.cursor()
1.8       casties   147:         assert self.db, "AIIEE no db cursor for %s!!"%options.dsn
1.1       casties   148:     
1.9       dwinter   149:         self.table = getattr(options,"table",None)
                    150:         self.update_fields = getattr(options,"update_fields",None)
                    151:         self.id_field = getattr(options,"id_field",None)
                    152:         self.sync_mode = getattr(options,"sync_mode",None)
                    153:         self.lc_names = getattr(options,"lc_names",None)
                    154:         self.keep_fields = getattr(options,"keep_fields",None)
                    155:         self.ascii_db = getattr(options,"ascii_db",None)
                    156:         self.replace_table = getattr(options,"replace_table",None)
                    157:         self.backup_table = getattr(options,"backup_table",None)
1.8       casties   158: 
1.15      casties   159:         self.logger.debug("dsn: "+repr(getattr(options,"dsn",None)))
                    160:         self.logger.debug("table: "+repr(self.table))
                    161:         self.logger.debug("update_fields: "+repr(self.update_fields))
                    162:         self.logger.debug("id_field: "+repr(self.id_field))
                    163:         self.logger.debug("sync_mode: "+repr(self.sync_mode))
                    164:         self.logger.debug("lc_names: "+repr(self.lc_names))
                    165:         self.logger.debug("keep_fields: "+repr(self.keep_fields))
                    166:         self.logger.debug("ascii_db: "+repr(self.ascii_db))
                    167:         self.logger.debug("replace_table: "+repr(self.replace_table))
                    168:         self.logger.debug("backup_table: "+repr(self.backup_table))
1.5       casties   169:         
                    170:         self.dbIDs = {}
                    171:         self.rowcnt = 0
1.8       casties   172:         
                    173:         if self.id_field is not None:
1.5       casties   174:             # prepare a list of ids for sync mode
1.8       casties   175:             qstr="select %s from %s"%(self.id_field,self.table)
1.5       casties   176:             for id in SimpleSearch(self.db, qstr):
                    177:                 # value 0: not updated
                    178:                 self.dbIDs[id[0]] = 0;
                    179:                 self.rowcnt += 1
                    180:                 
1.15      casties   181:             self.logger.info("%d entries in DB to sync"%self.rowcnt)
1.5       casties   182:         
1.8       casties   183:         # names of fields in XML file
                    184:         self.xml_field_names = []
                    185:         # map XML field names to SQL field names
                    186:         self.xml_field_map = {}
                    187:         # and vice versa
                    188:         self.sql_field_map = {}
1.5       casties   189:         
                    190:         return
                    191: 
                    192:     def handle_meta_fields(self, end_condition):
                    193:         dispatcher = {
                    194:             (saxtools.START_ELEMENT, fm_ns, u'FIELD'):
                    195:             self.handle_meta_field,
                    196:             }
                    197:         #First round through the generator corresponds to the
                    198:         #start element event
1.15      casties   199:         self.logger.info("reading metadata...")
                    200:         self.logger.debug("START METADATA")
1.5       casties   201:         yield None
1.1       casties   202:     
1.5       casties   203:         #delegate is a generator that handles all the events "within"
                    204:         #this element
                    205:         delegate = None
                    206:         while not self.event == end_condition:
                    207:             delegate = saxtools.tenorsax.event_loop_body(
                    208:                 dispatcher, delegate, self.event)
                    209:             yield None
                    210:         
                    211:         #Element closed. Wrap up
1.15      casties   212:         self.logger.debug("END METADATA")
1.8       casties   213:         
                    214:         # rename table for backup
                    215:         if self.backup_table:
                    216:             self.orig_table = self.table
1.16      casties   217:             self.tmp_table = self.table + "_tmp"
                    218:             backup_name = "%s_%s"%(self.table,time.strftime('%Y_%m_%d_%H_%M_%S'))
                    219:             
1.8       casties   220:             # remove old temp table
1.16      casties   221:             qstr = "DROP TABLE %s"%(self.tmp_table)
1.8       casties   222:             try:
                    223:                 self.db.execute(qstr)
                    224:             except:
                    225:                 pass
                    226:             
                    227:             self.dbCon.commit()
                    228:            
                    229:             if self.id_field:
1.16      casties   230:                 # sync mode -- copy backup table, update current table 
                    231:                 self.logger.info("copy table %s to %s"%(self.table,backup_name))
                    232:                 qstr = "CREATE TABLE %s AS (SELECT * FROM %s)"%(backup_name,self.table)
1.8       casties   233: 
                    234:             else:
1.16      casties   235:                 # replace mode -- create empty tmp table, insert into tmp table
                    236:                 self.table = self.tmp_table
1.15      casties   237:                 self.logger.info("create empty table %s"%(self.table))
1.8       casties   238:                 qstr = "CREATE TABLE %s AS (SELECT * FROM %s WHERE 1=0)"%(self.table,self.orig_table)
                    239:             
                    240:             self.db.execute(qstr)
                    241:             self.dbCon.commit()
                    242:         
                    243:         # delete data from table for replace
                    244:         if self.replace_table:
1.15      casties   245:             self.logger.info("delete data from table %s"%(self.table))
1.8       casties   246:             qstr = "TRUNCATE TABLE %s"%(self.table)
                    247:             self.db.execute(qstr)
                    248:             self.dbCon.commit()
                    249:            
                    250:         # try to match date style with XML
                    251:         self.db.execute("set datestyle to 'german'")
                    252:         
1.15      casties   253:         #self.logger.debug("xml-fieldnames:"+repr(self.xml_field_names))
1.8       casties   254:         # get list of fields and types of db table
                    255:         qstr="select attname, format_type(pg_attribute.atttypid, pg_attribute.atttypmod) from pg_attribute, pg_class where attrelid = pg_class.oid and pg_attribute.attnum > 0 and relname = '%s'"
                    256:         self.sql_fields={}
                    257:         for f in SimpleSearch(self.db, qstr%self.table):
                    258:             n = f[0]
                    259:             t = f[1]
                    260:             #print "SQL fields: %s (%s)"%(n,t)
                    261:             self.sql_fields[n] = TableColumn(n,t)
                    262:         
1.17      casties   263:         # translate id_field (SQL-name) to XML-name
                    264:         self.xml_id = self.sql_field_map.get(self.id_field, None)
                    265:         # get type of id_field
                    266:         if self.id_field:
                    267:             self.id_type = self.sql_fields[self.id_field].getType()
                    268:         else:
                    269:             self.id_type = None
                    270:         
1.8       casties   271:         # check fields to update
1.5       casties   272:         if self.update_fields is None:
1.8       casties   273:             if self.keep_fields:
1.13      casties   274:                 # update all existing fields from sql (when they are in the xml file)
                    275:                 self.update_fields = {}
                    276:                 for f in self.sql_fields.keys():
                    277:                     if self.sql_field_map.has_key(f):
                    278:                         xf = self.sql_field_map[f]
                    279:                         self.update_fields[f] = self.xml_field_map[xf]
                    280: 
1.8       casties   281:             else:
                    282:                 # update all fields
                    283:                 if self.lc_names:
                    284:                     # create dict with sql names
                    285:                     self.update_fields = {}
                    286:                     for f in self.xml_field_map.values():
                    287:                         self.update_fields[f.getName()] = f
                    288:                         
                    289:                 else:
                    290:                     self.update_fields = self.xml_field_map
1.17      casties   291:                                 
1.8       casties   292:         # and translate to list of xml fields
                    293:         if self.lc_names:
                    294:             self.xml_update_list = [self.sql_field_map[x] for x in self.update_fields]
                    295:         else:
                    296:             self.xml_update_list = self.update_fields.keys()
1.17      casties   297: 
1.8       casties   298:         if not self.keep_fields:
                    299:             # adjust db table to fields in XML and update_fields
                    300:             for f in self.xml_field_map.values():
1.15      casties   301:                 self.logger.debug("sync-fieldname: %s"%f.getName())
1.8       casties   302:                 sf = self.sql_fields.get(f.getName(), None)
                    303:                 uf = self.update_fields.get(f.getName(), None)
                    304:                 if sf is not None:
                    305:                     # name in db -- check type
                    306:                     if f.getType() != sf.getType():
1.15      casties   307:                         self.logger.debug("field %s has different type (%s vs %s)"%(f,f.getType(),sf.getType()))
1.8       casties   308:                 elif uf is not None:
                    309:                     # add field to table
                    310:                     qstr="alter table %s add %s %s"%(self.table,uf.getName(),uf.getType())
1.15      casties   311:                     self.logger.info("db add field:"+qstr)
1.8       casties   312:                     
                    313:                     if self.ascii_db and type(qstr)==types.UnicodeType:
                    314:                         qstr=qstr.encode('utf-8')
                    315:                         
                    316:                     self.db.execute(qstr)
                    317:                     self.dbCon.commit()
1.7       dwinter   318:                 
1.17      casties   319:         # prepare sql statements for update (do not update id_field)
                    320:         setStr=string.join(["%s = %%s"%self.xml_field_map[f] for f in self.xml_update_list if f != self.xml_id], ', ')
1.5       casties   321:         self.updQuery="UPDATE %s SET %s WHERE %s = %%s"%(self.table,setStr,self.id_field)
                    322:         # and insert
1.8       casties   323:         fields=string.join([self.xml_field_map[x].getName() for x in self.xml_update_list], ',')
                    324:         values=string.join(['%s' for f in self.xml_update_list], ',')
1.5       casties   325:         self.addQuery="INSERT INTO %s (%s) VALUES (%s)"%(self.table,fields,values)
1.15      casties   326:         self.logger.debug("update-query: "+self.updQuery)
                    327:         self.logger.debug("add-query: "+self.addQuery)
1.5       casties   328:         return
                    329: 
                    330:     def handle_meta_field(self, end_condition):
                    331:         name = self.params.get((None, u'NAME'))
                    332:         yield None
                    333:         #Element closed.  Wrap up
1.8       casties   334:         if self.lc_names:
                    335:             # clean name
                    336:             sqlname = name.replace(" ","_").lower() 
                    337:         else:
                    338:             sqlname = name
                    339:         self.xml_field_names.append(name)
                    340:         # map to sql name and default text type
                    341:         self.xml_field_map[name] = TableColumn(sqlname, 'text')
                    342:         self.sql_field_map[sqlname] = name
1.15      casties   343:         self.logger.debug("FIELD name: "+name)
1.5       casties   344:         return
                    345: 
1.8       casties   346:     def handle_data_fields(self, end_condition):
1.5       casties   347:         dispatcher = {
                    348:             (saxtools.START_ELEMENT, fm_ns, u'ROW'):
                    349:             self.handle_row,
                    350:             }
                    351:         #First round through the generator corresponds to the
                    352:         #start element event
1.15      casties   353:         self.logger.info("reading data...")
                    354:         self.logger.debug("START RESULTSET")
1.5       casties   355:         self.rowcnt = 0
                    356:         yield None
1.1       casties   357:     
1.5       casties   358:         #delegate is a generator that handles all the events "within"
                    359:         #this element
                    360:         delegate = None
                    361:         while not self.event == end_condition:
                    362:             delegate = saxtools.tenorsax.event_loop_body(
                    363:                 dispatcher, delegate, self.event)
                    364:             yield None
                    365:         
                    366:         #Element closed.  Wrap up
1.15      casties   367:         self.logger.debug("END RESULTSET")
1.5       casties   368:         self.dbCon.commit()
1.1       casties   369:         
1.5       casties   370:         if self.sync_mode:
                    371:             # delete unmatched entries in db
1.15      casties   372:             self.logger.info("deleting unmatched rows from db")
1.6       casties   373:             delQuery = "DELETE FROM %s WHERE %s = %%s"%(self.table,self.id_field)
1.5       casties   374:             for id in self.dbIDs.keys():
                    375:                 # find all not-updated fields
                    376:                 if self.dbIDs[id] == 0:
1.15      casties   377:                     self.logger.info(" delete:"+id)
1.8       casties   378:                     SimpleSearch(self.db, delQuery, [id], ascii=self.ascii_db)
1.6       casties   379:                     sys.exit(1)
1.1       casties   380:                     
1.5       casties   381:                 elif self.dbIDs[id] > 1:
1.15      casties   382:                     self.logger.info(" sync: ID %s used more than once?"%id)
1.8       casties   383:             
                    384:             self.dbCon.commit()
1.1       casties   385:             
1.8       casties   386:         # reinstate backup tables
1.16      casties   387:         if self.backup_table and not self.id_field:
1.8       casties   388:             backup_name = "%s_%s"%(self.orig_table,time.strftime('%Y_%m_%d_%H_%M_%S'))
1.15      casties   389:             self.logger.info("rename backup table %s to %s"%(self.orig_table,backup_name))
1.8       casties   390:             qstr = "ALTER TABLE %s RENAME TO %s"%(self.orig_table,backup_name)
                    391:             self.db.execute(qstr)
1.15      casties   392:             self.logger.info("rename working table %s to %s"%(self.table,self.orig_table))
1.8       casties   393:             qstr = "ALTER TABLE %s RENAME TO %s"%(self.table,self.orig_table)
                    394:             self.db.execute(qstr)
1.5       casties   395:             self.dbCon.commit()
1.1       casties   396:         
1.5       casties   397:         return
                    398: 
                    399:     def handle_row(self, end_condition):
                    400:         dispatcher = {
                    401:             (saxtools.START_ELEMENT, fm_ns, u'COL'):
                    402:             self.handle_col,
                    403:             }
1.15      casties   404:         self.logger.debug("START ROW")
1.8       casties   405:         self.xml_data = {}
1.5       casties   406:         self.colIdx = 0
                    407:         yield None
1.1       casties   408:     
1.5       casties   409:         #delegate is a generator that handles all the events "within"
                    410:         #this element
                    411:         delegate = None
                    412:         while not self.event == end_condition:
                    413:             delegate = saxtools.tenorsax.event_loop_body(
                    414:                 dispatcher, delegate, self.event)
                    415:             yield None
                    416:         
                    417:         #Element closed.  Wrap up
1.15      casties   418:         self.logger.debug("END ROW")
1.5       casties   419:         self.rowcnt += 1
                    420:         # process collected row data
                    421:         update=False
                    422:         id_val=''
                    423:         # synchronize by id_field
                    424:         if self.id_field:
1.17      casties   425:             if self.id_type == 'integer':
                    426:                 id_val = int(self.xml_data[self.xml_id])
                    427:             else:
                    428:                 id_val = self.xml_data[self.xml_id]
                    429:                 
1.5       casties   430:             if id_val in self.dbIDs:
                    431:                 self.dbIDs[id_val] += 1
                    432:                 update=True
1.8       casties   433: 
                    434:         # collect all values
                    435:         args = []
                    436:         for fn in self.xml_update_list:
1.17      casties   437:             # do not update id_field
1.19      casties   438:             if update and fn == self.xml_id:
1.17      casties   439:                 continue
                    440:             
1.8       casties   441:             f = self.xml_field_map[fn]
                    442:             val = self.xml_data[fn]
                    443:             type = self.sql_fields[f.getName()].getType()
                    444:             if type == "date" and len(val) == 0: 
                    445:                 # empty date field
                    446:                 val = None
                    447:                 
                    448:             elif type == "integer" and len(val) == 0: 
                    449:                 # empty int field
                    450:                 val = None
                    451:                 
                    452:             args.append(val)
                    453:                     
1.5       casties   454:         if update:
                    455:             # update existing row (by id_field)
1.8       casties   456:             # last argument is ID match
1.5       casties   457:             args.append(id_val)
1.15      casties   458:             self.logger.debug("update: %s = %s"%(id_val, args))
1.8       casties   459:             SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db)
                    460: 
1.5       casties   461:         else:
                    462:             # create new row
1.15      casties   463:             self.logger.debug("insert: %s"%args)
1.8       casties   464:             SimpleSearch(self.db, self.addQuery, args, ascii=self.ascii_db)
1.5       casties   465: 
1.15      casties   466:         #self.logger.info(" row:"+"%d (%s)"%(self.rowcnt,id_val))
                    467:         if (self.rowcnt % 100) == 0:
                    468:             self.logger.info(" row:"+"%d (id:%s)"%(self.rowcnt,id_val))
1.5       casties   469:             self.dbCon.commit()
                    470:             
                    471:         return
                    472: 
                    473:     def handle_col(self, end_condition):
                    474:         dispatcher = {
                    475:             (saxtools.START_ELEMENT, fm_ns, u'DATA'):
                    476:             self.handle_data_tag,
                    477:             }
                    478:         #print "START COL"
                    479:         yield None
                    480:         #delegate is a generator that handles all the events "within"
                    481:         #this element
                    482:         delegate = None
                    483:         while not self.event == end_condition:
                    484:             delegate = saxtools.tenorsax.event_loop_body(
                    485:                 dispatcher, delegate, self.event)
                    486:             yield None
                    487:         #Element closed.  Wrap up
                    488:         #print "END COL"
                    489:         self.colIdx += 1
                    490:         return
                    491: 
                    492:     def handle_data_tag(self, end_condition):
                    493:         #print "START DATA"
                    494:         content = u''
                    495:         yield None
                    496:         # gather child elements
                    497:         while not self.event == end_condition:
                    498:             if self.event[0] == saxtools.CHARACTER_DATA:
                    499:                 content += self.params
                    500:             yield None
                    501:         #Element closed.  Wrap up
1.8       casties   502:         fn = self.xml_field_names[self.colIdx]
                    503:         self.xml_data[fn] = content
1.5       casties   504:         return
                    505: 
                    506: 
1.11      dwinter   507: def importFMPXML(options):
1.14      casties   508:     """import FileMaker XML file (FMPXMLRESULT format) into the table.     
1.11      dwinter   509:         @param options: dict of options
                    510:         @param options.dsn: database connection string
                    511:         @param options.table: name of the table the xml shall be imported into
                    512:         @param options.filename: xmlfile filename
                    513:         @param options.update_fields: (optional) list of fields to update; default is to create all fields
                    514:         @param options.id_field: (optional) field which uniquely identifies an entry for updating purposes.
                    515:         @param options.sync_mode: (optional) really synchronise, i.e. delete entries not in XML file
                    516:         @param options.lc_names: (optional) lower case and clean up field names from XML
                    517:         @param options.keep_fields: (optional) don't add fields to SQL database
                    518:         @param options.ascii_db: (optional) assume ascii encoding in db
                    519:         @param options.replace_table: (optional) delete and re-insert data
1.16      casties   520:         @param options.backup_table: (optional) create backup of old table
1.11      dwinter   521:         """
1.15      casties   522:         
1.11      dwinter   523:     if getattr(options,'update_fields',None):
                    524:         uf = {}
                    525:         for f in options.update_fields.split(','):
1.12      casties   526:             if f.find(':') > 0:
                    527:                 (n,t) = f.split(':')
                    528:             else:
                    529:                 n = f
                    530:                 t = None
1.11      dwinter   531:             uf[n] = TableColumn(n,t)
                    532:             
                    533:         options.update_fields = uf
                    534:     
                    535:     if getattr(options,'id_field',None) and getattr(options,'replace_table',None):
                    536:         logging.error("ABORT: sorry, you can't do both sync (id_field) and replace")
1.15      casties   537:         return
1.11      dwinter   538:         
                    539:     parser = sax.make_parser()
                    540:     #The "consumer" is our own handler
                    541:     consumer = xml_handler(options)
                    542:     #Initialize Tenorsax with handler
                    543:     handler = saxtools.tenorsax(consumer)
                    544:     #Resulting tenorsax instance is the SAX handler 
                    545:     parser.setContentHandler(handler)
                    546:     parser.setFeature(sax.handler.feature_namespaces, 1)
                    547:     parser.parse(options.filename)  
                    548:     
1.1       casties   549: 
1.9       dwinter   550: if __name__ == "__main__":
                    551:     from optparse import OptionParser
1.5       casties   552: 
1.9       dwinter   553:     opars = OptionParser()
                    554:     opars.add_option("-f", "--file", 
                    555:                      dest="filename",
                    556:                      help="FMPXML file name", metavar="FILE")
                    557:     opars.add_option("-c", "--dsn", 
                    558:                      dest="dsn", 
                    559:                      help="database connection string")
                    560:     opars.add_option("-t", "--table", 
                    561:                      dest="table", 
                    562:                      help="database table name")
                    563:     opars.add_option("--fields", default=None, 
                    564:                      dest="update_fields", 
                    565:                      help="list of fields to update (comma separated, sql-names)", metavar="LIST")
                    566:     opars.add_option("--id-field", default=None, 
                    567:                      dest="id_field", 
                    568:                      help="name of id field for synchronisation (only appends data otherwise, sql-name)", metavar="NAME")
                    569:     opars.add_option("--sync", "--sync-mode", default=False, action="store_true", 
                    570:                      dest="sync_mode", 
                    571:                      help="do full sync based on id field (remove unmatched fields from db)")
                    572:     opars.add_option("--lc-names", default=False, action="store_true", 
                    573:                      dest="lc_names", 
                    574:                      help="clean and lower case field names from XML")
                    575:     opars.add_option("--keep-fields", default=False, action="store_true", 
                    576:                      dest="keep_fields", 
                    577:                      help="don't add fields from XML to SQL table")
                    578:     opars.add_option("--ascii-db", default=False, action="store_true", 
                    579:                      dest="ascii_db", 
                    580:                      help="the SQL database stores ASCII instead of unicode")
                    581:     opars.add_option("--replace", default=False, action="store_true", 
                    582:                      dest="replace_table", 
                    583:                      help="replace table i.e. delete and re-insert data")
                    584:     opars.add_option("--backup", default=False, action="store_true", 
                    585:                      dest="backup_table", 
1.16      casties   586:                      help="create backup of old table")
1.9       dwinter   587:     opars.add_option("-d", "--debug", default=False, action="store_true", 
                    588:                      dest="debug", 
                    589:                      help="debug mode (more output)")
                    590:     
                    591:     (options, args) = opars.parse_args()
                    592:     
                    593:     if len(sys.argv) < 2 or options.filename is None or options.dsn is None:
                    594:         print "importFMPXML "+version_string
                    595:         opars.print_help()
                    596:         sys.exit(1)
                    597:     
                    598:     if options.debug:
                    599:         loglevel = logging.DEBUG
                    600:     else:
                    601:         loglevel = logging.INFO
                    602:     
                    603:     logging.basicConfig(level=loglevel, 
                    604:                         format='%(asctime)s %(levelname)s %(message)s',
                    605:                         datefmt='%H:%M:%S')
                    606: 
1.11      dwinter   607:     importFMPXML(options)
                    608: 
1.9       dwinter   609: 
                    610:     
1.1       casties   611: 

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>