[BACK]Return to importCDLIimglist.py CVS log [TXT] [DIR] Up to [Repository] / ZSQLExtend

Diff for /ZSQLExtend/importCDLIimglist.py between versions 1.1 and 1.3

version 1.1, 2007/12/29 19:58:00 version 1.3, 2009/01/21 11:12:36
Line 11  from importASCII import ASCII_handler Line 11  from importASCII import ASCII_handler
 from importASCII import importASCII  from importASCII import importASCII
 from importASCII import SimpleSearch  from importASCII import SimpleSearch
   
 version_string = "V0.1 ROC 4.12.2007"  version_string = "V0.2.1 ROC 21.1.2009"
   
 # mapping img_type to SQL field names  # mapping img_type to SQL field names
 imgTypeMap = {  imgTypeMap = {
Line 22  imgTypeMap = { Line 22  imgTypeMap = {
     'l':'img_l',      'l':'img_l',
     'ld':'img_ld',      'ld':'img_ld',
     'ls':'img_ls'}      'ls':'img_ls'}
   # list of fields in constant order (for SQL queries)
   imgTypes = imgTypeMap.keys()
   
 upd_fields = "fn,,img_type,id_text"  upd_fields = "fn,,img_type,id_text"
 id_field = "id_text"  id_field = "id_text"
 img_type_field = "img_type"  img_type_field = "img_type"
   
   
 def setup(self):  def setup(self):
     """specialized setup version"""      """specialized setup version"""
     ASCII_handler._setup(self)      ASCII_handler._setup(self)
     # create special updQueries for img_type fields      # create special updQuery for img_type fields
     self.updQueries = dict([(t,"UPDATE %s SET %s = %%s WHERE id_text = %%s"%(self.table,imgTypeMap[t])) for t in imgTypeMap.keys()])      setStr=string.join(["%s = %%s"%imgTypeMap[f] for f in imgTypes], ', ')
       self.updQuery = "UPDATE %s SET %s WHERE id_text = %%s"%(self.table,setStr)
       # create special delQuery for img_type fields
       delStr=string.join(["%s = null"%imgTypeMap[f] for f in imgTypes], ', ')
       self.delQuery = "UPDATE %s SET %s WHERE id_text = %%s"%(self.table,delStr)
     # text file field for img_type      # text file field for img_type
     self.xml_img_type = self.sql_field_map[img_type_field]      self.xml_img_type = self.sql_field_map[img_type_field]
       # dict of all img fields
       self.img_data = {}
   
   
 def handle_line(self, line):  def handle_line(self, line):
Line 46  def handle_line(self, line): Line 53  def handle_line(self, line):
     self.rowcnt += 1      self.rowcnt += 1
     # process collected row data      # process collected row data
     update=False      update=False
   
     # synchronize by id_field      # synchronize by id_field
     id_val = self.xml_data[self.xml_id]      id_val = self.xml_data[self.xml_id]
     if id_val in self.dbIDs:  
         self.dbIDs[id_val] += 1  
         update=True  
   
     # get img_type      # get img_type
     img_type_val = self.xml_data[self.xml_img_type]      img_type_val = self.xml_data[self.xml_img_type]
Line 63  def handle_line(self, line): Line 68  def handle_line(self, line):
         self.logger.debug("END ROW")          self.logger.debug("END ROW")
         return          return
   
     args = [fn]      # is the entry new?
       if id_val in self.dbIDs:
           self.dbIDs[id_val] += 1
           update=True
   
     if update:      if update:
         # update existing row (by id_field)          # update existing row (by id_field)
         # last argument is ID match          if id_val in self.img_data:
         args.append(id_val)              self.img_data[id_val][img_type_val] = fn
         try:          else:
             query =  self.updQueries[img_type_val]              self.img_data[id_val] = {img_type_val:fn}
         except:  
             self.logger.error("unknown image type %s"%img_type_val)  
             return  
   
         self.logger.debug("update: %s = %s"%(id_val, args))          self.logger.debug("update: %s = %s"%(id_val, args))
         SimpleSearch(self.db, query, args, ascii=self.ascii_db)  
   
     elif not self.update_mode:      elif not self.update_mode:
         # create new row          # create new row (doesn't work)
         self.logger.debug("insert: %s"%args)          self.logger.debug("insert: %s"%args)
         #SimpleSearch(self.db, self.addQuery, args, ascii=self.ascii_db)          #SimpleSearch(self.db, self.addQuery, args, ascii=self.ascii_db)
   
     #self.logger.info(" row:"+"%d (%s)"%(self.rowcnt,id_val))      #self.logger.info(" row:"+"%d (%s)"%(self.rowcnt,id_val))
     if (self.rowcnt % 100) == 0:      if (self.rowcnt % 100) == 0:
         self.logger.info(" row:"+"%d (id:%s)"%(self.rowcnt,id_val))          self.logger.info(" row:"+"%d (id:%s)"%(self.rowcnt,id_val))
         self.dbCon.commit()  
   
     self.logger.debug("END ROW")      self.logger.debug("END ROW")
     return      return
   
   
   def parse(self, filename):
       """open file and read data"""
       self.logger.info("reading data...")
       self.rowcnt = 0
   
       fh = open(filename,"r")
       self.logger.debug("BEGIN RESULTSET")
       # parse line-wise
       for line in fh:
           self.handle_line(line)
   
       # done. Wrap up
       self.logger.debug("END RESULTSET")
   
       self.logger.info("importing rows in db...")
       i = 0
       for id in self.dbIDs.keys():
           # find all fields
           if self.dbIDs[id] == 0:
               # unmatched entry
               #self.logger.debug("CLEAN: %s with %s"%(self.delQuery,id))
               SimpleSearch(self.db, self.delQuery, [id], ascii=self.ascii_db, result=False)
   
           elif self.dbIDs[id] > 0:
               # assemble query
               imgd = self.img_data.get(id, None)
               if imgd is None:
                   self.logger.error("No data for id %s while marked for update!"%id)
                   continue
               
               args = [ imgd.get(f,None) for f in imgTypes ]
               args.append(id)
               # update
               #self.logger.debug("UPDATE: %s with %s"%(self.updQuery,args))
               SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db, result=False)
   
           i += 1
           if i % 100 == 0:
               self.logger.info(" import: %d (%s)"%(i,id))
               self.dbCon.commit()
   
       self.dbCon.commit()
       # reinstate backup tables
       if self.backup_table:
           backup_name = "%s_%s"%(self.orig_table,time.strftime('%Y_%m_%d_%H_%M_%S'))
           self.logger.info("rename backup table %s to %s"%(self.orig_table,backup_name))
           qstr = "ALTER TABLE %s RENAME TO %s"%(self.orig_table,backup_name)
           self.db.execute(qstr)
           self.logger.info("rename working table %s to %s"%(self.table,self.orig_table))
           qstr = "ALTER TABLE %s RENAME TO %s"%(self.table,self.orig_table)
           self.db.execute(qstr)
           self.dbCon.commit()
   
       return
   
 # monkey patch ASCII_handler  # monkey patch ASCII_handler
 ASCII_handler._handle_line = ASCII_handler.handle_line  ASCII_handler._handle_line = ASCII_handler.handle_line
 ASCII_handler.handle_line = handle_line  ASCII_handler.handle_line = handle_line
   ASCII_handler._parse = ASCII_handler.parse
   ASCII_handler.parse = parse
 ASCII_handler._setup = ASCII_handler.setup  ASCII_handler._setup = ASCII_handler.setup
 ASCII_handler.setup = setup  ASCII_handler.setup = setup
   
Removed from v.1.1  
changed lines
  Added in v.1.3

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>