version 1.1, 2007/12/29 19:58:00
|
version 1.2, 2007/12/31 09:47:12
|
Line 11 from importASCII import ASCII_handler
|
Line 11 from importASCII import ASCII_handler
|
from importASCII import importASCII |
from importASCII import importASCII |
from importASCII import SimpleSearch |
from importASCII import SimpleSearch |
|
|
version_string = "V0.1 ROC 4.12.2007" |
version_string = "V0.2 ROC 29.12.2007" |
|
|
# mapping img_type to SQL field names |
# mapping img_type to SQL field names |
imgTypeMap = { |
imgTypeMap = { |
Line 22 imgTypeMap = {
|
Line 22 imgTypeMap = {
|
'l':'img_l', |
'l':'img_l', |
'ld':'img_ld', |
'ld':'img_ld', |
'ls':'img_ls'} |
'ls':'img_ls'} |
|
# list of fields in constant order (for SQL queries) |
|
imgTypes = imgTypeMap.keys() |
|
|
upd_fields = "fn,,img_type,id_text" |
upd_fields = "fn,,img_type,id_text" |
id_field = "id_text" |
id_field = "id_text" |
img_type_field = "img_type" |
img_type_field = "img_type" |
|
|
|
|
def setup(self): |
def setup(self): |
"""specialized setup version""" |
"""specialized setup version""" |
ASCII_handler._setup(self) |
ASCII_handler._setup(self) |
# create special updQueries for img_type fields |
# create special updQuery for img_type fields |
self.updQueries = dict([(t,"UPDATE %s SET %s = %%s WHERE id_text = %%s"%(self.table,imgTypeMap[t])) for t in imgTypeMap.keys()]) |
setStr=string.join(["%s = %%s"%imgTypeMap[f] for f in imgTypes], ', ') |
|
self.updQuery = "UPDATE %s SET %s WHERE id_text = %%s"%(self.table,setStr) |
|
# create special delQuery for img_type fields |
|
delStr=string.join(["%s = null"%imgTypeMap[f] for f in imgTypes], ', ') |
|
self.delQuery = "UPDATE %s SET %s WHERE id_text = %%s"%(self.table,delStr) |
# text file field for img_type |
# text file field for img_type |
self.xml_img_type = self.sql_field_map[img_type_field] |
self.xml_img_type = self.sql_field_map[img_type_field] |
|
# dict of all img fields |
|
self.img_data = {} |
|
|
|
|
def handle_line(self, line): |
def handle_line(self, line): |
Line 63 def handle_line(self, line):
|
Line 70 def handle_line(self, line):
|
self.logger.debug("END ROW") |
self.logger.debug("END ROW") |
return |
return |
|
|
args = [fn] |
|
|
|
if update: |
if update: |
# update existing row (by id_field) |
# update existing row (by id_field) |
# last argument is ID match |
if id_val in self.img_data: |
args.append(id_val) |
self.img_data[id_val][img_type_val] = fn |
try: |
else: |
query = self.updQueries[img_type_val] |
self.img_data[id_val] = {img_type_val:fn} |
except: |
|
self.logger.error("unknown image type %s"%img_type_val) |
|
return |
|
|
|
self.logger.debug("update: %s = %s"%(id_val, args)) |
self.logger.debug("update: %s = %s"%(id_val, args)) |
SimpleSearch(self.db, query, args, ascii=self.ascii_db) |
|
|
|
elif not self.update_mode: |
elif not self.update_mode: |
# create new row |
# create new row (doesn't work) |
self.logger.debug("insert: %s"%args) |
self.logger.debug("insert: %s"%args) |
#SimpleSearch(self.db, self.addQuery, args, ascii=self.ascii_db) |
#SimpleSearch(self.db, self.addQuery, args, ascii=self.ascii_db) |
|
|
#self.logger.info(" row:"+"%d (%s)"%(self.rowcnt,id_val)) |
#self.logger.info(" row:"+"%d (%s)"%(self.rowcnt,id_val)) |
if (self.rowcnt % 100) == 0: |
if (self.rowcnt % 100) == 0: |
self.logger.info(" row:"+"%d (id:%s)"%(self.rowcnt,id_val)) |
self.logger.info(" row:"+"%d (id:%s)"%(self.rowcnt,id_val)) |
self.dbCon.commit() |
|
|
|
self.logger.debug("END ROW") |
self.logger.debug("END ROW") |
return |
return |
|
|
|
|
|
def parse(self, filename): |
|
"""open file and read data""" |
|
self.logger.info("reading data...") |
|
self.rowcnt = 0 |
|
|
|
fh = open(filename,"r") |
|
self.logger.debug("BEGIN RESULTSET") |
|
# parse line-wise |
|
for line in fh: |
|
self.handle_line(line) |
|
|
|
# done. Wrap up |
|
self.logger.debug("END RESULTSET") |
|
|
|
self.logger.info("importing rows in db...") |
|
i = 0 |
|
for id in self.dbIDs.keys(): |
|
# find all fields |
|
if self.dbIDs[id] == 0: |
|
# unmatched entry |
|
#self.logger.debug("CLEAN: %s with %s"%(self.delQuery,id)) |
|
SimpleSearch(self.db, self.delQuery, [id], ascii=self.ascii_db, result=False) |
|
|
|
elif self.dbIDs[id] > 0: |
|
# assemble query |
|
args = [ self.img_data[id].get(f,None) for f in imgTypes ] |
|
args.append(id) |
|
# update |
|
#self.logger.debug("UPDATE: %s with %s"%(self.updQuery,args)) |
|
SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db, result=False) |
|
|
|
i += 1 |
|
if i % 100 == 0: |
|
self.logger.info(" import: %d (%s)"%(i,id)) |
|
self.dbCon.commit() |
|
|
|
self.dbCon.commit() |
|
# reinstate backup tables |
|
if self.backup_table: |
|
backup_name = "%s_%s"%(self.orig_table,time.strftime('%Y_%m_%d_%H_%M_%S')) |
|
self.logger.info("rename backup table %s to %s"%(self.orig_table,backup_name)) |
|
qstr = "ALTER TABLE %s RENAME TO %s"%(self.orig_table,backup_name) |
|
self.db.execute(qstr) |
|
self.logger.info("rename working table %s to %s"%(self.table,self.orig_table)) |
|
qstr = "ALTER TABLE %s RENAME TO %s"%(self.table,self.orig_table) |
|
self.db.execute(qstr) |
|
self.dbCon.commit() |
|
|
|
return |
|
|
# monkey patch ASCII_handler |
# monkey patch ASCII_handler |
ASCII_handler._handle_line = ASCII_handler.handle_line |
ASCII_handler._handle_line = ASCII_handler.handle_line |
ASCII_handler.handle_line = handle_line |
ASCII_handler.handle_line = handle_line |
|
ASCII_handler._parse = ASCII_handler.parse |
|
ASCII_handler.parse = parse |
ASCII_handler._setup = ASCII_handler.setup |
ASCII_handler._setup = ASCII_handler.setup |
ASCII_handler.setup = setup |
ASCII_handler.setup = setup |
|
|