version 1.2, 2007/12/31 09:47:12
|
version 1.6, 2011/02/11 19:08:49
|
Line 11 from importASCII import ASCII_handler
|
Line 11 from importASCII import ASCII_handler
|
from importASCII import importASCII |
from importASCII import importASCII |
from importASCII import SimpleSearch |
from importASCII import SimpleSearch |
|
|
version_string = "V0.2 ROC 29.12.2007" |
version_string = "V0.2.3 ROC 25.3.2010" |
|
|
# mapping img_type to SQL field names |
# mapping img_type to SQL field names |
imgTypeMap = { |
imgTypeMap = { |
'p':'img_p', |
'p':'img_p', |
'd':'img_d', |
'd':'img_d', |
|
's':'img_s', |
'e':'img_e', |
'e':'img_e', |
'ed':'img_ed', |
'ed':'img_ed', |
'l':'img_l', |
'l':'img_l', |
Line 25 imgTypeMap = {
|
Line 26 imgTypeMap = {
|
# list of fields in constant order (for SQL queries) |
# list of fields in constant order (for SQL queries) |
imgTypes = imgTypeMap.keys() |
imgTypes = imgTypeMap.keys() |
|
|
upd_fields = "fn,,img_type,id_text" |
upd_fields = "fn,,img_type,%s" |
id_field = "id_text" |
#id_field = "id_text" |
img_type_field = "img_type" |
img_type_field = "img_type" |
|
|
def setup(self): |
def setup(self): |
Line 34 def setup(self):
|
Line 35 def setup(self):
|
ASCII_handler._setup(self) |
ASCII_handler._setup(self) |
# create special updQuery for img_type fields |
# create special updQuery for img_type fields |
setStr=string.join(["%s = %%s"%imgTypeMap[f] for f in imgTypes], ', ') |
setStr=string.join(["%s = %%s"%imgTypeMap[f] for f in imgTypes], ', ') |
self.updQuery = "UPDATE %s SET %s WHERE id_text = %%s"%(self.table,setStr) |
self.updQuery = "UPDATE %s SET %s WHERE %s = %%s"%(self.table,setStr,self.id_field) |
# create special delQuery for img_type fields |
# create special delQuery for img_type fields |
delStr=string.join(["%s = null"%imgTypeMap[f] for f in imgTypes], ', ') |
delStr=string.join(["%s = null"%imgTypeMap[f] for f in imgTypes], ', ') |
self.delQuery = "UPDATE %s SET %s WHERE id_text = %%s"%(self.table,delStr) |
self.delQuery = "UPDATE %s SET %s WHERE %s = %%s"%(self.table,delStr,self.id_field) |
# text file field for img_type |
# text file field for img_type |
self.xml_img_type = self.sql_field_map[img_type_field] |
self.xml_img_type = self.sql_field_map[img_type_field] |
# dict of all img fields |
# dict of all img fields |
Line 53 def handle_line(self, line):
|
Line 54 def handle_line(self, line):
|
self.rowcnt += 1 |
self.rowcnt += 1 |
# process collected row data |
# process collected row data |
update=False |
update=False |
|
|
# synchronize by id_field |
# synchronize by id_field |
id_val = self.xml_data[self.xml_id] |
id_val = self.xml_data[self.xml_id] |
if id_val in self.dbIDs: |
#logging.debug("id_val=%s xml_id=%s"%(id_val,self.xml_id)) |
self.dbIDs[id_val] += 1 |
|
update=True |
|
|
|
# get img_type |
# get img_type |
img_type_val = self.xml_data[self.xml_img_type] |
img_type_val = self.xml_data[self.xml_img_type] |
Line 70 def handle_line(self, line):
|
Line 70 def handle_line(self, line):
|
self.logger.debug("END ROW") |
self.logger.debug("END ROW") |
return |
return |
|
|
|
# is the entry new? |
|
if id_val in self.dbIDs: |
|
self.dbIDs[id_val] += 1 |
|
update=True |
|
|
if update: |
if update: |
# update existing row (by id_field) |
# update existing row (by id_field) |
if id_val in self.img_data: |
if id_val in self.img_data: |
Line 112 def parse(self, filename):
|
Line 117 def parse(self, filename):
|
# find all fields |
# find all fields |
if self.dbIDs[id] == 0: |
if self.dbIDs[id] == 0: |
# unmatched entry |
# unmatched entry |
#self.logger.debug("CLEAN: %s with %s"%(self.delQuery,id)) |
self.logger.debug("CLEAN: %s with %s"%(self.delQuery,id)) |
SimpleSearch(self.db, self.delQuery, [id], ascii=self.ascii_db, result=False) |
SimpleSearch(self.db, self.delQuery, [id], ascii=self.ascii_db, result=False) |
|
|
elif self.dbIDs[id] > 0: |
elif self.dbIDs[id] > 0: |
# assemble query |
# assemble query |
args = [ self.img_data[id].get(f,None) for f in imgTypes ] |
imgd = self.img_data.get(id, None) |
|
if imgd is None: |
|
self.logger.error("No data for id %s while marked for update!"%id) |
|
continue |
|
|
|
args = [ imgd.get(f,None) for f in imgTypes ] |
args.append(id) |
args.append(id) |
# update |
# update |
#self.logger.debug("UPDATE: %s with %s"%(self.updQuery,args)) |
self.logger.debug("UPDATE: %s with %s"%(self.updQuery,args)) |
SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db, result=False) |
SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db, result=False) |
|
|
i += 1 |
i += 1 |
Line 164 if __name__ == "__main__":
|
Line 174 if __name__ == "__main__":
|
opars.add_option("-t", "--table", |
opars.add_option("-t", "--table", |
dest="table", |
dest="table", |
help="database table name") |
help="database table name") |
|
opars.add_option("--id-field", default="id_text", |
|
dest="id_field", |
|
help="name of id field for synchronisation", metavar="NAME") |
opars.add_option("--ascii-db", default=False, action="store_true", |
opars.add_option("--ascii-db", default=False, action="store_true", |
dest="ascii_db", |
dest="ascii_db", |
help="the SQL database stores ASCII instead of unicode") |
help="the SQL database stores ASCII instead of unicode") |
Line 197 if __name__ == "__main__":
|
Line 210 if __name__ == "__main__":
|
datefmt='%H:%M:%S') |
datefmt='%H:%M:%S') |
|
|
# fixed settings for CDLI imglist |
# fixed settings for CDLI imglist |
options.update_fields = upd_fields |
options.update_fields = upd_fields%options.id_field |
options.id_field = id_field |
#options.id_field = id_field |
options.update_mode = True |
options.update_mode = True |
|
|
importASCII(options) |
importASCII(options) |