ZSQLExtend/importFMPXML.py - diff

Return to importFMPXML.py CVS log

Up to [Repository] / ZSQLExtend

Diff for /ZSQLExtend/importFMPXML.py between versions 1.18 and 1.35

version 1.18, 2007/12/11 20:30:40	version 1.35, 2012/02/15 08:41:01
Line 6 import logging	Line 6 import logging
import sys	import sys
import types	import types
import time	import time
	import re

from xml import sax	from xml import sax
from amara import saxtools	from xml.sax.handler import ContentHandler
	#from amara import saxtools

try:	try:
import psycopg2 as psycopg	import psycopg2 as psycopg
	import psycopg2.extensions
	# switch to unicode
	psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
psyco = 2	psyco = 2
except:	except:
import psycopg	import psycopg
Line 19 except:	Line 24 except:

fm_ns = 'http://www.filemaker.com/fmpxmlresult'	fm_ns = 'http://www.filemaker.com/fmpxmlresult'

version_string = "V0.5 ROC 11.12.2007"	version_string = "V0.6.7 ROC 21.6.2011"

	def unicodify(text, withNone=False):
	"""decode str (utf-8 or latin-1 representation) into unicode object"""
	if withNone and text is None:
	return None
	if not text:
	return u""
	if isinstance(text, str):
	try:
	return text.decode('utf-8')
	except:
	return text.decode('latin-1')
	else:
	return text

	def utf8ify(text, withNone=False):
	"""encode unicode object or string into byte string in utf-8 representation"""
	if withNone and text is None:
	return None
	if not text:
	return ""
	if isinstance(text, unicode):
	return text.encode('utf-8')
	else:
	return text

def getTextFromNode(nodename):	def getTextFromNode(nodename):
"""get the cdata content of a node"""	"""get the cdata content of a node"""
Line 40 def sql_quote(v):	Line 70 def sql_quote(v):
v=string.join(string.split(v,dkey),quote_dict[dkey])	v=string.join(string.split(v,dkey),quote_dict[dkey])
return "'%s'"%v	return "'%s'"%v

	def sqlName(s, lc=True, more=''):
	"""returns restricted ASCII-only version of string"""
	if s is None:
	return ""

	# remove '
	s = s.replace("'","")
	# all else -> "_"
	s = re.sub('[^A-Za-z0-9_'+more+']','_',s)
	if lc:
	return s.lower()

	return s

def SimpleSearch(curs,query, args=None, ascii=False):	def SimpleSearch(curs,query, args=None, ascii=False):
"""execute sql query and return data"""	"""execute sql query and return data"""
#logger.debug("executing: "+query)	#logger.debug("executing: "+query)
if ascii:	if ascii:
# encode all in UTF-8	# encode all in UTF-8
query = query.encode("UTF-8")	query = utf8ify(query)
if args is not None:	if args is not None:
encargs = []	encargs = []
for a in args:	for a in args:
if a is not None and isinstance(a, str):	encargs.append(utf8ify(a, withNone=True))
a = a.encode("UTF-8")
encargs.append(a)

args = encargs	args = encargs

Line 84 class TableColumn:	Line 126 class TableColumn:
return self.name	return self.name


class xml_handler:	class xml_handler(ContentHandler):
def __init__(self,options):	def __init__(self,options):
"""SAX handler to import FileMaker XML file (FMPXMLRESULT format) into the table.	"""SAX handler to import FileMaker XML file (FMPXMLRESULT format) into the table.
@param options: dict of options	@param options: dict of options
Line 110 class xml_handler:	Line 152 class xml_handler:


# set up parser	# set up parser
	self.result={}
self.event = None	self.event = None
self.top_dispatcher = {
(saxtools.START_ELEMENT, fm_ns, u'METADATA'):	# self.top_dispatcher = {
self.handle_meta_fields,	# (saxtools.START_ELEMENT, fm_ns, u'METADATA'):
(saxtools.START_ELEMENT, fm_ns, u'RESULTSET'):	# self.handle_meta_fields,
self.handle_data_fields,	# (saxtools.START_ELEMENT, fm_ns, u'RESULTSET'):
}	# self.handle_data_fields,
	# }

# connect database	# connect database
self.dbCon = psycopg.connect(options.dsn)	self.dbCon = psycopg.connect(options.dsn)
	logging.debug("DB encoding: %s"%getattr(self.dbCon, 'encoding', 'UNKNOWN'))
self.db = self.dbCon.cursor()	self.db = self.dbCon.cursor()
assert self.db, "AIIEE no db cursor for %s!!"%options.dsn	assert self.db, "AIIEE no db cursor for %s!!"%options.dsn

Line 132 class xml_handler:	Line 177 class xml_handler:
self.ascii_db = getattr(options,"ascii_db",None)	self.ascii_db = getattr(options,"ascii_db",None)
self.replace_table = getattr(options,"replace_table",None)	self.replace_table = getattr(options,"replace_table",None)
self.backup_table = getattr(options,"backup_table",None)	self.backup_table = getattr(options,"backup_table",None)
	self.read_before_update = getattr(options,"read_before_update",None)
	self.debug_data = getattr(options,"debug_data",None)

self.logger.debug("dsn: "+repr(getattr(options,"dsn",None)))	self.logger.debug("dsn: "+repr(getattr(options,"dsn",None)))
self.logger.debug("table: "+repr(self.table))	self.logger.debug("table: "+repr(self.table))
Line 143 class xml_handler:	Line 190 class xml_handler:
self.logger.debug("ascii_db: "+repr(self.ascii_db))	self.logger.debug("ascii_db: "+repr(self.ascii_db))
self.logger.debug("replace_table: "+repr(self.replace_table))	self.logger.debug("replace_table: "+repr(self.replace_table))
self.logger.debug("backup_table: "+repr(self.backup_table))	self.logger.debug("backup_table: "+repr(self.backup_table))
	self.logger.debug("read_before_update: "+repr(self.read_before_update))
	self.logger.debug("debug_data: "+repr(self.debug_data))

self.dbIDs = {}	self.dbIDs = {}
self.rowcnt = 0	self.rowcnt = 0

	self.currentName = None

if self.id_field is not None:	if self.id_field is not None:
# prepare a list of ids for sync mode	# prepare a list of ids for sync mode
qstr="select %s from %s"%(self.id_field,self.table)	qstr="select %s from %s"%(self.id_field,self.table)
Line 166 class xml_handler:	Line 217 class xml_handler:

return	return

def handle_meta_fields(self, end_condition):	def startElement(self, name, attrs):
dispatcher = {	logging.debug(name)
(saxtools.START_ELEMENT, fm_ns, u'FIELD'):	if (name.lower() == "field") :
self.handle_meta_field,	self.handle_meta_field(attrs)
}	if (name.lower() == "row") :
	logging.debug("handleROW")
	self.handle_row(attrs)
	if (name.lower()=="resultset"):
	self.handle_data_fields(attrs)

	if (name.lower()=="data"):
	self.handle_data_tag(attrs);

	def endElement(self,name):
	if (name.lower() == "resultset") :
	self.currentTag=""
	self.handle_end_data_fields()
	if (name.lower() == "field") :
	self.handle_end_meta_field()
	if (name.lower() == "metadata"):
	self.handle_end_meta_fields()
	if (name.lower() == "row") :
	logging.debug("handleROW")
	self.handle_end_row()

	if (name.lower() == "col") :
	self.handle_end_col()
	def characters(self,content):

	try:
	fn = self.xml_field_names[self.colIdx]

	contentTmp = self.xml_data.get(fn,'') #gibt es schon einen Inhalt, dann dieses hinzufuegen (in einem Tag kann u.U. der characters handler mehrfach aufgerufen werden.)
	self.xml_data[fn] = contentTmp+content
	except:
	logging.debug(content)
	pass

	# if self.currentName is not None:
	# logging.debug(self.currentName+" "+content)
	# self.currentRow[self.currentName]=content;
	#
	def handle_end_meta_fields(self):
	# dispatcher = {
	# (saxtools.START_ELEMENT, fm_ns, u'FIELD'):
	# self.handle_meta_field,
	# }
#First round through the generator corresponds to the	#First round through the generator corresponds to the
#start element event	#start element event
self.logger.info("reading metadata...")	# self.logger.info("reading metadata...")
self.logger.debug("START METADATA")	# if self.debug_data:
yield None	# self.logger.debug("START METADATA")
	# #yield None

#delegate is a generator that handles all the events "within"	#delegate is a generator that handles all the events "within"
#this element	#this element
delegate = None	# delegate = None
while not self.event == end_condition:	# while not self.event == end_condition:
delegate = saxtools.tenorsax.event_loop_body(	# delegate = saxtools.tenorsax.event_loop_body(
dispatcher, delegate, self.event)	# dispatcher, delegate, self.event)
yield None	# yield None
	#
#Element closed. Wrap up	# #Element closed. Wrap up
	if self.debug_data:
self.logger.debug("END METADATA")	self.logger.debug("END METADATA")

# rename table for backup	# rename table for backup
Line 232 class xml_handler:	Line 327 class xml_handler:
qstr="select attname, format_type(pg_attribute.atttypid, pg_attribute.atttypmod) from pg_attribute, pg_class where attrelid = pg_class.oid and pg_attribute.attnum > 0 and relname = '%s'"	qstr="select attname, format_type(pg_attribute.atttypid, pg_attribute.atttypmod) from pg_attribute, pg_class where attrelid = pg_class.oid and pg_attribute.attnum > 0 and relname = '%s'"
self.sql_fields={}	self.sql_fields={}
for f in SimpleSearch(self.db, qstr%self.table):	for f in SimpleSearch(self.db, qstr%self.table):
n = f[0]	fn = f[0]
t = f[1]	ft = f[1]
#print "SQL fields: %s (%s)"%(n,t)	#print "SQL fields: %s (%s)"%(n,t)
self.sql_fields[n] = TableColumn(n,t)	self.sql_fields[fn] = TableColumn(fn,ft)

# translate id_field (SQL-name) to XML-name	# translate id_field (SQL-name) to XML-name
self.xml_id = self.sql_field_map.get(self.id_field, None)	self.xml_id = self.sql_field_map.get(self.id_field, None)
Line 284 class xml_handler:	Line 379 class xml_handler:
self.logger.debug("field %s has different type (%s vs %s)"%(f,f.getType(),sf.getType()))	self.logger.debug("field %s has different type (%s vs %s)"%(f,f.getType(),sf.getType()))
elif uf is not None:	elif uf is not None:
# add field to table	# add field to table
qstr="alter table %s add %s %s"%(self.table,uf.getName(),uf.getType())	fn = uf.getName()
	ft = uf.getType()
	qstr="alter table %s add \"%s\" %s"%(self.table,fn,ft)
self.logger.info("db add field:"+qstr)	self.logger.info("db add field:"+qstr)

if self.ascii_db and type(qstr)==types.UnicodeType:	if self.ascii_db and type(qstr)==types.UnicodeType:
Line 292 class xml_handler:	Line 389 class xml_handler:

self.db.execute(qstr)	self.db.execute(qstr)
self.dbCon.commit()	self.dbCon.commit()
	# add field to field list
	self.sql_fields[fn] = TableColumn(fn, ft)

# prepare sql statements for update (do not update id_field)	# prepare sql statements for update (do not update id_field)
setStr=string.join(["%s = %%s"%self.xml_field_map[f] for f in self.xml_update_list if f != self.xml_id], ', ')	setStr=string.join(["\"%s\" = %%s"%self.xml_field_map[f] for f in self.xml_update_list if f != self.xml_id], ', ')
self.updQuery="UPDATE %s SET %s WHERE %s = %%s"%(self.table,setStr,self.id_field)	self.updQuery="UPDATE %s SET %s WHERE \"%s\" = %%s"%(self.table,setStr,self.id_field)
	# and select (for update check)
	selStr=string.join([self.xml_field_map[f].getName() for f in self.xml_update_list if f != self.xml_id], ', ')
	self.selQuery="SELECT %s FROM %s WHERE \"%s\" = %%s"%(selStr,self.table,self.id_field)
# and insert	# and insert
fields=string.join([self.xml_field_map[x].getName() for x in self.xml_update_list], ',')	fields=string.join(["\"%s\""%self.xml_field_map[x].getName() for x in self.xml_update_list], ',')
values=string.join(['%s' for f in self.xml_update_list], ',')	values=string.join(['%s' for f in self.xml_update_list], ',')
self.addQuery="INSERT INTO %s (%s) VALUES (%s)"%(self.table,fields,values)	self.addQuery="INSERT INTO %s (%s) VALUES (%s)"%(self.table,fields,values)
self.logger.debug("update-query: "+self.updQuery)	self.logger.debug("update-query: "+self.updQuery)
	self.logger.debug("sel-query: "+self.selQuery)
self.logger.debug("add-query: "+self.addQuery)	self.logger.debug("add-query: "+self.addQuery)
return	return

def handle_meta_field(self, end_condition):	def handle_meta_field(self, attrs):
name = self.params.get((None, u'NAME'))	self.currentName = attrs.get('NAME')
yield None	#yield None
	return
	def handle_end_meta_field(self):
#Element closed. Wrap up	#Element closed. Wrap up
	name = self.currentName
if self.lc_names:	if self.lc_names:
# clean name	# clean name
sqlname = name.replace(" ","_").lower()	sqlname = sqlName(name)
else:	else:
sqlname = name	sqlname = name
self.xml_field_names.append(name)	self.xml_field_names.append(name)
Line 320 class xml_handler:	Line 426 class xml_handler:
self.logger.debug("FIELD name: "+name)	self.logger.debug("FIELD name: "+name)
return	return

def handle_data_fields(self, end_condition):	def handle_data_fields(self, attrs):
dispatcher = {
(saxtools.START_ELEMENT, fm_ns, u'ROW'):
self.handle_row,
}
#First round through the generator corresponds to the	#First round through the generator corresponds to the
#start element event	#start element event
self.logger.info("reading data...")	self.logger.info("reading data...")
	if self.debug_data:
self.logger.debug("START RESULTSET")	self.logger.debug("START RESULTSET")
self.rowcnt = 0	self.rowcnt = 0
yield None	return

	def handle_end_data_fields(self):
#delegate is a generator that handles all the events "within"	#delegate is a generator that handles all the events "within"
#this element	#this element
delegate = None
while not self.event == end_condition:
delegate = saxtools.tenorsax.event_loop_body(
dispatcher, delegate, self.event)
yield None

#Element closed. Wrap up	#Element closed. Wrap up
	if self.debug_data:
self.logger.debug("END RESULTSET")	self.logger.debug("END RESULTSET")
self.dbCon.commit()	self.dbCon.commit()

if self.sync_mode:	if self.sync_mode:
# delete unmatched entries in db	# delete unmatched entries in db
	if self.rowcnt > 0:
self.logger.info("deleting unmatched rows from db")	self.logger.info("deleting unmatched rows from db")
delQuery = "DELETE FROM %s WHERE %s = %%s"%(self.table,self.id_field)	delQuery = "DELETE FROM %s WHERE \"%s\" = %%s"%(self.table,self.id_field)
for id in self.dbIDs.keys():	for id in self.dbIDs.keys():
# find all not-updated fields	# find all not-updated fields
if self.dbIDs[id] == 0:	if self.dbIDs[id] == 0:
self.logger.info(" delete:"+id)	self.logger.info(" delete: %s"%id)
SimpleSearch(self.db, delQuery, [id], ascii=self.ascii_db)	SimpleSearch(self.db, delQuery, [id], ascii=self.ascii_db)
sys.exit(1)

elif self.dbIDs[id] > 1:	elif self.dbIDs[id] > 1:
self.logger.info(" sync: ID %s used more than once?"%id)	self.logger.info(" sync: ID %s used more than once?"%id)

self.dbCon.commit()	self.dbCon.commit()

	else:
	# safety in case we had an empty file
	self.logger.warning("no rows read! not deleting unmatched rows!")

# reinstate backup tables	# reinstate backup tables
if self.backup_table and not self.id_field:	if self.backup_table and not self.id_field:
backup_name = "%s_%s"%(self.orig_table,time.strftime('%Y_%m_%d_%H_%M_%S'))	backup_name = "%s_%s"%(self.orig_table,time.strftime('%Y_%m_%d_%H_%M_%S'))
Line 371 class xml_handler:	Line 476 class xml_handler:
self.db.execute(qstr)	self.db.execute(qstr)
self.dbCon.commit()	self.dbCon.commit()

	self.logger.info("Done (%s rows)"%self.rowcnt)
return	return

def handle_row(self, end_condition):	def handle_row(self, end_condition):
dispatcher = {
(saxtools.START_ELEMENT, fm_ns, u'COL'):	if self.debug_data:
self.handle_col,
}
self.logger.debug("START ROW")	self.logger.debug("START ROW")
self.xml_data = {}	self.xml_data = {}
self.colIdx = 0	self.colIdx = 0
yield None

	return

	def handle_end_row(self):
#delegate is a generator that handles all the events "within"	#delegate is a generator that handles all the events "within"
#this element	#this element
delegate = None
while not self.event == end_condition:
delegate = saxtools.tenorsax.event_loop_body(
dispatcher, delegate, self.event)
yield None

#Element closed. Wrap up	#Element closed. Wrap up
	if self.debug_data:
self.logger.debug("END ROW")	self.logger.debug("END ROW")
self.rowcnt += 1	self.rowcnt += 1
# process collected row data	# process collected row data
Line 400 class xml_handler:	Line 502 class xml_handler:
# synchronize by id_field	# synchronize by id_field
if self.id_field:	if self.id_field:
if self.id_type == 'integer':	if self.id_type == 'integer':
	try:
id_val = int(self.xml_data[self.xml_id])	id_val = int(self.xml_data[self.xml_id])
	except:
	pass
else:	else:
id_val = self.xml_data[self.xml_id]	id_val = self.xml_data[self.xml_id]

	if not id_val:
	# abort update
	self.logger.error("ERROR: unable to sync! emtpy id in row %s"%self.rowcnt)
	return

if id_val in self.dbIDs:	if id_val in self.dbIDs:
self.dbIDs[id_val] += 1	self.dbIDs[id_val] += 1
update=True	update=True
Line 412 class xml_handler:	Line 522 class xml_handler:
args = []	args = []
for fn in self.xml_update_list:	for fn in self.xml_update_list:
# do not update id_field	# do not update id_field
if self.id_field and fn == self.xml_id:	if update and fn == self.xml_id:
continue	continue

f = self.xml_field_map[fn]	f = self.xml_field_map[fn]
val = self.xml_data[fn]	val = self.xml_data.get(fn,None)
type = self.sql_fields[f.getName()].getType()	type = self.sql_fields[f.getName()].getType()
if type == "date" and len(val) == 0:	if type == "date" and len(val.strip()) == 0:
# empty date field	# empty date field
val = None	val = None

Line 430 class xml_handler:	Line 540 class xml_handler:

if update:	if update:
# update existing row (by id_field)	# update existing row (by id_field)
# last argument is ID match	if self.read_before_update:
args.append(id_val)	# read data
	if self.debug_data:
	self.logger.debug("update check: %s = %s"%(id_val, args))
	oldrow = SimpleSearch(self.db, self.selQuery, [id_val], ascii=self.ascii_db)
	#i = 0
	#for v in oldrow[0]:
	# logging.debug("v: %s = %s (%s)"%(v,args[i],v==args[i]))
	# i += 1
	if tuple(oldrow[0]) != tuple(args):
	# data has changed -- update
	if self.debug_data:
	self.logger.debug("really update: %s = %s"%(id_val, args))
	args.append(id_val) # last arg is id
	SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db)

	else:
	# always update
	if self.debug_data:
self.logger.debug("update: %s = %s"%(id_val, args))	self.logger.debug("update: %s = %s"%(id_val, args))
	args.append(id_val) # last arg is id
SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db)	SimpleSearch(self.db, self.updQuery, args, ascii=self.ascii_db)

else:	else:
# create new row	# create new row
	if self.debug_data:
self.logger.debug("insert: %s"%args)	self.logger.debug("insert: %s"%args)
SimpleSearch(self.db, self.addQuery, args, ascii=self.ascii_db)	SimpleSearch(self.db, self.addQuery, args, ascii=self.ascii_db)

Line 447 class xml_handler:	Line 576 class xml_handler:

return	return

def handle_col(self, end_condition):	def handle_end_col(self):
dispatcher = {
(saxtools.START_ELEMENT, fm_ns, u'DATA'):
self.handle_data_tag,
}
#print "START COL"
yield None
#delegate is a generator that handles all the events "within"
#this element
delegate = None
while not self.event == end_condition:
delegate = saxtools.tenorsax.event_loop_body(
dispatcher, delegate, self.event)
yield None
#Element closed. Wrap up
#print "END COL"
self.colIdx += 1	self.colIdx += 1
return	return

def handle_data_tag(self, end_condition):
	def handle_data_tag(self, attrs):
#print "START DATA"	#print "START DATA"
content = u''	self.content = u''
yield None	# yield None
# gather child elements	# # gather child elements
while not self.event == end_condition:	# while not self.event == end_condition:
if self.event[0] == saxtools.CHARACTER_DATA:	# if self.event[0] == saxtools.CHARACTER_DATA:
content += self.params	# content += self.params
yield None	# yield None
#Element closed. Wrap up	# #Element closed. Wrap up
fn = self.xml_field_names[self.colIdx]	# fn = self.xml_field_names[self.colIdx]
self.xml_data[fn] = content	# self.xml_data[fn] = content
return	return


Line 517 def importFMPXML(options):	Line 634 def importFMPXML(options):
#The "consumer" is our own handler	#The "consumer" is our own handler
consumer = xml_handler(options)	consumer = xml_handler(options)
#Initialize Tenorsax with handler	#Initialize Tenorsax with handler
handler = saxtools.tenorsax(consumer)	#handler = saxtools.tenorsax(consumer)
#Resulting tenorsax instance is the SAX handler	#Resulting tenorsax instance is the SAX handler
parser.setContentHandler(handler)	parser.setContentHandler(consumer)
parser.setFeature(sax.handler.feature_namespaces, 1)	#parser.setFeature(sax.handler.feature_namespaces, 1)
parser.parse(options.filename)	parser.parse(options.filename)


Line 561 if __name__ == "__main__":	Line 678 if __name__ == "__main__":
opars.add_option("--backup", default=False, action="store_true",	opars.add_option("--backup", default=False, action="store_true",
dest="backup_table",	dest="backup_table",
help="create backup of old table")	help="create backup of old table")
	opars.add_option("--read-before-update", default=False, action="store_true",
	dest="read_before_update",
	help="read all data to check if it really changed")
opars.add_option("-d", "--debug", default=False, action="store_true",	opars.add_option("-d", "--debug", default=False, action="store_true",
dest="debug",	dest="debug",
help="debug mode (more output)")	help="debug mode (more output)")
	opars.add_option("--debug-data", default=False, action="store_true",
	dest="debug_data",
	help="debug mode for data (even more output)")

(options, args) = opars.parse_args()	(options, args) = opars.parse_args()

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>

Removed from v.1.18
changed lines
	Added in v.1.35