ZSQLExtend/importFMPXML.py - diff

Return to importFMPXML.py CVS log

Up to [Repository] / ZSQLExtend

Diff for /ZSQLExtend/importFMPXML.py between versions 1.31 and 1.34

version 1.31, 2010/05/10 15:50:32	version 1.34, 2012/02/15 08:08:42
Line 6 import logging	Line 6 import logging
import sys	import sys
import types	import types
import time	import time
	import re

from xml import sax	from xml import sax
from amara import saxtools	from xml.sax.handler import ContentHandler
	#from amara import saxtools

try:	try:
import psycopg2 as psycopg	import psycopg2 as psycopg
Line 22 except:	Line 24 except:

fm_ns = 'http://www.filemaker.com/fmpxmlresult'	fm_ns = 'http://www.filemaker.com/fmpxmlresult'

version_string = "V0.6.5 ROC 10.5.2010"	version_string = "V0.6.7 ROC 21.6.2011"

def unicodify(text, withNone=False):	def unicodify(text, withNone=False):
"""decode str (utf-8 or latin-1 representation) into unicode object"""	"""decode str (utf-8 or latin-1 representation) into unicode object"""
Line 68 def sql_quote(v):	Line 70 def sql_quote(v):
v=string.join(string.split(v,dkey),quote_dict[dkey])	v=string.join(string.split(v,dkey),quote_dict[dkey])
return "'%s'"%v	return "'%s'"%v

	def sqlName(s, lc=True, more=''):
	"""returns restricted ASCII-only version of string"""
	if s is None:
	return ""

	# remove '
	s = s.replace("'","")
	# all else -> "_"
	s = re.sub('[^A-Za-z0-9_'+more+']','_',s)
	if lc:
	return s.lower()

	return s

def SimpleSearch(curs,query, args=None, ascii=False):	def SimpleSearch(curs,query, args=None, ascii=False):
"""execute sql query and return data"""	"""execute sql query and return data"""
#logger.debug("executing: "+query)	#logger.debug("executing: "+query)
Line 110 class TableColumn:	Line 126 class TableColumn:
return self.name	return self.name


class xml_handler:	class xml_handler(ContentHandler):
def __init__(self,options):	def __init__(self,options):
"""SAX handler to import FileMaker XML file (FMPXMLRESULT format) into the table.	"""SAX handler to import FileMaker XML file (FMPXMLRESULT format) into the table.
@param options: dict of options	@param options: dict of options
Line 136 class xml_handler:	Line 152 class xml_handler:


# set up parser	# set up parser
	self.result={}
self.event = None	self.event = None
self.top_dispatcher = {
(saxtools.START_ELEMENT, fm_ns, u'METADATA'):	# self.top_dispatcher = {
self.handle_meta_fields,	# (saxtools.START_ELEMENT, fm_ns, u'METADATA'):
(saxtools.START_ELEMENT, fm_ns, u'RESULTSET'):	# self.handle_meta_fields,
self.handle_data_fields,	# (saxtools.START_ELEMENT, fm_ns, u'RESULTSET'):
}	# self.handle_data_fields,
	# }

# connect database	# connect database
self.dbCon = psycopg.connect(options.dsn)	self.dbCon = psycopg.connect(options.dsn)
Line 178 class xml_handler:	Line 196 class xml_handler:
self.dbIDs = {}	self.dbIDs = {}
self.rowcnt = 0	self.rowcnt = 0

	self.currentName = None

if self.id_field is not None:	if self.id_field is not None:
# prepare a list of ids for sync mode	# prepare a list of ids for sync mode
qstr="select %s from %s"%(self.id_field,self.table)	qstr="select %s from %s"%(self.id_field,self.table)
Line 197 class xml_handler:	Line 217 class xml_handler:

return	return

def handle_meta_fields(self, end_condition):	def startElement(self, name, attrs):
dispatcher = {	logging.debug(name)
(saxtools.START_ELEMENT, fm_ns, u'FIELD'):	if (name.lower() == "field") :
self.handle_meta_field,	self.handle_meta_field(attrs)
}	if (name.lower() == "row") :
	logging.debug("handleROW")
	self.handle_row(attrs)
	if (name.lower()=="resultset"):
	self.handle_data_fields(attrs)

	if (name.lower()=="data"):
	self.handle_data_tag(attrs);

	def endElement(self,name):
	if (name.lower() == "resultset") :
	self.currentTag=""
	self.handle_end_data_fields()
	if (name.lower() == "field") :
	self.handle_end_meta_field()
	if (name.lower() == "metadata"):
	self.handle_end_meta_fields()
	if (name.lower() == "row") :
	logging.debug("handleROW")
	self.handle_end_row()

	if (name.lower() == "col") :
	self.handle_end_col()
	def characters(self,content):

	try:
	fn = self.xml_field_names[self.colIdx]
	self.xml_data[fn] = content
	except:
	logging.debug(content)
	pass

	# if self.currentName is not None:
	# logging.debug(self.currentName+" "+content)
	# self.currentRow[self.currentName]=content;
	#
	def handle_end_meta_fields(self):
	# dispatcher = {
	# (saxtools.START_ELEMENT, fm_ns, u'FIELD'):
	# self.handle_meta_field,
	# }
#First round through the generator corresponds to the	#First round through the generator corresponds to the
#start element event	#start element event
self.logger.info("reading metadata...")	# self.logger.info("reading metadata...")
if self.debug_data:	# if self.debug_data:
self.logger.debug("START METADATA")	# self.logger.debug("START METADATA")
yield None	# #yield None

#delegate is a generator that handles all the events "within"	#delegate is a generator that handles all the events "within"
#this element	#this element
delegate = None	# delegate = None
while not self.event == end_condition:	# while not self.event == end_condition:
delegate = saxtools.tenorsax.event_loop_body(	# delegate = saxtools.tenorsax.event_loop_body(
dispatcher, delegate, self.event)	# dispatcher, delegate, self.event)
yield None	# yield None
	#
#Element closed. Wrap up	# #Element closed. Wrap up
if self.debug_data:	if self.debug_data:
self.logger.debug("END METADATA")	self.logger.debug("END METADATA")

Line 345 class xml_handler:	Line 405 class xml_handler:
self.logger.debug("add-query: "+self.addQuery)	self.logger.debug("add-query: "+self.addQuery)
return	return

def handle_meta_field(self, end_condition):	def handle_meta_field(self, attrs):
name = self.params.get((None, u'NAME'))	self.currentName = attrs.get('NAME')
yield None	#yield None
	return
	def handle_end_meta_field(self):
#Element closed. Wrap up	#Element closed. Wrap up
	name = self.currentName
if self.lc_names:	if self.lc_names:
# clean name	# clean name
sqlname = name.replace(" ","_").lower()	sqlname = sqlName(name)
else:	else:
sqlname = name	sqlname = name
self.xml_field_names.append(name)	self.xml_field_names.append(name)
Line 361 class xml_handler:	Line 424 class xml_handler:
self.logger.debug("FIELD name: "+name)	self.logger.debug("FIELD name: "+name)
return	return

def handle_data_fields(self, end_condition):	def handle_data_fields(self, attrs):
dispatcher = {
(saxtools.START_ELEMENT, fm_ns, u'ROW'):
self.handle_row,
}
#First round through the generator corresponds to the	#First round through the generator corresponds to the
#start element event	#start element event
self.logger.info("reading data...")	self.logger.info("reading data...")
if self.debug_data:	if self.debug_data:
self.logger.debug("START RESULTSET")	self.logger.debug("START RESULTSET")
self.rowcnt = 0	self.rowcnt = 0
yield None	return

	def handle_end_data_fields(self):
#delegate is a generator that handles all the events "within"	#delegate is a generator that handles all the events "within"
#this element	#this element
delegate = None
while not self.event == end_condition:
delegate = saxtools.tenorsax.event_loop_body(
dispatcher, delegate, self.event)
yield None

#Element closed. Wrap up	#Element closed. Wrap up
if self.debug_data:	if self.debug_data:
Line 389 class xml_handler:	Line 445 class xml_handler:

if self.sync_mode:	if self.sync_mode:
# delete unmatched entries in db	# delete unmatched entries in db
	if self.rowcnt > 0:
self.logger.info("deleting unmatched rows from db")	self.logger.info("deleting unmatched rows from db")
delQuery = "DELETE FROM %s WHERE \"%s\" = %%s"%(self.table,self.id_field)	delQuery = "DELETE FROM %s WHERE \"%s\" = %%s"%(self.table,self.id_field)
for id in self.dbIDs.keys():	for id in self.dbIDs.keys():
Line 402 class xml_handler:	Line 459 class xml_handler:

self.dbCon.commit()	self.dbCon.commit()

	else:
	# safety in case we had an empty file
	self.logger.warning("no rows read! not deleting unmatched rows!")

# reinstate backup tables	# reinstate backup tables
if self.backup_table and not self.id_field:	if self.backup_table and not self.id_field:
backup_name = "%s_%s"%(self.orig_table,time.strftime('%Y_%m_%d_%H_%M_%S'))	backup_name = "%s_%s"%(self.orig_table,time.strftime('%Y_%m_%d_%H_%M_%S'))
Line 417 class xml_handler:	Line 478 class xml_handler:
return	return

def handle_row(self, end_condition):	def handle_row(self, end_condition):
dispatcher = {
(saxtools.START_ELEMENT, fm_ns, u'COL'):
self.handle_col,
}
if self.debug_data:	if self.debug_data:
self.logger.debug("START ROW")	self.logger.debug("START ROW")
self.xml_data = {}	self.xml_data = {}
self.colIdx = 0	self.colIdx = 0
yield None

	return

	def handle_end_row(self):
#delegate is a generator that handles all the events "within"	#delegate is a generator that handles all the events "within"
#this element	#this element
delegate = None
while not self.event == end_condition:
delegate = saxtools.tenorsax.event_loop_body(
dispatcher, delegate, self.event)
yield None

#Element closed. Wrap up	#Element closed. Wrap up
if self.debug_data:	if self.debug_data:
Line 469 class xml_handler:	Line 524 class xml_handler:
continue	continue

f = self.xml_field_map[fn]	f = self.xml_field_map[fn]
val = self.xml_data[fn]	val = self.xml_data.get(fn,None)
type = self.sql_fields[f.getName()].getType()	type = self.sql_fields[f.getName()].getType()
if type == "date" and len(val.strip()) == 0:	if type == "date" and len(val.strip()) == 0:
# empty date field	# empty date field
Line 519 class xml_handler:	Line 574 class xml_handler:

return	return

def handle_col(self, end_condition):	def handle_end_col(self):
dispatcher = {
(saxtools.START_ELEMENT, fm_ns, u'DATA'):
self.handle_data_tag,
}
#print "START COL"
yield None
#delegate is a generator that handles all the events "within"
#this element
delegate = None
while not self.event == end_condition:
delegate = saxtools.tenorsax.event_loop_body(
dispatcher, delegate, self.event)
yield None
#Element closed. Wrap up
#print "END COL"
self.colIdx += 1	self.colIdx += 1
return	return

def handle_data_tag(self, end_condition):
	def handle_data_tag(self, attrs):
#print "START DATA"	#print "START DATA"
content = u''	self.content = u''
yield None	# yield None
# gather child elements	# # gather child elements
while not self.event == end_condition:	# while not self.event == end_condition:
if self.event[0] == saxtools.CHARACTER_DATA:	# if self.event[0] == saxtools.CHARACTER_DATA:
content += self.params	# content += self.params
yield None	# yield None
#Element closed. Wrap up	# #Element closed. Wrap up
fn = self.xml_field_names[self.colIdx]	# fn = self.xml_field_names[self.colIdx]
self.xml_data[fn] = content	# self.xml_data[fn] = content
return	return


Line 589 def importFMPXML(options):	Line 632 def importFMPXML(options):
#The "consumer" is our own handler	#The "consumer" is our own handler
consumer = xml_handler(options)	consumer = xml_handler(options)
#Initialize Tenorsax with handler	#Initialize Tenorsax with handler
handler = saxtools.tenorsax(consumer)	#handler = saxtools.tenorsax(consumer)
#Resulting tenorsax instance is the SAX handler	#Resulting tenorsax instance is the SAX handler
parser.setContentHandler(handler)	parser.setContentHandler(consumer)
parser.setFeature(sax.handler.feature_namespaces, 1)	#parser.setFeature(sax.handler.feature_namespaces, 1)
parser.parse(options.filename)	parser.parse(options.filename)

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>

Removed from v.1.31
changed lines
	Added in v.1.34