--- ZSQLExtend/ZSQLExtend.py 2006/04/20 17:40:11 1.81 +++ ZSQLExtend/ZSQLExtend.py 2006/12/14 17:10:23 1.99 @@ -17,9 +17,15 @@ import Shared.DC.ZRDB.DA import zLOG import os.path import os +import copy +import unicodedata + from OFS.SimpleItem import SimpleItem def getTextFromNode(nodename): + """get the cdata content of a node""" + if nodename is None: + return "" nodelist=nodename.childNodes rc = "" for node in nodelist: @@ -27,7 +33,6 @@ def getTextFromNode(nodename): rc = rc + node.data return rc - def analyseIntSearch(word): #analyse integer searches @@ -80,6 +85,29 @@ class ZSQLExtendFolder(Folder,Persistent """quote str for sql""" return sql_quote(str) + + def normalizeField(self,table,fieldname, newFieldName=None,mode="alter", RESPONSE=None): + """normalize a field""" + import unicodedata + + if not newFieldName: + newFieldName=fieldname+"_normal" + + def normal(str): + if str: + return unicodedata.normalize('NFKD', str.decode('utf-8')).encode('ASCII', 'ignore') + else: + return "" + if mode=="create": # create the field + qstr="""alter table %s add %s %s""" + self.ZSQLSimpleSearch(qstr%(table,newFieldName,'text')) + + qstr="select oid,%s from %s"%(fieldname,table) + for result in self.ZSQLSimpleSearch(qstr): + qstr="update %s set %s = %s where oid = %s" + + self.ZSQLSimpleSearch(qstr%(table,newFieldName,self.ZSQLQuote(normal(getattr(result,fieldname))),result.oid)) + def importAccessModell(self,configFileName,RESPONSE=None): """import tables from access @param configFileName: xml-configfile @@ -102,7 +130,7 @@ class ZSQLExtendFolder(Folder,Persistent containers=db.xpath("./@container") identifiers=db.xpath("./@identify") - print containers + if not (len(containers)==1): return False else: @@ -124,6 +152,7 @@ class ZSQLExtendFolder(Folder,Persistent @param elementNameForTable: must be a element of type complex type. the element of the sequence in the complex type define the columns of the table >table< @param data (optional): data to be imported + @param filename (optional) or filename @param identify: (optional) field res. tag which identifies a entry uniquely for updating purposes. @param RESPONSE: (optional) @@ -164,7 +193,7 @@ class ZSQLExtendFolder(Folder,Persistent #check if table exists qstr="""select relname from pg_class where relname = '%s'"""%table - if len (self.ZSQLSimpleSearch(qstr))<1: # if not the create the table + if not(self.ZSQLSimpleSearch(qstr)) or (len (self.ZSQLSimpleSearch(qstr))<1): # if not the create the table columns=[] create=True else: @@ -240,7 +269,7 @@ class ZSQLExtendFolder(Folder,Persistent if col.nodeType is col.ELEMENT_NODE: data=col.nodeName dataSet[data]=getTextFromNode(col) - print dataSet.keys() + update=False if identify: @@ -269,7 +298,7 @@ class ZSQLExtendFolder(Folder,Persistent fields=",".join(dataSet.keys()) - values=",".join([""" %s """%self.ZSQLQuote(x) for x in dataSet.keys()]) + values=",".join([""" %s """%self.ZSQLQuote(dataSet[x]) for x in dataSet.keys()]) queryStr="""INSERT INTO %s (%s) VALUES (%s)"""%(table,fields,values) @@ -288,36 +317,20 @@ class ZSQLExtendFolder(Folder,Persistent Import XML file into the table @param table: name of the table the xml shall be imported into @param containerTagName: XML-Tag which describes a dataset - @param data: data to be imported + @param file: xmlfile handle @param identify: (optional) field res. tag which identifies a entry uniquely for updating purposes. @param RESPONSE: (optional) ''' - from xml.dom.pulldom import parseString,parse - - zLOG.LOG("import xml",zLOG.INFO,"called") - #fh=file("/tmp/fmpxml.xml") - import bz2 - import base64 - - ret="" - if data: - data=bz2.decompress(base64.decodestring(data)) - - zLOG.LOG("import xml",zLOG.INFO,"received file") - doc=parseString(data) - zLOG.LOG("import xml",zLOG.INFO,"parsed file") + from xml.dom.pulldom import parseString - elif filename: - fh=file(filename) - doc=parse(fh) - zLOG.LOG("import xml",zLOG.INFO,"parsed file") + doc=parseString(file.read()) while 1: node=doc.getEvent() - + if node is None: break; else: - if node[1].nodeName=='ROW': + if node[1].nodeName==containerTagName: doc.expandNode(node[1]) cols=node[1].getElementsByTagName('COL') dataSet=[] @@ -383,8 +396,151 @@ class ZSQLExtendFolder(Folder,Persistent #nf=file("xtf/"+fn+".xtf",'w') #nf.write(""""""+node[1].toxml()+"") #print "wrote: %s"%fn - return ret - + + + def importXMLFileFMP(self,table,data=None,filename=None,update_fields=None,id_field=None,sync_mode=False,RESPONSE=None): + ''' + Import FileMaker XML file (FMPXMLRESULT format) into the table. + @param table: name of the table the xml shall be imported into + @param data: xml data as bz2 string + @param filename: xmlfile filename + @param update_fields: (optional) list of fields to update; default is to create all fields + @param id_field: (optional) field which uniquely identifies an entry for updating purposes. + @param sync_mode: (optional) really synchronise, i.e. delete entries not in XML file + @param RESPONSE: (optional) + ''' + from xml.dom.pulldom import parseString,parse + import transaction + + ret = "" + + if data: + data=bz2.decompress(base64.decodestring(data)) + zLOG.LOG("fmpxml",zLOG.INFO,"received file") + doc=parseString(data) + zLOG.LOG("fmpxml",zLOG.INFO,"parsed file") + + elif filename: + fh=file(filename) + zLOG.LOG("fmpxml",zLOG.INFO,"reading file") + doc=parse(fh) + zLOG.LOG("fmpxml",zLOG.INFO,"parsed file") + + dbIDs = {} + rowcnt = 0 + + if id_field is not None: + # prepare a list of ids for sync mode + qstr="select %s from %s"%(id_field,table) + for id in self.ZSQLSimpleSearch(qstr): + # value 0: not updated + dbIDs[id[0]] = 0; + rowcnt += 1 + + zLOG.LOG("fmpxml",zLOG.INFO,"%d entries in DB to sync"%rowcnt) + + fieldNames = [] + rowcnt = 0 + id_val = '' + + while 1: + node=doc.getEvent() + + if node is None: + break; + + # METADATA tag defines number and names of fields in FMPXMLRESULT + if node[1].nodeName == 'METADATA': + doc.expandNode(node[1]) + + names=node[1].getElementsByTagName('FIELD') + + for name in names: + fn = name.getAttribute('NAME') + fieldNames.append(fn) + + if update_fields is None: + # update all fields + update_fields = fieldNames + + zLOG.LOG("fmpxml fieldnames:",zLOG.INFO,repr(fieldNames)) + # get list of fields in db table + qstr="""select attname from pg_attribute, pg_class where attrelid = pg_class.oid and relname = '%s'""" + columns=[x.attname for x in self.ZSQLSimpleSearch(qstr%table)] + + # adjust db table to fields in XML and fieldlist + for fieldName in fieldNames: + zLOG.LOG("fmpxml fieldname:",zLOG.INFO,repr(fieldName)) + if (fieldName not in columns) and (fieldName in update_fields): + qstr="""alter table %s add %s %s""" + self.ZSQLSimpleSearch(qstr%(table,fieldName,'text')) + zLOG.LOG("fmpxml add field:",zLOG.INFO,qstr%(table,fieldName,'text')) + + # ROW tags (in RESULTSET tag) hold data + elif node[1].nodeName == 'ROW': + rowcnt += 1 + + doc.expandNode(node[1]) + cols=node[1].getElementsByTagName('COL') + dataSet={} + i = 0 + # populate with data + for col in cols: + data=col.getElementsByTagName('DATA') + dataSet[fieldNames[i]] = getTextFromNode(data[0]) + i += 1 + + update=False + + # synchronize by id_field + if id_field: + id_val=dataSet[id_field] + if id_val in dbIDs: + dbIDs[id_val] += 1 + update=True + + if update: + # update existing row (by id_field) + setvals=[] + for fieldName in update_fields: + setvals.append("%s = %s"%(fieldName,self.ZSQLQuote(dataSet[fieldName]))) + setStr=string.join(setvals, ',') + id_val=dataSet[id_field] + qstr="""UPDATE %s SET %s WHERE %s = '%s' """%(table,setStr,id_field,id_val) + #zLOG.LOG("fmpxml update:",zLOG.INFO,queryStr) + self.ZSQLSimpleSearch(qstr) + ret+="up: %s \n"%id_val + else: + # create new row + fields=string.join(update_fields, ',') + values=string.join([" %s "%self.ZSQLQuote(dataSet[x]) for x in update_fields], ',') + qstr="""INSERT INTO %s (%s) VALUES (%s)"""%(table,fields,values) + self.ZSQLSimpleSearch(qstr) + #zLOG.LOG("fmpxml: insert",zLOG.INFO,queryStr) + ret+="ad: %s \n"%dataSet.get(id_field, rowcnt) + + #zLOG.LOG("fmpxml row:",zLOG.INFO,"%d (%s)"%(rowcnt,id_val)) + if (rowcnt % 10) == 0: + zLOG.LOG("fmpxml row:",zLOG.INFO,"%d (%s)"%(rowcnt,id_val)) + transaction.commit() + + transaction.commit() + if sync_mode: + # delete unmatched entries in db + for id in dbIDs.keys(): + # find all not-updated fields + if dbIDs[id] == 0: + zLOG.LOG("fmpxml delete:",zLOG.INFO,id) + qstr = "DELETE FROM %s WHERE %s = '%s'" + self.ZSQLSimpleSearch(qstr%(table,id_field,id)) + + elif dbIDs[id] > 1: + zLOG.LOG("fmpxml sync:",zLOG.INFO,"id used more than once?"+id) + + transaction.commit() + + return ret + def generateIndex(self,field,index_name,table,RESPONSE=None): """erzeuge index aus feld""" index={} @@ -457,22 +613,14 @@ class ZSQLExtendFolder(Folder,Persistent return "" def getLabel(self): - """getLabel""" + """getLabe""" try: return self.label except: return "" - - def getTitle(self): - """getTitle""" - try: - return self.title - except: - return "" - def getDescription(self): - """getDescription""" + """getLabe""" try: return self.description except: @@ -501,10 +649,6 @@ class ZSQLExtendFolder(Folder,Persistent def formatAscii(self,str,url=None): """ersetze ascii umbrueche durch
""" #url=None - - if not str: - return "" - str=str.rstrip().lstrip() if url and str: @@ -632,7 +776,7 @@ class ZSQLExtendFolder(Folder,Persistent ret+="""""" return ret - def ZSQLOptionsFromSearchList(self,fieldname,results,fieldName,valueName=None,start=None, multiple='',startValue=None,additionalSelect="",size=None,linelen=None): + def ZSQLOptionsFromSearchList(self,fieldname,results,fieldName,valueName=None,start=None, multiple='',startValue=None,additionalSelect="",size=None,linelen=None,selected=None): """generate select options form a search list es wird