ZSQLExtend/ZSQLExtend.py - diff

Return to ZSQLExtend.py CVS log

Up to [Repository] / ZSQLExtend

Diff for /ZSQLExtend/ZSQLExtend.py between versions 1.105 and 1.113

version 1.105, 2007/02/22 13:23:20	version 1.113, 2007/04/19 20:47:00
Line 4 from Globals import DTMLFile,package_hom	Line 4 from Globals import DTMLFile,package_hom
import urllib	import urllib
import re	import re
import string	import string
import sys
#from pyPgSQL import libpq	#from pyPgSQL import libpq
from AccessControl import getSecurityManager	from AccessControl import getSecurityManager,Unauthorized
from Products.PageTemplates.ZopePageTemplate import ZopePageTemplate	from Products.PageTemplates.ZopePageTemplate import ZopePageTemplate
from Products.PageTemplates.PageTemplateFile import PageTemplateFile	from Products.PageTemplates.PageTemplateFile import PageTemplateFile

Line 19 import os.path	Line 18 import os.path
import os	import os
import copy	import copy
import unicodedata	import unicodedata
	import tempfile
import logging	import sys

#ersetzt logging	#ersetzt logging
def logger(txt,method,txt2):	def logger(txt,method,txt2):
Line 70 def sql_quote(v):	Line 69 def sql_quote(v):
def showSQLConnectionIDs(self):	def showSQLConnectionIDs(self):
return SQLConnectionIDs(self)	return SQLConnectionIDs(self)

	class Options:
	"""options class"""

class ZSQLIndex(SimpleItem):	class ZSQLIndex(SimpleItem):
"""index"""	"""index"""
meta_type="ZSQLIndex"	meta_type="ZSQLIndex"
Line 319 class ZSQLExtendFolder(Folder,Persistent	Line 321 class ZSQLExtendFolder(Folder,Persistent
return ret	return ret


def importXMLFile(self,table,data=None,identify=None,filename=None,RESPONSE=None):	def importXMLFile(self,table,containerTagName,fieldNames,data=None,identify=None,filename=None,RESPONSE=None):
#TODO: finish importXMLFile	#TODO: finish importXMLFile
'''	'''
Import XML file into the table	Import XML file into the table
Line 329 class ZSQLExtendFolder(Folder,Persistent	Line 331 class ZSQLExtendFolder(Folder,Persistent
@param identify: (optional) field res. tag which identifies a entry uniquely for updating purposes.	@param identify: (optional) field res. tag which identifies a entry uniquely for updating purposes.
@param RESPONSE: (optional)	@param RESPONSE: (optional)
'''	'''
	ret=""
from xml.dom.pulldom import parseString	from xml.dom.pulldom import parseString

doc=parseString(file.read())	doc=parseString(file.read())
Line 406 class ZSQLExtendFolder(Folder,Persistent	Line 409 class ZSQLExtendFolder(Folder,Persistent
#print "wrote: %s"%fn	#print "wrote: %s"%fn


def importXMLFileFMP(self,table,data=None,filename=None,update_fields=None,id_field=None,sync_mode=False,RESPONSE=None):	def importXMLFileFMP(self,table,dsn=None,uploadfile=None,update_fields=None,id_field=None,sync_mode=False,replace=False,redirect_url=None,ascii_db=False,RESPONSE=None):
'''	'''
Import FileMaker XML file (FMPXMLRESULT format) into the table.	Import FileMaker XML file (FMPXMLRESULT format) into the table.
	@param dsn: database connection string
@param table: name of the table the xml shall be imported into	@param table: name of the table the xml shall be imported into
@param data: xml data as bz2 string	@param uploadfile: xmlfile file
@param filename: xmlfile filename
@param update_fields: (optional) list of fields to update; default is to create all fields	@param update_fields: (optional) list of fields to update; default is to create all fields
@param id_field: (optional) field which uniquely identifies an entry for updating purposes.	@param id_field: (optional) field which uniquely identifies an entry for updating purposes.
@param sync_mode: (optional) really synchronise, i.e. delete entries not in XML file	@param sync_mode: (optional) really synchronise, i.e. delete entries not in XML file
@param RESPONSE: (optional)	@param RESPONSE: (optional)
	@param redirect_url: (optional) url for redirecting after the upload is done
'''	'''
from xml.dom.pulldom import parseString,parse
import transaction

ret = ""	tfilehd,filename=tempfile.mkstemp()
	tfile=os.fdopen(tfilehd,'w')
	logging.error("import %s"%uploadfile)
	for c in uploadfile.read():
	tfile.write(c)
	tfile.close()

	from importFMPXML import importFMPXML

	if not dsn:
	dsn=self.getConnectionObj().connection_string

	options=Options()
	options.dsn=dsn
	options.table=table
	options.filename=filename
	options.update_fields=update_fields
	options.id_field=id_field
	options.sync_mode=sync_mode
	options.replace_table=replace
	options.lc_names=True
	options.ascii_db=ascii_db
	importFMPXML(options)

if data:	os.remove(filename)
data=bz2.decompress(base64.decodestring(data))
logger("fmpxml",logging.INFO,"received file")
doc=parseString(data)
logger("fmpxml",logging.INFO,"parsed file")

elif filename:	if RESPONSE and redirect_url:
fh=file(filename)	RESPONSE.redirect(redirect_url)
logger("fmpxml",logging.INFO,"reading file")
doc=parse(fh)
logger("fmpxml",logging.INFO,"parsed file")

dbIDs = {}
rowcnt = 0

if id_field is not None:
# prepare a list of ids for sync mode
qstr="select %s from %s"%(id_field,table)
for id in self.ZSQLSimpleSearch(qstr):
# value 0: not updated
dbIDs[id[0]] = 0;
rowcnt += 1

logger("fmpxml",logging.INFO,"%d entries in DB to sync"%rowcnt)

fieldNames = []
rowcnt = 0
id_val = ''

while 1:
node=doc.getEvent()

if node is None:
break;

# METADATA tag defines number and names of fields in FMPXMLRESULT
if node[1].nodeName == 'METADATA':
doc.expandNode(node[1])

names=node[1].getElementsByTagName('FIELD')

for name in names:
fn = name.getAttribute('NAME')
fieldNames.append(fn)

if update_fields is None:
# update all fields
update_fields = fieldNames

logger("fmpxml fieldnames:",logging.INFO,repr(fieldNames))
# get list of fields in db table
qstr="""select attname from pg_attribute, pg_class where attrelid = pg_class.oid and relname = '%s'"""
columns=[x.attname for x in self.ZSQLSimpleSearch(qstr%table)]

# adjust db table to fields in XML and fieldlist
for fieldName in fieldNames:
logger("fmpxml fieldname:",logging.INFO,repr(fieldName))
if (fieldName not in columns) and (fieldName in update_fields):
qstr="""alter table %s add %s %s"""
self.ZSQLSimpleSearch(qstr%(table,fieldName,'text'))
logger("fmpxml add field:",logging.INFO,qstr%(table,fieldName,'text'))

# ROW tags (in RESULTSET tag) hold data
elif node[1].nodeName == 'ROW':
rowcnt += 1

doc.expandNode(node[1])
cols=node[1].getElementsByTagName('COL')
dataSet={}
i = 0
# populate with data
for col in cols:
data=col.getElementsByTagName('DATA')
dataSet[fieldNames[i]] = getTextFromNode(data[0])
i += 1

update=False

# synchronize by id_field
if id_field:
id_val=dataSet[id_field]
if id_val in dbIDs:
dbIDs[id_val] += 1
update=True

if update:
# update existing row (by id_field)
setvals=[]
for fieldName in update_fields:
setvals.append("%s = %s"%(fieldName,self.ZSQLQuote(dataSet[fieldName])))
setStr=string.join(setvals, ',')
id_val=dataSet[id_field]
qstr="""UPDATE %s SET %s WHERE %s = '%s' """%(table,setStr,id_field,id_val)
#logger("fmpxml update:",logging.INFO,queryStr)
self.ZSQLSimpleSearch(qstr)
ret+="up: %s \n"%id_val
else:
# create new row
fields=string.join(update_fields, ',')
values=string.join([" %s "%self.ZSQLQuote(dataSet[x]) for x in update_fields], ',')
qstr="""INSERT INTO %s (%s) VALUES (%s)"""%(table,fields,values)
self.ZSQLSimpleSearch(qstr)
#logger("fmpxml: insert",logging.INFO,queryStr)
ret+="ad: %s \n"%dataSet.get(id_field, rowcnt)

#logger("fmpxml row:",logging.INFO,"%d (%s)"%(rowcnt,id_val))
if (rowcnt % 10) == 0:
logger("fmpxml row:",logging.INFO,"%d (%s)"%(rowcnt,id_val))
transaction.commit()

transaction.commit()
if sync_mode:
# delete unmatched entries in db
for id in dbIDs.keys():
# find all not-updated fields
if dbIDs[id] == 0:
logger("fmpxml delete:",logging.INFO,id)
qstr = "DELETE FROM %s WHERE %s = '%s'"
self.ZSQLSimpleSearch(qstr%(table,id_field,id))

elif dbIDs[id] > 1:
logger("fmpxml sync:",logging.INFO,"id used more than once?"+id)

transaction.commit()

return ret

def generateIndex(self,field,index_name,table,RESPONSE=None):	def generateIndex(self,field,index_name,table,RESPONSE=None):
"""erzeuge index aus feld"""	"""erzeuge index aus feld"""
Line 721 class ZSQLExtendFolder(Folder,Persistent	Line 623 class ZSQLExtendFolder(Folder,Persistent
return 1	return 1
return 0	return 0

	def ZSQLMultiSearch(self,_table,_searchField,_value,_idField,_additionalStatement="",_select=None,_storename=None):
	"""
	Durchsucht in einer Tabelle "table" die Spalte "searchfield" nach dem allen Vorkommnissen
	von Worten in value und gibt alle Werte mit gleichem id field zur�ck, d.h. es wird die "und" suche realisiert,
	z.B. f�r simplesearch ueber mehrere Felder
	"""
	if _storename:
	"""store"""
	else:
	_storename="foundCount"

	queries=[]
	#baue jede einzelne abfrage
	splitted=_value.split(" ")
	if not _select:
	_select=_idField

	queries.append("select %s from %s %s where lower(%s) like '%%%s%%'"%(_select,_table,_additionalStatement,_searchField,splitted[0].lower()))

	if len(splitted)>1:
	for v in splitted[1:]:
	queries.append("select %s from %s where lower(%s) like '%%%s%%'"%(_idField,_table,_searchField,v.lower()))

	q=" and %s in ("%_idField
	query=q.join(queries) # nun baue sie zusammen
	for i in range(len(queries)-1):
	query+=")" #noch die klammern schliessen


	if _additionalStatement:
	query=query+" "

	logging.info("ZSQLSimple: %s"%query)
	retT=self.ZSQLSimpleSearch(query)
	logging.info("ZSQLSimple: %s"%retT)

	retFinalT={}
	for x in retT:
	split=_idField.split(".")
	if len(split)>1:
	f=split[1]
	else:
	f=_idField

	retFinalT[getattr(x,f)]=x

	ret=list(retFinalT.values())



	if not self.REQUEST.SESSION.has_key(_storename):
	self.REQUEST.SESSION[_storename]={}

	self.REQUEST.SESSION[_storename]['searchFieldsOnly']={}
	self.REQUEST.SESSION[_storename]['qs']=query
	return ret

def ZSQLsearchOptions(self,fieldname=""):	def ZSQLsearchOptions(self,fieldname=""):
"""return HTML Fragment with search options"""	"""return HTML Fragment with search options"""

Line 784 class ZSQLExtendFolder(Folder,Persistent	Line 743 class ZSQLExtendFolder(Folder,Persistent
ret+="""</select>"""	ret+="""</select>"""
return ret	return ret

def ZSQLOptionsFromSearchList(self,fieldname,results,fieldName,valueName=None,start=None, multiple='',startValue=None,additionalSelect="",size=None,linelen=None,selected=None):	def ZSQLOptionsFromSearchList(self,fieldname,
	results,fieldName,
	valueName=None,start=None,
	multiple='',startValue=None,
	additionalSelect="",size=None,
	linelen=None,selected=None,
	clear=False):
"""generate select options form a search list	"""generate select options form a search list
es wird	es wird
<select name=fieldname mutiple>	<select name=fieldname mutiple>
Line 792 class ZSQLExtendFolder(Folder,Persistent	Line 757 class ZSQLExtendFolder(Folder,Persistent
<option value=result.fieldName>result.fieldValue</option>	<option value=result.fieldName>result.fieldValue</option>
erzeugt.	erzeugt.

@parameter fieldname: Name fuer name-wert im select-tag	@param fieldname: Name fuer name-wert im select-tag
@results results: Resultobject einer SQL-suche	@param results: Resultobject einer SQL-suche
@parameter fieldName: Name des Feldes, das als value in den option-tag geschrieben werden soll.	@param fieldName: Name des Feldes, das als value in den option-tag geschrieben werden soll.
@parameter valueName: (optional) Name des Feldes, dass als im option-tag ausgegeben wird, default wert ist valueName=fieldName	@param valueName: (optional) Name des Feldes, dass als im option-tag ausgegeben wird, default wert ist valueName=fieldName
@start: (optional) falls zusaetzliches option tag erzeugt werden soll, gibt start an was im option tag steht	@param start: (optional) falls zusaetzliches option tag erzeugt werden soll, gibt start an was im option tag steht
@startValue: gibt den entsprechenden Wert an.	@param startValue (optional): gibt den entsprechenden Wert an.
@linelen: (optional) maximale laenge eines Eintrages	@param selected (optional): Wert der ausgewaehlt sein soll.
	@param linelen: (optional) maximale laenge eines Eintrages
der im Klappmenue noch angezeigt wird, laengeres wird abgeschnitten.	der im Klappmenue noch angezeigt wird, laengeres wird abgeschnitten.
	@param addionalSaelect (optional): zusaetzlicher text fuer den select tag
	@param clear (optional): setze auf den startwert.
"""	"""
if not valueName:	if not valueName:
valueName=fieldName	valueName=fieldName
Line 816 class ZSQLExtendFolder(Folder,Persistent	Line 784 class ZSQLExtendFolder(Folder,Persistent
if not startValue:	if not startValue:
startValue=start	startValue=start

	if clear:
	ret+="""<option selected value="%s" >%s</option>"""%(startValue,start)
	else:
ret+="""<option value="%s" >%s</option>"""%(startValue,start)	ret+="""<option value="%s" >%s</option>"""%(startValue,start)

for result in results:	for result in results:
field=getattr(result,fieldName)	field=getattr(result,fieldName)
fieldValue=getattr(result,valueName)	fieldValue=getattr(result,valueName)

if fieldValue:	if linelen and fieldValue and (len(fieldValue) > string.atoi(linelen)):
	displayValue = fieldValue[:string.atoi(linelen)]
if not linelen:	else:
	displayValue = fieldValue

	if displayValue: #show only if value not none
if field == selected:	if field == selected:

ret+="""<option value="%s" selected>%s</option>"""%(field,fieldValue)	ret+="""<option value="%s" selected>%s</option>"""%(field,displayValue)
else:	else:
ret+="""<option value="%s">%s</option>"""%(field,fieldValue)	ret+="""<option value="%s">%s</option>"""%(field,displayValue)

else:
mist = """%s"""%(fieldValue)
if len(mist) > string.atoi(linelen):
mist = mist[:string.atoi(linelen)]
ret+="""<option value="%s">%s</option>"""%(field,mist)
ret+="""</select>"""	ret+="""</select>"""
return ret	return ret

Line 918 class ZSQLExtendFolder(Folder,Persistent	Line 886 class ZSQLExtendFolder(Folder,Persistent
if x:	if x:
value=x	value=x
else:	else:

value=str(argTmp[a])	value=str(argTmp[a])
qs.append(aFiltered+"="+urllib.quote(value))	qs.append(aFiltered+"="+urllib.quote(value))

Line 947 class ZSQLExtendFolder(Folder,Persistent	Line 916 class ZSQLExtendFolder(Folder,Persistent

def ZSQLSimpleSearch(self,query=None,max_rows=1000000):	def ZSQLSimpleSearch(self,query=None,max_rows=1000000):
"""simple search"""	"""simple search"""
	logging.error("ZSQLSimpleSearch X %s"%query)
#print query	#print query
if not query:	if not query:
query=self.query	query=self.query
Line 959 class ZSQLExtendFolder(Folder,Persistent	Line 928 class ZSQLExtendFolder(Folder,Persistent

self._v_searchSQL.max_rows_=max_rows	self._v_searchSQL.max_rows_=max_rows
try:	try:
return self._v_searchSQL.__call__(var=query)	logging.error("I am here")
	t=self._v_searchSQL.__call__(var=query)
	logging.error("I am here %s"%t)
	return t
except :	except :
	logger("ZSQLSimpleSearch ERROR1",logging.ERROR, '%s %s'%sys.exc_info()[:2])
if sys.exc_info()[0]=="Database Error":	if sys.exc_info()[0]=="Database Error":
try:	try:
self.getConnectionObj().manage_open_connection()	self.getConnectionObj().manage_open_connection()
except:	except:
logger("ZSQLSimpleSearch",logging.ERROR, '%s %s'%sys.exc_info()[:2])	logger("ZSQLSimpleSearch ERROR2",logging.ERROR, '%s %s'%sys.exc_info()[:2])
else:	else:
try:	try:

Line 974 class ZSQLExtendFolder(Folder,Persistent	Line 946 class ZSQLExtendFolder(Folder,Persistent

return self._v_searchSQL.__call__(var=query)	return self._v_searchSQL.__call__(var=query)
except :	except :
	logger("ZSQLSimpleSearch ERROR2",logging.ERROR, '%s %s'%sys.exc_info()[:2])
if sys.exc_info()[0]=="Database Error":	if sys.exc_info()[0]=="Database Error":
try:	try:
self.getConnectionObj().manage_open_connection()	self.getConnectionObj().manage_open_connection()
Line 1107 class ZSQLExtendFolder(Folder,Persistent	Line 1079 class ZSQLExtendFolder(Folder,Persistent
table=urllib.unquote(value)	table=urllib.unquote(value)
elif name=="-identify":	elif name=="-identify":
identify=urllib.unquote(value)	identify=urllib.unquote(value)
identify=identify.split("=")[0]+"="+sql_quote(identify.split("=")[1])	identify="lower("+identify.split("=")[0]+")="+sql_quote(identify.split("=")[1].lower())
elif name=="-format":	elif name=="-format":
format=urllib.unquote(value)	format=urllib.unquote(value)
#elif (not (name[0]=="-" or name[0]=="_")) and (not len(value)==0):	#elif (not (name[0]=="-" or name[0]=="_")) and (not len(value)==0):
Line 1132 class ZSQLExtendFolder(Folder,Persistent	Line 1104 class ZSQLExtendFolder(Folder,Persistent
return True	return True


def ZSQLFindIndexed(self,qs="",select="oid,",storename=None,indexedFields=['data_line'],restrictField='id_text',*argv):	def ZSQLFindIndexed(self,tableList=[],qs="",select="oid,",storename=None,indexedFields=['data_line'],restrictField='id_text',*argv):
"""find2"""	"""find2"""

for index in self.ZopeFind(self,obj_ids=indexedFields):	for index in self.ZopeFind(self,obj_ids=indexedFields):
Line 1218 class ZSQLExtendFolder(Folder,Persistent	Line 1190 class ZSQLExtendFolder(Folder,Persistent


#print "calling Query with",repr(NoQuery)	#print "calling Query with",repr(NoQuery)

ret=self.parseQueryString(qs,"-",select=select,storemax="yes",storename=storename,tableExt=tableExt,NoQuery=NoQuery,NoLimit=NoLimit,restrictField=restrictField,restrictConnect=restrictConnect,filter=filter)	ret=self.parseQueryString(qs,"-",select=select,storemax="yes",storename=storename,tableExt=tableExt,NoQuery=NoQuery,NoLimit=NoLimit,restrictField=restrictField,restrictConnect=restrictConnect,filter=filter)
#print self.REQUEST.SESSION["foundCount"]	#print self.REQUEST.SESSION["foundCount"]



return ret	return ret

def ZSQLFoundCountLen(self,var):	def ZSQLFoundCountLen(self,var):
Line 1268 class ZSQLExtendFolder(Folder,Persistent	Line 1242 class ZSQLExtendFolder(Folder,Persistent
for query in querys:	for query in querys:
arg=query.split("=")[0]	arg=query.split("=")[0]
if arg[0]=="_": arg="-"+arg[1:] # sicherstellen, dass an Anfang stets "_"	if arg[0]=="_": arg="-"+arg[1:] # sicherstellen, dass an Anfang stets "_"
	try:
queryList[arg]=query.split("=")[1]	queryList[arg]=query.split("=")[1]
	except:
	queryList[arg]=''

argList=[]	argList=[]
arg=""	arg=""
Line 1315 class ZSQLExtendFolder(Folder,Persistent	Line 1291 class ZSQLExtendFolder(Folder,Persistent
searchFieldsOnly={}	searchFieldsOnly={}
queryTemplate=[]	queryTemplate=[]
outerjoin=""	outerjoin=""
	debug=None

if not select:	if not select:
select="oid,*"	select="oid,*"
Line 1333 class ZSQLExtendFolder(Folder,Persistent	Line 1310 class ZSQLExtendFolder(Folder,Persistent
for q in splitted:	for q in splitted:

name=re.sub("r'+'"," ",q.split("=")[0].lower())	name=re.sub("r'+'"," ",q.split("=")[0].lower())
	if name=="_debug":
	debug=True

try:	try:
value=urllib.unquote(q.split("=",1)[1])	value=urllib.unquote(q.split("=",1)[1])
except:	except:
Line 1440 class ZSQLExtendFolder(Folder,Persistent	Line 1420 class ZSQLExtendFolder(Folder,Persistent
#something is defined by _op_TABELLE.SUCHFELD_IN_DIESER_TABELLE.SELECT_FIELD.IDENTIFIER_IN_TABELLE	#something is defined by _op_TABELLE.SUCHFELD_IN_DIESER_TABELLE.SELECT_FIELD.IDENTIFIER_IN_TABELLE

elif (not name[0]==iCT) and len(punktsplit)==4:	elif (not name[0]==iCT) and len(punktsplit)==4:

if opfields.has_key(name):	if opfields.has_key(name):
op=opfields[name]	op=opfields[name]
else:	else:
Line 1480 class ZSQLExtendFolder(Folder,Persistent	Line 1461 class ZSQLExtendFolder(Folder,Persistent

op="all"	op="all"

	if value!='': #lehre Werte werde nicht hinzugefuegt
searchTmp="""%s in (select %s from %s where %s)"""%(punktsplit[3],punktsplit[2],punktsplit[0],tmp)	searchTmp="""%s in (select %s from %s where %s)"""%(punktsplit[3],punktsplit[2],punktsplit[0],tmp)

queryTemplate.append(searchTmp)	queryTemplate.append(searchTmp)
Line 1530 class ZSQLExtendFolder(Folder,Persistent	Line 1511 class ZSQLExtendFolder(Folder,Persistent

op="all"	op="all"

if (not tableExt) or (namealt.split('.')[0]==tableExt):	if (value!='') and ((not tableExt) or (namealt.split('.')[0]==tableExt)): #keine leeren werde und keine auschluss
if searchFields.has_key(namealt):	if searchFields.has_key(namealt):
searchFields[namealt]+=lopfields.get(name,'OR')+" "+tmp	searchFields[namealt]+=lopfields.get(name,'OR')+" "+tmp
searchFieldsOnly[namealt]+=lopfields.get(name,'OR')+" "+value	searchFieldsOnly[namealt]+=lopfields.get(name,'OR')+" "+value
Line 1616 class ZSQLExtendFolder(Folder,Persistent	Line 1597 class ZSQLExtendFolder(Folder,Persistent

self.REQUEST.SESSION[storename]['searchFieldsOnly']=searchFieldsOnly	self.REQUEST.SESSION[storename]['searchFieldsOnly']=searchFieldsOnly

	if debug:
	logging.error("ZSQLSimpleSearch %s"%query)
if not NoQuery:	if not NoQuery:

return self.ZSQLQuery(query)	return self.ZSQLQuery(query)

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>

Removed from v.1.105
changed lines
	Added in v.1.113