ZSQLExtend/ZSQLExtend.py - diff

Return to ZSQLExtend.py CVS log

Up to [Repository] / ZSQLExtend

Diff for /ZSQLExtend/ZSQLExtend.py between versions 1.79 and 1.92

version 1.79, 2006/03/28 19:54:41	version 1.92, 2006/09/16 15:17:24
Line 1	Line 1

from OFS.Folder import Folder	from OFS.Folder import Folder
from Acquisition import Implicit	from Acquisition import Implicit
from Globals import DTMLFile,package_home,Persistent	from Globals import DTMLFile,package_home,Persistent
Line 17 import Shared.DC.ZRDB.DA	Line 18 import Shared.DC.ZRDB.DA
import zLOG	import zLOG
import os.path	import os.path
import os	import os
from OFS.SimpleItem import SimpleItem	import copy

def getTextFromNode(nodename):
nodelist=nodename.childNodes
rc = ""
for node in nodelist:
if node.nodeType == node.TEXT_NODE:
rc = rc + node.data
return rc

	from OFS.SimpleItem import SimpleItem

def analyseIntSearch(word):	def analyseIntSearch(word):
#analyse integer searches	#analyse integer searches
Line 81 class ZSQLExtendFolder(Folder,Persistent	Line 75 class ZSQLExtendFolder(Folder,Persistent
return sql_quote(str)	return sql_quote(str)


	def normalizeField(self,table,fieldname, newFieldName=None,mode="alter", RESPONSE=None):
	"""normalize a field"""
	import unicodedata

def importXMLFile(self,table,data,identify=None,RESPONSE=None):	if not newFieldName:
#TODO: finish importXMLFile	newFieldName=fieldname+"_normal"

	def normal(str):
	if str:
	return unicodedata.normalize('NFKD', str.decode('utf-8')).encode('ASCII', 'ignore')
	else:
	return ""
	if mode=="create": # create the field
	qstr="""alter table %s add %s %s"""
	self.ZSQLSimpleSearch(qstr%(table,newFieldName,'text'))

	qstr="select oid,%s from %s"%(fieldname,table)
	for result in self.ZSQLSimpleSearch(qstr):
	qstr="update %s set %s = %s where oid = %s"

	self.ZSQLSimpleSearch(qstr%(table,newFieldName,self.ZSQLQuote(normal(getattr(result,fieldname))),result.oid))

	def importAccessModell(self,configFileName,RESPONSE=None):
	"""import tables from access
	@param configFileName: xml-configfile
	"""
	from Ft.Xml import Parse
	fh=file(configFileName)

	doc=Parse(fh)

	x=doc.xpath("//pathToFolder/@path")


	if not (len(x)==1): # tag ist nich eineindeutig
	return False

	pathToFolder=x[0].value

	for db in doc.xpath("//db"):

	containers=db.xpath("./@container")
	identifiers=db.xpath("./@identify")


	if not (len(containers)==1):
	return False
	else:
	container=containers[0].value

	if not (len(identifiers)==1):
	identifier=None
	else:
	identifier=identifiers[0].value

	self.xsdToTable(container.lower(),container,modus="drop",filename=os.path.join(pathToFolder,container.lower()+".xsd"))
	self.importXMLFileAccess(container.lower(),container,filename=os.path.join(pathToFolder,container.lower()+".xml"),identify=identifier)

	return "<html><body>DONE</body></html>"

	def xsdToTable(self,table,elementNameForTable,modus="update", filename=None,data=None,RESPONSE=None):
	"""reads an xsd file an creates the columns of a table out of its structure
	@param table: name of the table the xml shall be imported into
	@param elementNameForTable: must be a element of type complex type. the element of the sequence in the complex type
	define the columns of the table >table<
	@param data (optional): data to be imported

	@param filename (optional) or filename
	@param identify: (optional) field res. tag which identifies a entry uniquely for updating purposes.
	@param RESPONSE: (optional)

	"""
	#from xml.dom.minidom import parseString,parse

	from Ft.Xml import Parse
	zLOG.LOG("import xsd",zLOG.INFO,"called")
	#fh=file("/tmp/fmpxml.xml")
	import bz2
	import base64


	ret=""
	if data:
	data=bz2.decompress(base64.decodestring(data))

	#zLOG.LOG("import xsd",zLOG.INFO,"received file")
	doc=Parse(data)
	#zLOG.LOG("import xsd",zLOG.INFO,"parsed file")

	elif filename:
	fh=file(filename)
	txt=fh.read()

	doc=Parse(txt)
	#zLOG.LOG("import xsd",zLOG.INFO,"parsed file")


	Nss={'xsd':'http://www.w3.org/2001/XMLSchema'}
	definingSequence=doc.xpath("""//xsd:element[@name='%s']/xsd:complexType/xsd:sequence/xsd:element/@name"""%elementNameForTable,explicitNss=Nss)

	fieldNames=[x.value for x in definingSequence]



	#check if table exists

	qstr="""select relname from pg_class where relname = '%s'"""%table
	if not(self.ZSQLSimpleSearch(qstr)) or (len (self.ZSQLSimpleSearch(qstr))<1): # if not the create the table
	columns=[]
	create=True
	else:
	create=False

	zLOG.LOG("update xsd: fieldnames",zLOG.INFO,repr(fieldNames))
	qstr="""select attname from pg_attribute, pg_class where attrelid = pg_class.oid and relname = '%s' """
	columns=[x.attname for x in self.ZSQLSimpleSearch(qstr%table)]


	if (modus=="drop") and (not create): #table shall be deleted, therefore it should exist (not create)
	print "drop"
	qstr="""DROP TABLE %s """
	self.ZSQLSimpleSearch(qstr%table)
	columns=[]
	create=True

	for fieldName in fieldNames:
	if type(fieldName) is UnicodeType:
	fieldName=fieldName.encode('utf-8')
	zLOG.LOG("update xml: fieldname",zLOG.INFO,repr(fieldName))
	if fieldName.lower() not in columns:

	if create:# table does not exist therefore create with one column
	qstr="""create table %s (%s %s)"""
	create=False
	else:# otherwise add the field
	qstr="""alter table %s add %s %s"""

	self.ZSQLSimpleSearch(qstr%(table,fieldName,'text'))
	zLOG.LOG("update xsd: fieldname add",zLOG.INFO,qstr%(table,fieldName,'text'))


	def importXMLFileAccess(self,table,container,data=None,identify=None,filename=None,RESPONSE=None):
'''	'''
Import XML file into the table	Import XML file in access format into the table
@param table: name of the table the xml shall be imported into	@param table: name of the table the xml shall be imported into
@param containerTagName: XML-Tag which describes a dataset	@param containerTagName: XML-Tag which describes a dataset
@param data: data to be imported	@param data: data to be imported
@param identify: (optional) field res. tag which identifies a entry uniquely for updating purposes.	@param identify: (optional) field res. tag which identifies a entry uniquely for updating purposes.
@param RESPONSE: (optional)	@param RESPONSE: (optional)
'''	'''
from xml.dom.pulldom import parseString	from xml.dom.pulldom import parseString,parse


	zLOG.LOG("import xml",zLOG.INFO,"called")
#fh=file("/tmp/fmpxml.xml")	#fh=file("/tmp/fmpxml.xml")
import bz2	import bz2
import base64	import base64

ret=""	ret=""
	if data:
data=bz2.decompress(base64.decodestring(data))	data=bz2.decompress(base64.decodestring(data))
#data=fh.read()

	zLOG.LOG("import xml",zLOG.INFO,"received file")
doc=parseString(data)	doc=parseString(data)
	zLOG.LOG("import xml",zLOG.INFO,"parsed file")

	elif filename:
	fh=file(filename)
	doc=parse(fh)
	zLOG.LOG("import xml",zLOG.INFO,"parsed file")
while 1:	while 1:
node=doc.getEvent()	node=doc.getEvent()

if node is None:	if node is None:
break;	break;
else:	else:
if node[1].nodeName=='ROW':	if node[1].nodeName.lower()==container.lower(): # make everything case insensitive
doc.expandNode(node[1])	doc.expandNode(node[1])
cols=node[1].getElementsByTagName('COL')
dataSet=[]	dataSet={}
for col in cols:	for col in node[1].childNodes:
data=col.getElementsByTagName('DATA')	if col.nodeType is col.ELEMENT_NODE:
dataSet.append(getTextFromNode(data[0]))	data=col.nodeName
	dataSet[data]=getTextFromNode(col)

update=False	update=False

if identify:	if identify:

nr=fieldNames.index(identify)	field=dataSet[identify]
field=dataSet[nr]

searchStr="""select %s from %s where %s = '%s'"""%(identify,table,identify,field)	searchStr="""select %s from %s where %s = '%s'"""%(identify,table,identify,field)
	zLOG.LOG("import xml",zLOG.INFO,searchStr)
search=self.ZSQLSimpleSearch(searchStr)	search=self.ZSQLSimpleSearch(searchStr)
if search:	if search:
update=True	update=True

if update:	if update:
tmp=[]	tmp=[]
for fieldName in fieldNames:	for fieldName in dataSet.keys():
tmp.append("""%s = %s"""%(fieldName,self.ZSQLQuote(dataSet[fieldNames.index(fieldName)])))	tmp.append("""%s = %s"""%(fieldName,self.ZSQLQuote(dataSet[fieldName])))
setStr=",".join(tmp)	setStr=",".join(tmp)
nr=fieldNames.index(identify)
field=dataSet[nr]	field=dataSet[identify]

queryStr="""UPDATE %s SET %s WHERE %s = '%s' """%(table,setStr,identify,field)	queryStr="""UPDATE %s SET %s WHERE %s = '%s' """%(table,setStr,identify,field)
	zLOG.LOG("update xml",zLOG.INFO,queryStr)
self.ZSQLSimpleSearch(queryStr)	self.ZSQLSimpleSearch(queryStr)
ret+="ud: %s \n"%field	ret+="ud: %s \n"%field
else:	else:


fields=",".join(fieldNames)	fields=",".join(dataSet.keys())
values=",".join([""" %s """%self.ZSQLQuote(x) for x in dataSet])	values=",".join([""" %s """%self.ZSQLQuote(dataSet[x]) for x in dataSet.keys()])


queryStr="""INSERT INTO %s (%s) VALUES (%s)"""%(table,fields,values)	queryStr="""INSERT INTO %s (%s) VALUES (%s)"""%(table,fields,values)
self.ZSQLSimpleSearch(queryStr)	self.ZSQLSimpleSearch(queryStr)
ret+="ad: %s \n"%field	zLOG.LOG("update xml",zLOG.INFO,queryStr)

elif node[1].nodeName=="METADATA":
fieldNames=[]
doc.expandNode(node[1])

names=node[1].getElementsByTagName('FIELD')

for name in names:
fieldNames.append(name.getAttribute('NAME'))

qstr="""select attname from pg_attribute, pg_class where attrelid = pg_class.oid and relname = '%s' """	return ret
columns=[x.attname for x in self.ZSQLSimpleSearch(qstr%table)]

for fieldName in fieldNames:
if fieldName not in columns:
qstr="""alter table %s add %s %s"""
self.ZSQLSimpleSearch(qstr%(table,fieldName,'text'))

	def importXMLFile(self,table,data=None,identify=None,filename=None,RESPONSE=None):
	#TODO: finish importXMLFile
	'''
	Import XML file into the table
	@param table: name of the table the xml shall be imported into
	@param containerTagName: XML-Tag which describes a dataset
	@param file: xmlfile handle
	@param identify: (optional) field res. tag which identifies a entry uniquely for updating purposes.
	@param RESPONSE: (optional)
	'''
	from xml.dom.pulldom import parseString

	doc=parseString(file.read())
	while 1:
	node=doc.getEvent()

	if node is None:
	break;
	else:
	if node[1].nodeName==containerTagName:
	doc.expandNode(node[1])
#fn=node[1].getAttribute("xml:id")	#fn=node[1].getAttribute("xml:id")
#nf=file("xtf/"+fn+".xtf",'w')	#nf=file("xtf/"+fn+".xtf",'w')
#nf.write("""<texts xmlns="http://emegir.info/xtf" xmlns:lem="http://emegir.info/lemma" >"""+node[1].toxml()+"</texts>")	#nf.write("""<texts xmlns="http://emegir.info/xtf" xmlns:lem="http://emegir.info/lemma" >"""+node[1].toxml()+"</texts>")
#print "wrote: %s"%fn	#print "wrote: %s"%fn
return ret

def generateIndex(self,field,index_name,table,RESPONSE=None):	def generateIndex(self,field,index_name,table,RESPONSE=None):
"""erzeuge index aus feld"""	"""erzeuge index aus feld"""
Line 245 class ZSQLExtendFolder(Folder,Persistent	Line 399 class ZSQLExtendFolder(Folder,Persistent
return ""	return ""

def getLabel(self):	def getLabel(self):
"""getLabel"""	"""getLabe"""
try:	try:
return self.label	return self.label
except:	except:
return ""	return ""

def getTitle(self):
"""getTitle"""
try:
return self.title
except:
return ""


def getDescription(self):	def getDescription(self):
"""getDescription"""	"""getLabe"""
try:	try:
return self.description	return self.description
except:	except:
Line 289 class ZSQLExtendFolder(Folder,Persistent	Line 435 class ZSQLExtendFolder(Folder,Persistent
def formatAscii(self,str,url=None):	def formatAscii(self,str,url=None):
"""ersetze ascii umbrueche durch <br>"""	"""ersetze ascii umbrueche durch <br>"""
#url=None	#url=None

if not str:
return ""

str=str.rstrip().lstrip()	str=str.rstrip().lstrip()

if url and str:	if url and str:
Line 420 class ZSQLExtendFolder(Folder,Persistent	Line 562 class ZSQLExtendFolder(Folder,Persistent
ret+="""</select>"""	ret+="""</select>"""
return ret	return ret

def ZSQLOptionsFromSearchList(self,fieldname,results,fieldName,valueName=None,start=None, multiple='',startValue=None,additionalSelect="",size=None,linelen=None):	def ZSQLOptionsFromSearchList(self,fieldname,results,fieldName,valueName=None,start=None, multiple='',startValue=None,additionalSelect="",size=None,linelen=None,selected=None):
"""generate select options form a search list	"""generate select options form a search list
es wird	es wird
<select name=fieldname mutiple>	<select name=fieldname mutiple>
Line 455 class ZSQLExtendFolder(Folder,Persistent	Line 597 class ZSQLExtendFolder(Folder,Persistent
field=getattr(result,fieldName)	field=getattr(result,fieldName)
fieldValue=getattr(result,valueName)	fieldValue=getattr(result,valueName)
if fieldValue:	if fieldValue:

if not linelen:	if not linelen:


	if field == selected:

	ret+="""<option value="%s" selected>%s</option>"""%(field,fieldValue)
	else:
ret+="""<option value="%s">%s</option>"""%(field,fieldValue)	ret+="""<option value="%s">%s</option>"""%(field,fieldValue)

else:	else:
mist = """%s"""%(fieldValue)	mist = """%s"""%(fieldValue)
if len(mist) > string.atoi(linelen):	if len(mist) > string.atoi(linelen):
Line 466 class ZSQLExtendFolder(Folder,Persistent	Line 616 class ZSQLExtendFolder(Folder,Persistent
return ret	return ret


def ZSQLInlineSearchU(self,storename=None,**argv):	def ZSQLInlineSearchU(self,storename=None,args=None,**argv):
"""one element if exists"""	"""one element if exists"""
qs=[]	qs=[]
if storename:	if storename:
Line 475 class ZSQLExtendFolder(Folder,Persistent	Line 625 class ZSQLExtendFolder(Folder,Persistent
else:	else:
storename="foundCount"	storename="foundCount"

	if args:
	argTmp=args
	else:
	argTmp=argv


#print "INLINE:",argv	#print "INLINE:",argv
for a in argv.keys():	for a in argTmp.keys():
qs.append(a+"="+urllib.quote(str(argv[a])))	qs.append(a+"="+urllib.quote(str(argTmp[a])))
#return []	#return []
ret = self.parseQueryString(string.join(qs,","),"_",storename=storename)	ret = self.parseQueryString(string.join(qs,","),"_",storename=storename)

Line 488 class ZSQLExtendFolder(Folder,Persistent	Line 642 class ZSQLExtendFolder(Folder,Persistent
except:	except:
return None	return None

def ZSQLInlineSearch(self,storename=None,**argv):
	def ZSQLSequentialSearch(self,_fieldlist,_searchTerm,storename=None,args=None,**argv):
	"""get a list of tuples (fields,searchoptions,table) to search the searchTerm in, returns dictionary. """
	#print "do search with:",_fieldlist,_searchTerm,storename,args,argv
	ret={}
	if args:
	argTmp=args
	else:
	argTmp=argv

	for field in _fieldlist:

	argTmp2=copy.deepcopy(argTmp)
	argTmp2[field[0]]=_searchTerm
	argTmp2['_op_'+field[0]]=field[1]
	argTmp2['_table']=field[2]

	ret[field[0]]=(self.ZSQLInlineSearch(storename=storename,args=argTmp2),field[3],field[4])

	return ret
	def ZSQLInlineSearch(self,storename=None,args=None,**argv):
"""inlinesearch"""	"""inlinesearch"""

qs=[]	qs=[]
Line 499 class ZSQLExtendFolder(Folder,Persistent	Line 673 class ZSQLExtendFolder(Folder,Persistent



	if args:
	argTmp=args
	else:
	argTmp=argv


#print "INLINE:",argv	#print "INLINE:",argv
for a in argv.keys():	for a in argTmp.keys():
try:	try:
qs.append(a+"="+urllib.quote(str(argv[a])))	qs.append(a+"="+urllib.quote(str(argTmp[a])))
except:	except:
import urllib	import urllib
qs.append(a+"="+urllib.quote(str(argv[a])))	qs.append(a+"="+urllib.quote(str(argTmp[a])))

#return []	#return []

return self.parseQueryString(string.join(qs,","),"_",storename=storename)	return self.parseQueryString(string.join(qs,","),"_",storename=storename)

def ZSQLInlineSearch2(self,query):	def ZSQLInlineSearch2(self,query):
Line 529 class ZSQLExtendFolder(Folder,Persistent	Line 710 class ZSQLExtendFolder(Folder,Persistent
except:	except:
zLOG.LOG("ZSQLResetConnection",zLOG.ERROR, '%s %s'%sys.exc_info()[:2])	zLOG.LOG("ZSQLResetConnection",zLOG.ERROR, '%s %s'%sys.exc_info()[:2])

def ZSQLSimpleSearch(self,query=None,max_rows=1000000,debug=None):	def ZSQLSimpleSearch(self,query=None,max_rows=1000000):
"""simple search"""	"""simple search"""


if not query:	if not query:
query=self.query	query=self.query

if debug:
print "DEBUG: ZSQLSimpleSearch:", query
if (hasattr(self,"_v_searchSQL") and (self._v_searchSQL == None)) or (not hasattr(self,"_v_searchSQL")):	if (hasattr(self,"_v_searchSQL") and (self._v_searchSQL == None)) or (not hasattr(self,"_v_searchSQL")):

self._v_searchSQL=Shared.DC.ZRDB.DA.DA("_v_searchSQL","_v_searchSQL",self.getConnectionObj().getId(),"var","<dtml-var var>")	self._v_searchSQL=Shared.DC.ZRDB.DA.DA("_v_searchSQL","_v_searchSQL",self.getConnectionObj().getId(),"var","<dtml-var var>")
Line 597 class ZSQLExtendFolder(Folder,Persistent	Line 777 class ZSQLExtendFolder(Folder,Persistent



def ZSQLAdd(self,format=None,RESPONSE=None,**argv):	def ZSQLAdd(self,format=None,RESPONSE=None,args=None,**argv):
"""Neuer Eintrag"""	"""Neuer Eintrag"""

	if args:
	argTmp=args
	else:
	argTmp=argv

qs_temp=[]	qs_temp=[]

for a in self.REQUEST.form.keys():	for a in self.REQUEST.form.keys():
Line 607 class ZSQLExtendFolder(Folder,Persistent	Line 792 class ZSQLExtendFolder(Folder,Persistent

qs=string.join(qs_temp,",")	qs=string.join(qs_temp,",")

for field in argv.keys():	for field in argTmp.keys():
if field[0]=="_":	if field[0]=="_":
fieldTmp="-"+field[1:]	fieldTmp="-"+field[1:]
else:	else:
fieldTmp=field	fieldTmp=field

qs+=",%s=%s"%(fieldTmp,argv[field])	qs+=",%s=%s"%(fieldTmp,argTmp[field])


addList={}	addList={}
Line 655 class ZSQLExtendFolder(Folder,Persistent	Line 840 class ZSQLExtendFolder(Folder,Persistent
qs_temp=[]	qs_temp=[]
if USE_FORM or RESPONSE:	if USE_FORM or RESPONSE:
for a in self.REQUEST.form.keys():	for a in self.REQUEST.form.keys():

qs_temp.append(a+"="+urllib.quote(str(self.REQUEST.form[a])))	qs_temp.append(a+"="+urllib.quote(str(self.REQUEST.form[a])))


Line 685 class ZSQLExtendFolder(Folder,Persistent	Line 871 class ZSQLExtendFolder(Folder,Persistent
identify=identify.split("=")[0]+"="+sql_quote(identify.split("=")[1])	identify=identify.split("=")[0]+"="+sql_quote(identify.split("=")[1])
elif name=="-format":	elif name=="-format":
format=urllib.unquote(value)	format=urllib.unquote(value)
elif (not (name[0]=="-" or name[0]=="_")) and (not len(value)==0):	#elif (not (name[0]=="-" or name[0]=="_")) and (not len(value)==0):
	elif (not (name[0]=="-" or name[0]=="_")):

changeList.append("\""+name+"\"="+sql_quote(urllib.unquote(value)))	changeList.append("\""+name+"\"="+sql_quote(urllib.unquote(value)))

changeString=string.join(changeList,",")	changeString=string.join(changeList,",")

queryString="UPDATE %s SET %s WHERE %s"%(table,changeString,identify)	queryString="UPDATE %s SET %s WHERE %s"%(table,changeString,identify)
	zLOG.LOG("ZSQLExtend",zLOG.INFO,"CHANGE: "+queryString)

self.ZSQLSimpleSearch(queryString)	self.ZSQLSimpleSearch(queryString)

Line 748 class ZSQLExtendFolder(Folder,Persistent	Line 936 class ZSQLExtendFolder(Folder,Persistent
if field[0]=="_":	if field[0]=="_":
fieldTmp="-"+field[1:]	fieldTmp="-"+field[1:]
else:	else:
fieldTmp=urllib.unqoute(field)	fieldTmp=field

qs+=",%s=%s"%(fieldTmp,argv[field])	qs+=",%s=%s"%(fieldTmp,argv[field])

Line 1188 class ZSQLExtendFolder(Folder,Persistent	Line 1376 class ZSQLExtendFolder(Folder,Persistent
self.REQUEST.SESSION[storename]['rangeEnd']=int(rangeStart)+int(limit)	self.REQUEST.SESSION[storename]['rangeEnd']=int(rangeStart)+int(limit)
self.REQUEST.SESSION[storename]['rangeSize']=limit	self.REQUEST.SESSION[storename]['rangeSize']=limit
self.REQUEST.SESSION[storename]['searchFields']=searchFields	self.REQUEST.SESSION[storename]['searchFields']=searchFields
self.REQUEST.SESSION[storename]['searchFieldsOnly']=searchFieldsOnly

	self.REQUEST.SESSION[storename]['searchFieldsOnly']=searchFieldsOnly

if not NoQuery:	if not NoQuery:

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>

Removed from v.1.79
changed lines
	Added in v.1.92