ZSQLExtend/ZSQLExtend.py - diff

Return to ZSQLExtend.py CVS log

Up to [Repository] / ZSQLExtend

Diff for /ZSQLExtend/ZSQLExtend.py between versions 1.118 and 1.141

version 1.118, 2007/07/31 11:28:48	version 1.141, 2011/02/23 19:43:04
Line 10 from Products.PageTemplates.ZopePageTemp	Line 10 from Products.PageTemplates.ZopePageTemp
from Products.PageTemplates.PageTemplateFile import PageTemplateFile	from Products.PageTemplates.PageTemplateFile import PageTemplateFile

from Products.ZSQLMethods.SQL import SQLConnectionIDs	from Products.ZSQLMethods.SQL import SQLConnectionIDs
	from Shared.DC.ZRDB.Results import Results

from xml.sax.saxutils import escape	from xml.sax.saxutils import escape
from types import *	from types import *
import Shared.DC.ZRDB.DA	import Shared.DC.ZRDB.DA
Line 78 def utf8ify(str):	Line 80 def utf8ify(str):
return str.encode('utf-8')	return str.encode('utf-8')


	def setPsycopg2UseUnicode():
	"""force Psycopg2DA to return unicode objects"""
	try:
	import psycopg2
	import psycopg2.extensions
	psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
	except:
	logging.error("Unable to force psycopg2 to use unicode")


def sql_quote(v):	def sql_quote(v):
# quote dictionary	# quote dictionary
Line 118 class ZSQLExtendFolder(Folder,Persistent	Line 129 class ZSQLExtendFolder(Folder,Persistent
"""quote str for sql"""	"""quote str for sql"""
return sql_quote(str)	return sql_quote(str)

	def unicodify(self, s):
	"""return unicode object for string (utf-8 or latin1) or unicode object s"""
	return unicodify(s)

	def utf8ify(self, s):
	"""return utf-8 encoded string object for string or unicode object s"""
	return utf8ify(s)


def normalizeField(self,table,fieldname, newFieldName=None,mode="alter", RESPONSE=None):	def normalizeField(self,table,fieldname, newFieldName=None,mode="alter", RESPONSE=None):
"""normalize a field, d.h. entfernt alle diakritischen Zeichen und ersetzt diese	"""normalize a field, d.h. entfernt alle diakritischen Zeichen und ersetzt diese
Line 350 class ZSQLExtendFolder(Folder,Persistent	Line 369 class ZSQLExtendFolder(Folder,Persistent

def importXMLFileFMP(self,table,dsn=None,uploadfile=None,update_fields=None,id_field=None,sync_mode=False,	def importXMLFileFMP(self,table,dsn=None,uploadfile=None,update_fields=None,id_field=None,sync_mode=False,
lc_names=True,keep_fields=False,ascii_db=False,replace=False,backup=False,	lc_names=True,keep_fields=False,ascii_db=False,replace=False,backup=False,
	debug=False,log_to_response=False,
redirect_url=None,RESPONSE=None):	redirect_url=None,RESPONSE=None):
'''	"""
Import FileMaker XML file (FMPXMLRESULT format) into the table.	Import FileMaker XML file (FMPXMLRESULT format) into the table.
@param dsn: database connection string	@param dsn: database connection string
@param table: name of the table the xml shall be imported into	@param table: name of the table the xml shall be imported into (may be comma-separated list)
@param uploadfile: xmlfile file	@param uploadfile: xmlfile file
@param update_fields: (optional) list of fields to update; default is to create all fields	@param update_fields: (optional) list of fields to update; default is to create all fields
@param id_field: (optional) field which uniquely identifies an entry for updating purposes.	@param id_field: (optional) field which uniquely identifies an entry for updating purposes.
Line 366 class ZSQLExtendFolder(Folder,Persistent	Line 386 class ZSQLExtendFolder(Folder,Persistent
@param backup: (optional) create backup of old table (breaks indices)	@param backup: (optional) create backup of old table (breaks indices)
@param RESPONSE: (optional)	@param RESPONSE: (optional)
@param redirect_url: (optional) url for redirecting after the upload is done	@param redirect_url: (optional) url for redirecting after the upload is done
'''	"""

tfilehd,filename=tempfile.mkstemp()	tfilehd,filename=tempfile.mkstemp()
tfile=os.fdopen(tfilehd,'w')	tfile=os.fdopen(tfilehd,'w')
logging.info("import %s"%uploadfile)	logging.info("importXMLFileFMP: importing %s"%uploadfile)
for c in uploadfile.read():	for c in uploadfile.read():
tfile.write(c)	tfile.write(c)
tfile.close()	tfile.close()
Line 380 class ZSQLExtendFolder(Folder,Persistent	Line 400 class ZSQLExtendFolder(Folder,Persistent
if not dsn:	if not dsn:
dsn=self.getConnectionObj().connection_string	dsn=self.getConnectionObj().connection_string

	logging.debug("dsn: %s"%dsn)
	logging.debug("table: %s"%table)
	logging.debug("update_fields: %s"%update_fields)
	logging.debug("id_field: %s"%id_field)
	logging.debug("sync_mode: %s"%sync_mode)
	logging.debug("lc_names: %s"%lc_names)
	logging.debug("keep_fields: %s"%keep_fields)
	logging.debug("ascii_db: %s"%ascii_db)
	logging.debug("replace: %s"%replace)
	logging.debug("backup: %s"%backup)
	logging.debug("debug: %s"%debug)
	logging.debug("log_to_response: %s"%log_to_response)
	logging.debug("RESPONSE: %s"%repr(RESPONSE))

	tablelist=table.split(',')
	logging.debug("tablelist: %s"%tablelist)
	#table=tables

	for t in tablelist :
	logging.debug("table: %s"%table)
options=Options()	options=Options()
options.dsn=dsn	options.dsn=dsn
options.table=table	options.table=t
options.filename=filename	options.filename=filename
options.update_fields=update_fields	options.update_fields=update_fields
options.id_field=id_field	options.id_field=id_field
Line 393 class ZSQLExtendFolder(Folder,Persistent	Line 433 class ZSQLExtendFolder(Folder,Persistent
options.ascii_db=ascii_db	options.ascii_db=ascii_db
options.replace_table=replace	options.replace_table=replace
options.backup_table=backup	options.backup_table=backup
	options.debug=debug

importFMPXML(options)	if RESPONSE and log_to_response:
	# set up logging to response as plain text
	logging.debug("Setting up logging to RESPONSE")
	RESPONSE.setHeader("Content-Type","text/plain; charset=utf-8")
	RESPONSE.write("Import FMPXML file...\n\n")
	RESPONSE.flush()
	loghandler = logging.StreamHandler(RESPONSE)
	if debug:
	loghandler.setLevel(logging.DEBUG)
	else:
	loghandler.setLevel(logging.INFO)
	logger = logging.getLogger('db.import.fmpxml')
	logger.addHandler(loghandler)
	options.use_logger_instance = logger

os.remove(filename)	try:
	err = None
	importFMPXML(options)
	logging.info("importXMLFileFMP: done")
	except Exception, err:
	logging.error("Error importing: %s"%err)

	if RESPONSE and log_to_response:
	loghandler.flush()
	if err is not None:
	RESPONSE.write("\n\nERROR while importing: %s"%err)
	else:
	RESPONSE.write("\n\n DONE!")

if RESPONSE and redirect_url:	elif RESPONSE and redirect_url:
RESPONSE.redirect(redirect_url)	RESPONSE.redirect(redirect_url)

	os.remove(filename)


def generateIndex(self,field,index_name,table,RESPONSE=None):	def generateIndex(self,field,index_name,table,RESPONSE=None):
"""erzeuge ein Index Objekt einem Feld (experimental)	"""erzeuge ein Index Objekt einem Feld (experimental)
Line 440 class ZSQLExtendFolder(Folder,Persistent	Line 508 class ZSQLExtendFolder(Folder,Persistent


def URLquote(self,txt):	def URLquote(self,txt):
"""urlquote"	"""urlquote
@param txt: text der urlgequoted werden soll.	@param txt: text der urlgequoted werden soll.
"""	"""
return urllib.quote(txt)	return urllib.quote(txt)


	def createIdSet(self, resultset, idField=None):
	"""returns a (frozen)set of IDs from a SQL-resultset (using idField) or a list (if idField=None)"""
	logging.debug("createidset for idfield %s"%idField)
	if idField is None:
	return frozenset(resultset)
	else:
	idlist = [r[idField] for r in resultset]
	return frozenset(idlist)

	def opIdSet(self, a, b, op):
	"""operate on sets a and b"""
	logging.debug("opidset with op %s"%op)
	if (op == 'intersect'):
	return a.intersection(b)
	elif (op == 'union'):
	return a.union(b)
	elif (op == 'diff'):
	return a.difference(b)


def searchRel(self,relStatement,statement,wherePart,classes):	def searchRel(self,relStatement,statement,wherePart,classes):
"""suche relative haufigkeiten (experimental)"""	"""suche relative haufigkeiten (experimental)"""
ret={}	ret={}
Line 499 class ZSQLExtendFolder(Folder,Persistent	Line 588 class ZSQLExtendFolder(Folder,Persistent
return pt()	return pt()


def changeZSQLExtend(self,label,description,weight=0,REQUEST=None,connection_id=None):	def changeZSQLExtend(self,label,description,weight=0,connection_id=None,autocommit=None,REQUEST=None,):
"""change the Konfiguration"""	"""change the Konfiguration"""
self.connection_id=connection_id	self.connection_id=connection_id
self.weight=weight	self.weight=weight
self.label=label	self.label=label
self.description=description	self.description=description
	self.autocommit = (autocommit == "on")

if REQUEST is not None:	if REQUEST is not None:
return self.manage_main(self, REQUEST)	return self.manage_main(self, REQUEST)
Line 589 class ZSQLExtendFolder(Folder,Persistent	Line 679 class ZSQLExtendFolder(Folder,Persistent
def ZSQLMultiSearch(self,_table,_searchField,_value,_idField,_additionalStatement="",_select=None,_subselectAddition="",_storename=None):	def ZSQLMultiSearch(self,_table,_searchField,_value,_idField,_additionalStatement="",_select=None,_subselectAddition="",_storename=None):
"""	"""
Durchsucht in einer Tabelle "table" die Spalte "searchfield" nach dem allen Vorkommnissen	Durchsucht in einer Tabelle "table" die Spalte "searchfield" nach dem allen Vorkommnissen
von Worten in value und gibt alle Werte mit gleichem id field zur�ck, d.h. es wird die "und" suche �ber mehrere Eintrsege in einer	von Worten in value und gibt alle Werte mit gleichem id field zurueck, d.h. es wird die "und" suche ueber mehrere Eintrsege in einer
Tabelle mit gleichem idField werd realisiert,	Tabelle mit gleichem idField werd realisiert,
z.B. f�r simplesearch ueber mehrere Felder	z.B. fuer simplesearch ueber mehrere Felder
@param _table: Tabelle, die durchsucht werden soll.	@param _table: Tabelle, die durchsucht werden soll.
@param _searchField: Feld, das durchsucht wird	@param _searchField: Feld, das durchsucht wird
@param _value: String der gesucht werden soll, gesucht wird nach allen Worten des Strings, die durch " "-getrennt sind.	@param _value: String der gesucht werden soll, gesucht wird nach allen Worten des Strings, die durch " "-getrennt sind.
@param _idField: Feld mit id f�r die identifikation gleicher Eintr�ge	@param _idField: Feld mit id fuer die identifikation gleicher Eintraege
@param _additionalStatement: (optional) Zusaetzliches SQL Statement, dass zwischen dem ersten "select from" und dem ersten "where" eingegef�gt wird.	@param _additionalStatement: (optional) Zusaetzliches SQL Statement, dass zwischen dem ersten "select from" und dem ersten "where" eingegefuegt wird.
@param _select: (optional) Alternativer Wert f�r den ersten SELECT Aufruf.	@param _subselectAddition: (optiona) Zusaetliche SQL Statement die hinter das select statement der subselects eingefuegt werde.
	@param _select: (optional) Alternativer Wert fuer den ersten SELECT Aufruf.
@param _storename: (optional) Name fuer die Zwischenspeicherung von Werten in der Session	@param _storename: (optional) Name fuer die Zwischenspeicherung von Werten in der Session
"""	"""
if _storename:	if _storename:
Line 842 class ZSQLExtendFolder(Folder,Persistent	Line 933 class ZSQLExtendFolder(Folder,Persistent

def ZSQLInlineSearch(self,storename=None,args=None,**argv):	def ZSQLInlineSearch(self,storename=None,args=None,**argv):
"""inlinesearch"""	"""inlinesearch"""
	#logging.debug("ZSQLInlineSearch args=%s argv=%s"%(args,argv))
qs=[]	qs=[]
if storename:	if storename:
"""store"""	"""store"""
else:	else:
storename="foundCount"	storename="foundCount"



if args:	if args:
argTmp=args	argTmp=args
else:	else:
Line 864 class ZSQLExtendFolder(Folder,Persistent	Line 953 class ZSQLExtendFolder(Folder,Persistent
if type(argTmp[a]) is ListType: # ein parameter zweimal	if type(argTmp[a]) is ListType: # ein parameter zweimal
value=""	value=""
#TODO find a better solution, currently only the last non empty entry is used.	#TODO find a better solution, currently only the last non empty entry is used.
for x in argTmp[a]:	#for x in argTmp[a]:
if x:	# if x:
value=x	# value=x
	# version: join with spaces (makes sense with checkbox and -op=all)
	value = " ".join(argTmp[a])
else:	else:
	try:
value=str(argTmp[a])	value=str(argTmp[a])
qs.append(aFiltered+"="+urllib.quote(value))	except:
	value=utf8ify(argTmp[a])

	qs.append(aFiltered+"="+urllib.quote(value))
	#logging.debug("InlineSearch:"+string.join(qs,","))

#return []	#return []

Line 896 class ZSQLExtendFolder(Folder,Persistent	Line 990 class ZSQLExtendFolder(Folder,Persistent
except:	except:
logger("ZSQLResetConnection",logging.ERROR, '%s %s'%sys.exc_info()[:2])	logger("ZSQLResetConnection",logging.ERROR, '%s %s'%sys.exc_info()[:2])


def ZSQLSimpleSearch(self,query=None,max_rows=1000000):	def ZSQLSimpleSearch(self,query=None,max_rows=1000000):
	"""new simple search"""
	logging.debug("new ZSQLSimpleSearch X %s"%query)
	# get Connection instance
	con = self.getConnectionObj()
	# call to get db object
	dbc = con()
	if getattr(self, 'autocommit', False):
	# force transaction isolation level (for psycopg2 0=autocommit)
	logging.debug(" old tilevel="+dbc.tilevel)
	dbc.tilevel = 0
	# modified code from ZPsycopgDA.db without _register:
	c = dbc.getcursor()
	desc = ()
	r = []
	try:
	try:
	c.execute(qs)

	except psycopg2.OperationalError:
	#logging.exception("Operational error on connection, closing it.")
	try:
	# Only close our connection
	dbc.putconn(True)
	except:
	#logging.debug("Something went wrong when we tried to close the pool", exc_info=True)
	pass

	if c.description is not None:
	if max_rows:
	r = c.fetchmany(max_rows)
	else:
	r = c.fetchall()
	desc = c.description

	dbc.failures = 0

	except StandardError, err:
	raise err

	res = (dbc.convert_description(desc), r)

	else:
	# just use DA's query method
	res = dbc.query(query, max_rows=max_rows)

	# return result set as Result object with Brains
	return Results(res)

	def oldZSQLSimpleSearch(self,query=None,max_rows=1000000):
"""simple search"""	"""simple search"""
logging.error("ZSQLSimpleSearch X %s"%query)	logging.error("ZSQLSimpleSearch X %s"%query)
#print query	#print query
Line 971 class ZSQLExtendFolder(Folder,Persistent	Line 1115 class ZSQLExtendFolder(Folder,Persistent



def ZSQLAdd(self,format=None,RESPONSE=None,args=None,**argv):	def ZSQLAdd(self,format=None,RESPONSE=None,args=None,_useRequest=True,**argv):
"""Neuer Eintrag"""	"""Neuer Eintrag"""

if args:	if args:
Line 981 class ZSQLExtendFolder(Folder,Persistent	Line 1125 class ZSQLExtendFolder(Folder,Persistent

qs_temp=[]	qs_temp=[]

	if _useRequest:
for a in self.REQUEST.form.keys():	for a in self.REQUEST.form.keys():
qs_temp.append(a+"="+urllib.quote(str(self.REQUEST.form[a])))	qs_temp.append(a+"="+urllib.quote(str(self.REQUEST.form[a])))

Line 997 class ZSQLExtendFolder(Folder,Persistent	Line 1142 class ZSQLExtendFolder(Folder,Persistent

addList={}	addList={}
for q in qs.split(","):	for q in qs.split(","):
	if len(q.split("="))<2:
	continue
name=re.sub("r'+'"," ",q.split("=")[0].lower())	name=re.sub("r'+'"," ",q.split("=")[0].lower())

value=q.split("=")[1]	value=q.split("=")[1]
value=re.sub(r'\+'," ",value)	value=re.sub(r'\+'," ",value)
value=urllib.unquote(value)	value=urllib.unquote(value)
Line 1066 class ZSQLExtendFolder(Folder,Persistent	Line 1214 class ZSQLExtendFolder(Folder,Persistent
table=urllib.unquote(value)	table=urllib.unquote(value)
elif name=="-identify":	elif name=="-identify":
identify=urllib.unquote(value)	identify=urllib.unquote(value)
identify="lower("+identify.split("=")[0]+")="+sql_quote(identify.split("=")[1].lower())	# old code did identify with lower() which doesn't work for oids
	#identify="lower("+identify.split("=")[0]+")="+sql_quote(identify.split("=")[1].lower())
	(k,v) = identify.split("=")
	identify="%s=%s"%(k,sql_quote(v))
elif name=="-format":	elif name=="-format":
format=urllib.unquote(value)	format=urllib.unquote(value)
#elif (not (name[0]=="-" or name[0]=="_")) and (not len(value)==0):	#elif (not (name[0]=="-" or name[0]=="_")) and (not len(value)==0):
Line 1118 class ZSQLExtendFolder(Folder,Persistent	Line 1269 class ZSQLExtendFolder(Folder,Persistent
"""search in database"""	"""search in database"""

def delEmpty(list):	def delEmpty(list):
""""loesche leere elemente aus der liste"""	"""loesche leere elemente aus der liste"""
ret=[]	ret=[]
for x in list:	for x in list:
splitted=x.split("=")	splitted=x.split("=")
Line 1224 class ZSQLExtendFolder(Folder,Persistent	Line 1375 class ZSQLExtendFolder(Folder,Persistent
querys=qs.split(",")	querys=qs.split(",")

#which arguments are in the old query string	#which arguments are in the old query string

queryList={}	queryList={}
for query in querys:	for query in querys:
arg=query.split("=")[0]	arg=query.split("=")[0]
if arg[0]=="_": arg="-"+arg[1:] # sicherstellen, dass an Anfang stets "_"	if arg[0]=="_": arg="-"+arg[1:] # sicherstellen, dass an Anfang stets "_"
try:	try:
queryList[arg]=query.split("=")[1]	queryList[arg]=urllib.unquote_plus(query.split("=")[1])
except:	except:
queryList[arg]=''	queryList[arg]=''

argList=[]	argList=[]
arg=""	arg=""



#gehe durch die zu aendernden Argumente	#gehe durch die zu aendernden Argumente
for argTmp in argv.keys():	for argTmp in argv.keys():

arg=argTmp[0:]# sicherstellen, dass der string auh kopiert wird	arg=argTmp[0:]# sicherstellen, dass der string auh kopiert wird
if arg[0]=="_": arg="-"+arg[1:] # sicherstellen, dass an Anfang stets "_"	if arg[0]=="_": arg="-"+arg[1:] # sicherstellen, dass an Anfang stets "_"

Line 1254 class ZSQLExtendFolder(Folder,Persistent	Line 1400 class ZSQLExtendFolder(Folder,Persistent

return str	return str


def parseQueryString(self,qs,iCT,storemax="no",select=None,nostore=None,storename="foundCount",tableExt=None,NoQuery=None,NoLimit=None,restrictField=None,restrictConnect=None,filter=None):	def parseQueryString(self,qs,iCT,storemax="no",select=None,nostore=None,storename="foundCount",tableExt=None,NoQuery=None,NoLimit=None,restrictField=None,restrictConnect=None,filter=None):
"""analysieren den QueryString"""	"""analysieren den QueryString"""
	logging.debug("parseQueryString qs=%s"%qs)

#setzte generische werte	#setzte generische werte

Line 1412 class ZSQLExtendFolder(Folder,Persistent	Line 1559 class ZSQLExtendFolder(Folder,Persistent
op=opfields[name]	op=opfields[name]
else:	else:
op="ct"	op="ct"

namealt=name	namealt=name
name="LOWER("+punktsplit[1]+")"	name="LOWER("+punktsplit[1]+")"
value=value.lower()	value=value.lower()
Line 1436 class ZSQLExtendFolder(Folder,Persistent	Line 1584 class ZSQLExtendFolder(Folder,Persistent

elif op=="numerical":	elif op=="numerical":
term=analyseIntSearch(value)	term=analyseIntSearch(value)
tmp=(name+" "+term)	tmp=(namealt+" "+term) # take namealt without LOWER
elif op=="grep":	elif op=="grep":
tmp=(name+" ~* "+sql_quote(value))	tmp=(name+" ~* "+sql_quote(value))
elif op=="one":	elif op=="one":
Line 1486 class ZSQLExtendFolder(Folder,Persistent	Line 1634 class ZSQLExtendFolder(Folder,Persistent

elif op=="numerical":	elif op=="numerical":
term=analyseIntSearch(value)	term=analyseIntSearch(value)
tmp=(name+" "+term)	tmp=(namealt+" "+term) # take namealt without LOWER
elif op=="grep":	elif op=="grep":
tmp=(name+" ~* "+sql_quote(value))	tmp=(name+" ~* "+sql_quote(value))
elif op=="one":	elif op=="one":
Line 1768 class ZSQLExtendFolder(Folder,Persistent	Line 1916 class ZSQLExtendFolder(Folder,Persistent
return "<a href='%s'>%s</a>"%(self.REQUEST['URL']+"?"+newquerystring,html)	return "<a href='%s'>%s</a>"%(self.REQUEST['URL']+"?"+newquerystring,html)


	def pydev_settrace(self):
	"""do settrace to start debugging"""
	import pydevd
	pydevd.settrace()


manage_addZSQLExtendFolderForm=DTMLFile('ZSQLExtendFolderAdd', globals())	manage_addZSQLExtendFolderForm=DTMLFile('ZSQLExtendFolderAdd', globals())
Line 2081 def manage_addZSQLBibliography(self, id,	Line 2233 def manage_addZSQLBibliography(self, id,

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>

Removed from v.1.118
changed lines
	Added in v.1.141