ZSQLExtend/ZSQLExtend.py - diff

Return to ZSQLExtend.py CVS log

Up to [Repository] / ZSQLExtend

Diff for /ZSQLExtend/ZSQLExtend.py between versions 1.116 and 1.137

version 1.116, 2007/05/25 15:01:32	version 1.137, 2011/02/11 19:08:49
Line 56 def analyseIntSearch(word):	Line 56 def analyseIntSearch(word):
else:	else:
return "BETWEEN "+splitted[0]+" AND "+splitted[1]	return "BETWEEN "+splitted[0]+" AND "+splitted[1]

	def unicodify(str):
	"""decode str (utf-8 or latin-1 representation) into unicode object"""
	if not str:
	return u""
	if type(str) is StringType:
	try:
	return str.decode('utf-8')
	except:
	return str.decode('latin-1')
	else:
	return str

	def utf8ify(str):
	"""encode unicode object or string into byte string in utf-8 representation"""
	if not str:
	return ""
	if type(str) is StringType:
	return str
	else:
	return str.encode('utf-8')


	def setPsycopg2UseUnicode():
	"""force Psycopg2DA to return unicode objects"""
	try:
	import psycopg2
	import psycopg2.extensions
	psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
	except:
	logging.error("Unable to force psycopg2 to use unicode")


def sql_quote(v):	def sql_quote(v):
Line 97 class ZSQLExtendFolder(Folder,Persistent	Line 127 class ZSQLExtendFolder(Folder,Persistent
"""quote str for sql"""	"""quote str for sql"""
return sql_quote(str)	return sql_quote(str)

	def unicodify(self, s):
	"""return unicode object for string (utf-8 or latin1) or unicode object s"""
	return unicodify(s)

	def utf8ify(self, s):
	"""return utf-8 encoded string object for string or unicode object s"""
	return utf8ify(s)


def normalizeField(self,table,fieldname, newFieldName=None,mode="alter", RESPONSE=None):	def normalizeField(self,table,fieldname, newFieldName=None,mode="alter", RESPONSE=None):
"""normalize a field, d.h. entfernt alle diakritischen Zeichen und ersetzt diese	"""normalize a field, d.h. entfernt alle diakritischen Zeichen und ersetzt diese
Line 328 class ZSQLExtendFolder(Folder,Persistent	Line 366 class ZSQLExtendFolder(Folder,Persistent


def importXMLFileFMP(self,table,dsn=None,uploadfile=None,update_fields=None,id_field=None,sync_mode=False,	def importXMLFileFMP(self,table,dsn=None,uploadfile=None,update_fields=None,id_field=None,sync_mode=False,
lc_names=True,keep_fields=False,replace=False,ascii_db=False,	lc_names=True,keep_fields=False,ascii_db=False,replace=False,backup=False,
	debug=False,log_to_response=False,
redirect_url=None,RESPONSE=None):	redirect_url=None,RESPONSE=None):
'''	"""
Import FileMaker XML file (FMPXMLRESULT format) into the table.	Import FileMaker XML file (FMPXMLRESULT format) into the table.
@param dsn: database connection string	@param dsn: database connection string
@param table: name of the table the xml shall be imported into	@param table: name of the table the xml shall be imported into (may be comma-separated list)
@param uploadfile: xmlfile file	@param uploadfile: xmlfile file
@param update_fields: (optional) list of fields to update; default is to create all fields	@param update_fields: (optional) list of fields to update; default is to create all fields
@param id_field: (optional) field which uniquely identifies an entry for updating purposes.	@param id_field: (optional) field which uniquely identifies an entry for updating purposes.
Line 342 class ZSQLExtendFolder(Folder,Persistent	Line 381 class ZSQLExtendFolder(Folder,Persistent
@param keep_fields: (optional) don't add fields to SQL database	@param keep_fields: (optional) don't add fields to SQL database
@param ascii_db: (optional) assume ascii encoding in db	@param ascii_db: (optional) assume ascii encoding in db
@param replace: (optional) delete and re-insert data	@param replace: (optional) delete and re-insert data
	@param backup: (optional) create backup of old table (breaks indices)
@param RESPONSE: (optional)	@param RESPONSE: (optional)
@param redirect_url: (optional) url for redirecting after the upload is done	@param redirect_url: (optional) url for redirecting after the upload is done
'''	"""

tfilehd,filename=tempfile.mkstemp()	tfilehd,filename=tempfile.mkstemp()
tfile=os.fdopen(tfilehd,'w')	tfile=os.fdopen(tfilehd,'w')
logging.info("import %s"%uploadfile)	logging.info("importXMLFileFMP: importing %s"%uploadfile)
for c in uploadfile.read():	for c in uploadfile.read():
tfile.write(c)	tfile.write(c)
tfile.close()	tfile.close()
Line 358 class ZSQLExtendFolder(Folder,Persistent	Line 398 class ZSQLExtendFolder(Folder,Persistent
if not dsn:	if not dsn:
dsn=self.getConnectionObj().connection_string	dsn=self.getConnectionObj().connection_string

	logging.debug("dsn: %s"%dsn)
	logging.debug("table: %s"%table)
	logging.debug("update_fields: %s"%update_fields)
	logging.debug("id_field: %s"%id_field)
	logging.debug("sync_mode: %s"%sync_mode)
	logging.debug("lc_names: %s"%lc_names)
	logging.debug("keep_fields: %s"%keep_fields)
	logging.debug("ascii_db: %s"%ascii_db)
	logging.debug("replace: %s"%replace)
	logging.debug("backup: %s"%backup)
	logging.debug("debug: %s"%debug)
	logging.debug("log_to_response: %s"%log_to_response)
	logging.debug("RESPONSE: %s"%repr(RESPONSE))

	tablelist=table.split(',')
	logging.debug("tablelist: %s"%tablelist)
	#table=tables

	for t in tablelist :
	logging.debug("table: %s"%table)
options=Options()	options=Options()
options.dsn=dsn	options.dsn=dsn
options.table=table	options.table=t
options.filename=filename	options.filename=filename
options.update_fields=update_fields	options.update_fields=update_fields
options.id_field=id_field	options.id_field=id_field
Line 370 class ZSQLExtendFolder(Folder,Persistent	Line 430 class ZSQLExtendFolder(Folder,Persistent
options.keep_fields=keep_fields	options.keep_fields=keep_fields
options.ascii_db=ascii_db	options.ascii_db=ascii_db
options.replace_table=replace	options.replace_table=replace
	options.backup_table=backup
	options.debug=debug

importFMPXML(options)	if RESPONSE and log_to_response:
	# set up logging to response as plain text
	logging.debug("Setting up logging to RESPONSE")
	RESPONSE.setHeader("Content-Type","text/plain; charset=utf-8")
	RESPONSE.write("Import FMPXML file...\n\n")
	RESPONSE.flush()
	loghandler = logging.StreamHandler(RESPONSE)
	if debug:
	loghandler.setLevel(logging.DEBUG)
	else:
	loghandler.setLevel(logging.INFO)
	logger = logging.getLogger('db.import.fmpxml')
	logger.addHandler(loghandler)
	options.use_logger_instance = logger

os.remove(filename)	try:
	err = None
	importFMPXML(options)
	logging.info("importXMLFileFMP: done")
	except Exception, err:
	logging.error("Error importing: %s"%err)

	if RESPONSE and log_to_response:
	loghandler.flush()
	if err is not None:
	RESPONSE.write("\n\nERROR while importing: %s"%err)
	else:
	RESPONSE.write("\n\n DONE!")

if RESPONSE and redirect_url:	elif RESPONSE and redirect_url:
RESPONSE.redirect(redirect_url)	RESPONSE.redirect(redirect_url)

	os.remove(filename)


def generateIndex(self,field,index_name,table,RESPONSE=None):	def generateIndex(self,field,index_name,table,RESPONSE=None):
"""erzeuge ein Index Objekt einem Feld (experimental)	"""erzeuge ein Index Objekt einem Feld (experimental)
Line 417 class ZSQLExtendFolder(Folder,Persistent	Line 506 class ZSQLExtendFolder(Folder,Persistent


def URLquote(self,txt):	def URLquote(self,txt):
"""urlquote"	"""urlquote
@param txt: text der urlgequoted werden soll.	@param txt: text der urlgequoted werden soll.
"""	"""
return urllib.quote(txt)	return urllib.quote(txt)


	def createIdSet(self, resultset, idField=None):
	"""returns a (frozen)set of IDs from a SQL-resultset (using idField) or a list (if idField=None)"""
	logging.debug("createidset for idfield %s"%idField)
	if idField is None:
	return frozenset(resultset)
	else:
	idlist = [r[idField] for r in resultset]
	return frozenset(idlist)

	def opIdSet(self, a, b, op):
	"""operate on sets a and b"""
	logging.debug("opidset with op %s"%op)
	if (op == 'intersect'):
	return a.intersection(b)
	elif (op == 'union'):
	return a.union(b)
	elif (op == 'diff'):
	return a.difference(b)


def searchRel(self,relStatement,statement,wherePart,classes):	def searchRel(self,relStatement,statement,wherePart,classes):
"""suche relative haufigkeiten (experimental)"""	"""suche relative haufigkeiten (experimental)"""
ret={}	ret={}
Line 476 class ZSQLExtendFolder(Folder,Persistent	Line 586 class ZSQLExtendFolder(Folder,Persistent
return pt()	return pt()


def changeZSQLExtend(self,label,description,weight=0,REQUEST=None,connection_id=None):	def changeZSQLExtend(self,label,description,weight=0,connection_id=None,REQUEST=None,):
"""change the Konfiguration"""	"""change the Konfiguration"""
self.connection_id=connection_id	self.connection_id=connection_id
self.weight=weight	self.weight=weight
Line 491 class ZSQLExtendFolder(Folder,Persistent	Line 601 class ZSQLExtendFolder(Folder,Persistent
@param str: string der Formatiert werden soll.	@param str: string der Formatiert werden soll.
@param url: (optional) default ist "None", sonderfall erzeugt einen Link aus String mit unterliegender url	@param url: (optional) default ist "None", sonderfall erzeugt einen Link aus String mit unterliegender url
"""	"""
#url=None	#logging.debug("formatascii str=%s url=%s"%(repr(str),repr(url)))

	if not str:
	return ""

str=str.rstrip().lstrip()	str=str.rstrip().lstrip()

if url and str:	if url and str:
Line 505 class ZSQLExtendFolder(Folder,Persistent	Line 619 class ZSQLExtendFolder(Folder,Persistent
retStr+="""<a href="%s">%s</a><br/>"""%(strUrl,word)	retStr+="""<a href="%s">%s</a><br/>"""%(strUrl,word)
str=retStr	str=retStr
if str:	if str:
return re.sub(r"[\n]","<br/>",str)	retStr = re.sub(r"[\n]","<br/>",str)
	#logging.debug("formatascii out=%s"%(repr(retStr)))
	return retStr
else:	else:
return ""	return ""

Line 560 class ZSQLExtendFolder(Folder,Persistent	Line 676 class ZSQLExtendFolder(Folder,Persistent
def ZSQLMultiSearch(self,_table,_searchField,_value,_idField,_additionalStatement="",_select=None,_subselectAddition="",_storename=None):	def ZSQLMultiSearch(self,_table,_searchField,_value,_idField,_additionalStatement="",_select=None,_subselectAddition="",_storename=None):
"""	"""
Durchsucht in einer Tabelle "table" die Spalte "searchfield" nach dem allen Vorkommnissen	Durchsucht in einer Tabelle "table" die Spalte "searchfield" nach dem allen Vorkommnissen
von Worten in value und gibt alle Werte mit gleichem id field zur�ck, d.h. es wird die "und" suche �ber mehrere Eintrsege in einer	von Worten in value und gibt alle Werte mit gleichem id field zurueck, d.h. es wird die "und" suche ueber mehrere Eintrsege in einer
Tabelle mit gleichem idField werd realisiert,	Tabelle mit gleichem idField werd realisiert,
z.B. f�r simplesearch ueber mehrere Felder	z.B. fuer simplesearch ueber mehrere Felder
@param _table: Tabelle, die durchsucht werden soll.	@param _table: Tabelle, die durchsucht werden soll.
@param _searchField: Feld, das durchsucht wird	@param _searchField: Feld, das durchsucht wird
@param _value: String der gesucht werden soll, gesucht wird nach allen Worten des Strings, die durch " "-getrennt sind.	@param _value: String der gesucht werden soll, gesucht wird nach allen Worten des Strings, die durch " "-getrennt sind.
@param _idField: Feld mit id f�r die identifikation gleicher Eintr�ge	@param _idField: Feld mit id fuer die identifikation gleicher Eintraege
@param _additionalStatement: (optional) Zusaetzliches SQL Statement, dass zwischen dem ersten "select from" und dem ersten "where" eingegef�gt wird.	@param _additionalStatement: (optional) Zusaetzliches SQL Statement, dass zwischen dem ersten "select from" und dem ersten "where" eingegefuegt wird.
@param _select: (optional) Alternativer Wert f�r den ersten SELECT Aufruf.	@param _subselectAddition: (optiona) Zusaetliche SQL Statement die hinter das select statement der subselects eingefuegt werde.
	@param _select: (optional) Alternativer Wert fuer den ersten SELECT Aufruf.
@param _storename: (optional) Name fuer die Zwischenspeicherung von Werten in der Session	@param _storename: (optional) Name fuer die Zwischenspeicherung von Werten in der Session
"""	"""
if _storename:	if _storename:
Line 813 class ZSQLExtendFolder(Folder,Persistent	Line 930 class ZSQLExtendFolder(Folder,Persistent

def ZSQLInlineSearch(self,storename=None,args=None,**argv):	def ZSQLInlineSearch(self,storename=None,args=None,**argv):
"""inlinesearch"""	"""inlinesearch"""
	#logging.debug("ZSQLInlineSearch args=%s argv=%s"%(args,argv))
qs=[]	qs=[]
if storename:	if storename:
"""store"""	"""store"""
else:	else:
storename="foundCount"	storename="foundCount"



if args:	if args:
argTmp=args	argTmp=args
else:	else:
Line 835 class ZSQLExtendFolder(Folder,Persistent	Line 950 class ZSQLExtendFolder(Folder,Persistent
if type(argTmp[a]) is ListType: # ein parameter zweimal	if type(argTmp[a]) is ListType: # ein parameter zweimal
value=""	value=""
#TODO find a better solution, currently only the last non empty entry is used.	#TODO find a better solution, currently only the last non empty entry is used.
for x in argTmp[a]:	#for x in argTmp[a]:
if x:	# if x:
value=x	# value=x
	# version: join with spaces (makes sense with checkbox and -op=all)
	value = " ".join(argTmp[a])
else:	else:
	try:
value=str(argTmp[a])	value=str(argTmp[a])
qs.append(aFiltered+"="+urllib.quote(value))	except:
	value=utf8ify(argTmp[a])

	qs.append(aFiltered+"="+urllib.quote(value))
	#logging.debug("InlineSearch:"+string.join(qs,","))

#return []	#return []

Line 878 class ZSQLExtendFolder(Folder,Persistent	Line 998 class ZSQLExtendFolder(Folder,Persistent
if (hasattr(self,"_v_searchSQL") and (self._v_searchSQL == None)) or (not hasattr(self,"_v_searchSQL")):	if (hasattr(self,"_v_searchSQL") and (self._v_searchSQL == None)) or (not hasattr(self,"_v_searchSQL")):

self._v_searchSQL=Shared.DC.ZRDB.DA.DA("_v_searchSQL","_v_searchSQL",self.getConnectionObj().getId(),"var","<dtml-var var>")	self._v_searchSQL=Shared.DC.ZRDB.DA.DA("_v_searchSQL","_v_searchSQL",self.getConnectionObj().getId(),"var","<dtml-var var>")
	#self._v_searchSQL=self.getConnectionObj()()

self._v_searchSQL.max_rows_=max_rows	self._v_searchSQL.max_rows_=max_rows
	#self._v_searchSQL.set_client_encoding('UNICODE')
try:	try:
logging.error("I am here")	logging.error("I am here")
t=self._v_searchSQL.__call__(var=query)	t=self._v_searchSQL.__call__(var=query)
	#t=self._v_searchSQL.query(query)
logging.error("I am here %s"%t)	logging.error("I am here %s"%t)
return t	return t
except :	except :
Line 896 class ZSQLExtendFolder(Folder,Persistent	Line 1019 class ZSQLExtendFolder(Folder,Persistent
try:	try:

self._v_searchSQL.max_rows_=max_rows	self._v_searchSQL.max_rows_=max_rows
	#self._v_searchSQL.set_client_encoding('UNICODE')

return self._v_searchSQL.__call__(var=query)	return self._v_searchSQL.__call__(var=query)
	#return self._v_searchSQL.query(query)
except :	except :
logger("ZSQLSimpleSearch ERROR2",logging.ERROR, '%s %s'%sys.exc_info()[:2])	logger("ZSQLSimpleSearch ERROR2",logging.ERROR, '%s %s'%sys.exc_info()[:2])
if sys.exc_info()[0]=="Database Error":	if sys.exc_info()[0]=="Database Error":
Line 937 class ZSQLExtendFolder(Folder,Persistent	Line 1062 class ZSQLExtendFolder(Folder,Persistent



def ZSQLAdd(self,format=None,RESPONSE=None,args=None,**argv):	def ZSQLAdd(self,format=None,RESPONSE=None,args=None,_useRequest=True,**argv):
"""Neuer Eintrag"""	"""Neuer Eintrag"""

if args:	if args:
Line 947 class ZSQLExtendFolder(Folder,Persistent	Line 1072 class ZSQLExtendFolder(Folder,Persistent

qs_temp=[]	qs_temp=[]

	if _useRequest:
for a in self.REQUEST.form.keys():	for a in self.REQUEST.form.keys():
qs_temp.append(a+"="+urllib.quote(str(self.REQUEST.form[a])))	qs_temp.append(a+"="+urllib.quote(str(self.REQUEST.form[a])))

Line 963 class ZSQLExtendFolder(Folder,Persistent	Line 1089 class ZSQLExtendFolder(Folder,Persistent

addList={}	addList={}
for q in qs.split(","):	for q in qs.split(","):
	if len(q.split("="))<2:
	continue
name=re.sub("r'+'"," ",q.split("=")[0].lower())	name=re.sub("r'+'"," ",q.split("=")[0].lower())

value=q.split("=")[1]	value=q.split("=")[1]
value=re.sub(r'\+'," ",value)	value=re.sub(r'\+'," ",value)
value=urllib.unquote(value)	value=urllib.unquote(value)
Line 1032 class ZSQLExtendFolder(Folder,Persistent	Line 1161 class ZSQLExtendFolder(Folder,Persistent
table=urllib.unquote(value)	table=urllib.unquote(value)
elif name=="-identify":	elif name=="-identify":
identify=urllib.unquote(value)	identify=urllib.unquote(value)
identify="lower("+identify.split("=")[0]+")="+sql_quote(identify.split("=")[1].lower())	# old code did identify with lower() which doesn't work for oids
	#identify="lower("+identify.split("=")[0]+")="+sql_quote(identify.split("=")[1].lower())
	(k,v) = identify.split("=")
	identify="%s=%s"%(k,sql_quote(v))
elif name=="-format":	elif name=="-format":
format=urllib.unquote(value)	format=urllib.unquote(value)
#elif (not (name[0]=="-" or name[0]=="_")) and (not len(value)==0):	#elif (not (name[0]=="-" or name[0]=="_")) and (not len(value)==0):
Line 1084 class ZSQLExtendFolder(Folder,Persistent	Line 1216 class ZSQLExtendFolder(Folder,Persistent
"""search in database"""	"""search in database"""

def delEmpty(list):	def delEmpty(list):
""""loesche leere elemente aus der liste"""	"""loesche leere elemente aus der liste"""
ret=[]	ret=[]
for x in list:	for x in list:
splitted=x.split("=")	splitted=x.split("=")
Line 1190 class ZSQLExtendFolder(Folder,Persistent	Line 1322 class ZSQLExtendFolder(Folder,Persistent
querys=qs.split(",")	querys=qs.split(",")

#which arguments are in the old query string	#which arguments are in the old query string

queryList={}	queryList={}
for query in querys:	for query in querys:
arg=query.split("=")[0]	arg=query.split("=")[0]
if arg[0]=="_": arg="-"+arg[1:] # sicherstellen, dass an Anfang stets "_"	if arg[0]=="_": arg="-"+arg[1:] # sicherstellen, dass an Anfang stets "_"
try:	try:
queryList[arg]=query.split("=")[1]	queryList[arg]=urllib.unquote_plus(query.split("=")[1])
except:	except:
queryList[arg]=''	queryList[arg]=''

argList=[]	argList=[]
arg=""	arg=""



#gehe durch die zu aendernden Argumente	#gehe durch die zu aendernden Argumente
for argTmp in argv.keys():	for argTmp in argv.keys():

arg=argTmp[0:]# sicherstellen, dass der string auh kopiert wird	arg=argTmp[0:]# sicherstellen, dass der string auh kopiert wird
if arg[0]=="_": arg="-"+arg[1:] # sicherstellen, dass an Anfang stets "_"	if arg[0]=="_": arg="-"+arg[1:] # sicherstellen, dass an Anfang stets "_"

Line 1220 class ZSQLExtendFolder(Folder,Persistent	Line 1347 class ZSQLExtendFolder(Folder,Persistent

return str	return str


def parseQueryString(self,qs,iCT,storemax="no",select=None,nostore=None,storename="foundCount",tableExt=None,NoQuery=None,NoLimit=None,restrictField=None,restrictConnect=None,filter=None):	def parseQueryString(self,qs,iCT,storemax="no",select=None,nostore=None,storename="foundCount",tableExt=None,NoQuery=None,NoLimit=None,restrictField=None,restrictConnect=None,filter=None):
"""analysieren den QueryString"""	"""analysieren den QueryString"""
	logging.debug("parseQueryString qs=%s"%qs)

#setzte generische werte	#setzte generische werte

Line 1378 class ZSQLExtendFolder(Folder,Persistent	Line 1506 class ZSQLExtendFolder(Folder,Persistent
op=opfields[name]	op=opfields[name]
else:	else:
op="ct"	op="ct"

namealt=name	namealt=name
name="LOWER("+punktsplit[1]+")"	name="LOWER("+punktsplit[1]+")"
value=value.lower()	value=value.lower()
Line 1402 class ZSQLExtendFolder(Folder,Persistent	Line 1531 class ZSQLExtendFolder(Folder,Persistent

elif op=="numerical":	elif op=="numerical":
term=analyseIntSearch(value)	term=analyseIntSearch(value)
tmp=(name+" "+term)	tmp=(namealt+" "+term) # take namealt without LOWER
elif op=="grep":	elif op=="grep":
tmp=(name+" ~* "+sql_quote(value))	tmp=(name+" ~* "+sql_quote(value))
elif op=="one":	elif op=="one":
Line 1452 class ZSQLExtendFolder(Folder,Persistent	Line 1581 class ZSQLExtendFolder(Folder,Persistent

elif op=="numerical":	elif op=="numerical":
term=analyseIntSearch(value)	term=analyseIntSearch(value)
tmp=(name+" "+term)	tmp=(namealt+" "+term) # take namealt without LOWER
elif op=="grep":	elif op=="grep":
tmp=(name+" ~* "+sql_quote(value))	tmp=(name+" ~* "+sql_quote(value))
elif op=="one":	elif op=="one":
Line 1734 class ZSQLExtendFolder(Folder,Persistent	Line 1863 class ZSQLExtendFolder(Folder,Persistent
return "<a href='%s'>%s</a>"%(self.REQUEST['URL']+"?"+newquerystring,html)	return "<a href='%s'>%s</a>"%(self.REQUEST['URL']+"?"+newquerystring,html)


	def pydev_settrace(self):
	"""do settrace to start debugging"""
	import pydevd
	pydevd.settrace()


manage_addZSQLExtendFolderForm=DTMLFile('ZSQLExtendFolderAdd', globals())	manage_addZSQLExtendFolderForm=DTMLFile('ZSQLExtendFolderAdd', globals())
Line 2047 def manage_addZSQLBibliography(self, id,	Line 2180 def manage_addZSQLBibliography(self, id,

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>

Removed from v.1.116
changed lines
	Added in v.1.137