--- ZSQLExtend/ZSQLExtend.py 2010/05/11 13:35:14 1.134
+++ ZSQLExtend/ZSQLExtend.py 2012/09/03 13:06:11 1.144
@@ -5,11 +5,14 @@ import urllib
import re
import string
#from pyPgSQL import libpq
+import psycopg2
from AccessControl import getSecurityManager,Unauthorized
from Products.PageTemplates.ZopePageTemplate import ZopePageTemplate
from Products.PageTemplates.PageTemplateFile import PageTemplateFile
from Products.ZSQLMethods.SQL import SQLConnectionIDs
+from Shared.DC.ZRDB.Results import Results
+
from xml.sax.saxutils import escape
from types import *
import Shared.DC.ZRDB.DA
@@ -96,8 +99,8 @@ def sql_quote(v):
v=string.join(string.split(v,dkey),quote_dict[dkey])
return "'%s'" % v
-def showSQLConnectionIDs(self):
- return SQLConnectionIDs(self)
+#def showSQLConnectionIDs(self):
+# return SQLConnectionIDs(self)
class Options:
"""options class"""
@@ -123,6 +126,41 @@ class ZSQLExtendFolder(Folder,Persistent
"""
meta_type="ZSQLExtendFolder"
+ #primaryKeys={}
+ #primaryKey=
+
+ def getSQLConnectionIDs(self):
+ logging.debug(SQLConnectionIDs(self))
+ return SQLConnectionIDs(self)
+
+ def getPrimaryKey(self,table=None):
+ """returns primary key for the database"""
+ if table is None:
+ return getattr(self,'primaryKey','oid')
+
+
+
+ pks= getattr(self, "primaryKeys",{})
+
+ k = pks.get(table,None)
+
+ if k is None:
+ logging.debug("getPrimaryKey: no primary key for table %s stored use standard key %s"%(table,getattr(self,'primaryKey','oid')))
+ logging.debug(pks)
+ return getattr(self,'primaryKey','oid')
+
+ return k
+
+
+ def getPrimaryKeysAsString(self):
+ """get all keys"""
+
+ if not hasattr(self, 'primaryKeys'):
+ return ""
+
+ return ";".join(["%s:%s"%val for val in self.primaryKeys.items()])
+
+
def ZSQLQuote(self,str):
"""quote str for sql"""
return sql_quote(str)
@@ -160,11 +198,11 @@ class ZSQLExtendFolder(Folder,Persistent
qstr="""alter table %s add %s %s"""
self.ZSQLSimpleSearch(qstr%(table,newFieldName,'text'))
- qstr="select oid,%s from %s"%(fieldname,table)
+ qstr="select %s from %s"%(fieldname,table)
for result in self.ZSQLSimpleSearch(qstr):
- qstr="update %s set %s = %s where oid = %s"
+ qstr="update %s set %s = %s where "+self.getPrimaryKey()+" = %s"
- self.ZSQLSimpleSearch(qstr%(table,newFieldName,self.ZSQLQuote(normal(getattr(result,fieldname))),result.oid))
+ self.ZSQLSimpleSearch(qstr%(table,newFieldName,self.ZSQLQuote(normal(getattr(result,fieldname))),getattr(result,self.getPrimaryKey)))
def importAccessModell(self,configFileName,RESPONSE=None):
"""import tables from access
@@ -369,7 +407,7 @@ class ZSQLExtendFolder(Folder,Persistent
lc_names=True,keep_fields=False,ascii_db=False,replace=False,backup=False,
debug=False,log_to_response=False,
redirect_url=None,RESPONSE=None):
- '''
+ """
Import FileMaker XML file (FMPXMLRESULT format) into the table.
@param dsn: database connection string
@param table: name of the table the xml shall be imported into (may be comma-separated list)
@@ -384,7 +422,7 @@ class ZSQLExtendFolder(Folder,Persistent
@param backup: (optional) create backup of old table (breaks indices)
@param RESPONSE: (optional)
@param redirect_url: (optional) url for redirecting after the upload is done
- '''
+ """
tfilehd,filename=tempfile.mkstemp()
tfile=os.fdopen(tfilehd,'w')
@@ -432,7 +470,7 @@ class ZSQLExtendFolder(Folder,Persistent
options.replace_table=replace
options.backup_table=backup
options.debug=debug
-
+ logging.debug(options)
if RESPONSE and log_to_response:
# set up logging to response as plain text
logging.debug("Setting up logging to RESPONSE")
@@ -453,12 +491,12 @@ class ZSQLExtendFolder(Folder,Persistent
importFMPXML(options)
logging.info("importXMLFileFMP: done")
except Exception, err:
- logging.error("Error importing: "+err)
+ logging.error("Error importing: %s"%err)
if RESPONSE and log_to_response:
loghandler.flush()
if err is not None:
- RESPONSE.write("\n\nERROR while importing:"+err)
+ RESPONSE.write("\n\nERROR while importing: %s"%err)
else:
RESPONSE.write("\n\n DONE!")
@@ -476,7 +514,7 @@ class ZSQLExtendFolder(Folder,Persistent
index={}
- founds=self.ZSQLSimpleSearch("""SELECT %s,oid FROM %s """%(field,table))
+ founds=self.ZSQLSimpleSearch("""SELECT %s FROM %s """%(field,table))
for found in founds:
tmp=getattr(found,field,None)
@@ -484,9 +522,9 @@ class ZSQLExtendFolder(Folder,Persistent
strings=tmp.split(" ")
for string in strings:
if index.get(string):
- index[string].append(found.oid)
+ index[string].append(gettr(found,self.getPrimaryKey()))
else:
- index[string]=[found.oid]
+ index[string]=[gettr(found,self.getPrimaryKey())]
RESPONSE.write(string+"\n")
if not hasattr(self,index_name):
@@ -506,7 +544,7 @@ class ZSQLExtendFolder(Folder,Persistent
def URLquote(self,txt):
- """urlquote"
+ """urlquote
@param txt: text der urlgequoted werden soll.
"""
return urllib.quote(txt)
@@ -586,13 +624,29 @@ class ZSQLExtendFolder(Folder,Persistent
return pt()
- def changeZSQLExtend(self,label,description,weight=0,connection_id=None,REQUEST=None,):
+ def changeZSQLExtend(self,label,description,primaryKey,primaryKeys="",weight=0,connection_id=None,autocommit=None,REQUEST=None,):
"""change the Konfiguration"""
self.connection_id=connection_id
self.weight=weight
self.label=label
self.description=description
+ self.autocommit = (autocommit == "on")
+ self.primaryKey=primaryKey
+ if primaryKeys.lstrip().rstrip()!="":
+
+ for vals in primaryKeys.split(";"):
+ splitted=vals.split(":")
+ if len(splitted)<2:
+ if REQUEST is not None:
+ return """
Wrong Syntax of keystring: %s
%s is not of the form TABLE:key."""%(primaryKeys,vals)
+
+ self.primaryKeys[splitted[0]]=splitted[1]
+
+
+
+
+
if REQUEST is not None:
return self.manage_main(self, REQUEST)
@@ -930,7 +984,7 @@ class ZSQLExtendFolder(Folder,Persistent
def ZSQLInlineSearch(self,storename=None,args=None,**argv):
"""inlinesearch"""
-
+ #logging.debug("ZSQLInlineSearch args=%s argv=%s"%(args,argv))
qs=[]
if storename:
"""store"""
@@ -950,14 +1004,19 @@ class ZSQLExtendFolder(Folder,Persistent
if type(argTmp[a]) is ListType: # ein parameter zweimal
value=""
#TODO find a better solution, currently only the last non empty entry is used.
- for x in argTmp[a]:
- if x:
- value=x
+ #for x in argTmp[a]:
+ # if x:
+ # value=x
+ # version: join with spaces (makes sense with checkbox and -op=all)
+ value = " ".join(argTmp[a])
else:
-
+ try:
value=str(argTmp[a])
+ except:
+ value=utf8ify(argTmp[a])
+
qs.append(aFiltered+"="+urllib.quote(value))
-
+ #logging.debug("InlineSearch:"+string.join(qs,","))
#return []
@@ -982,7 +1041,58 @@ class ZSQLExtendFolder(Folder,Persistent
except:
logger("ZSQLResetConnection",logging.ERROR, '%s %s'%sys.exc_info()[:2])
+
def ZSQLSimpleSearch(self,query=None,max_rows=1000000):
+ """new simple search"""
+ logging.debug("new ZSQLSimpleSearch %s"%query)
+ # get Connection instance
+ con = self.getConnectionObj()
+ # call to get db object
+ dbc = con()
+ if getattr(self, 'autocommit', False):
+ # force transaction isolation level (for psycopg2 0=autocommit)
+ logging.debug(" old tilevel=%s"%dbc.tilevel)
+ dbc.tilevel = 0
+ # modified code from ZPsycopgDA.db without _register:
+ c = dbc.getcursor()
+ desc = ()
+ r = []
+ try:
+ try:
+ c.execute(query)
+
+ except psycopg2.OperationalError:
+ #logging.exception("Operational error on connection, closing it.")
+ try:
+ # Only close our connection
+ dbc.putconn(True)
+ except:
+ #logging.debug("Something went wrong when we tried to close the pool", exc_info=True)
+ pass
+
+ if c.description is not None:
+ if max_rows:
+ r = c.fetchmany(max_rows)
+ else:
+ r = c.fetchall()
+ desc = c.description
+
+ dbc.failures = 0
+
+ except StandardError, err:
+ raise err
+
+ res = (dbc.convert_description(desc), r)
+
+ else:
+ logging.debug(" no autocommit")
+ # just use DA's query method
+ res = dbc.query(query, max_rows=max_rows)
+
+ # return result set as Result object with Brains
+ return Results(res)
+
+ def oldZSQLSimpleSearch(self,query=None,max_rows=1000000):
"""simple search"""
logging.error("ZSQLSimpleSearch X %s"%query)
#print query
@@ -1184,7 +1294,7 @@ class ZSQLExtendFolder(Folder,Persistent
return True
- def ZSQLFindIndexed(self,tableList=[],qs="",select="oid,*",storename=None,indexedFields=['data_line'],restrictField='id_text',**argv):
+ def ZSQLFindIndexed(self,tableList=[],qs="",select="*",storename=None,indexedFields=['data_line'],restrictField='id_text',**argv):
"""find2"""
for index in self.ZopeFind(self,obj_ids=indexedFields):
@@ -1197,7 +1307,7 @@ class ZSQLExtendFolder(Folder,Persistent
search2 = self.ZSQLFind(tableExt=tableList[0],qs=qs,select=select,storename=storename,restrictConnect=(tableList[0]+"."+restrictField,search1),**argv)
return search2
- def ZSQLFind2(self,qs="",select="oid,*",storename=None,tableList=['cdli_translit','cdli_cat'],restrictField='id_text',**argv):
+ def ZSQLFind2(self,qs="",select="*",storename=None,tableList=['cdli_translit','cdli_cat'],restrictField='id_text',**argv):
"""find2"""
search1= self.ZSQLFind(qs=qs,select=select,storename=storename,tableExt=tableList[1],restrictField=restrictField,NoQuery='yes',NoLimit='yes',**argv)
@@ -1207,11 +1317,11 @@ class ZSQLExtendFolder(Folder,Persistent
- def ZSQLFind(self,qs="",select="oid,*",storename="foundCount",tableExt=None,NoQuery=None,NoLimit=None,restrictField=None,restrictConnect=None,filter=None,**argv):
+ def ZSQLFind(self,qs="",select="*",storename="foundCount",tableExt=None,NoQuery=None,NoLimit=None,restrictField=None,restrictConnect=None,filter=None,**argv):
"""search in database"""
def delEmpty(list):
- """"loesche leere elemente aus der liste"""
+ """loesche leere elemente aus der liste"""
ret=[]
for x in list:
splitted=x.split("=")
@@ -1345,7 +1455,7 @@ class ZSQLExtendFolder(Folder,Persistent
def parseQueryString(self,qs,iCT,storemax="no",select=None,nostore=None,storename="foundCount",tableExt=None,NoQuery=None,NoLimit=None,restrictField=None,restrictConnect=None,filter=None):
"""analysieren den QueryString"""
-
+ logging.debug("parseQueryString qs=%s"%qs)
#setzte generische werte
@@ -1370,7 +1480,7 @@ class ZSQLExtendFolder(Folder,Persistent
debug=None
if not select:
- select="oid,*"
+ select="*"
#check for op
splitted=qs.split(",")
@@ -1382,17 +1492,41 @@ class ZSQLExtendFolder(Folder,Persistent
select=restrictField
- #erster durchgang suche operatoren
+ params={}
for q in splitted:
+ name=re.sub("r'+'"," ",q.split("=")[0].lower())
+ if name=="_debug":
+ debug=True
+
+ try:
+ value=urllib.unquote(q.split("=",1)[1])
+ except:
+ value=""
- name=re.sub("r'+'"," ",q.split("=")[0].lower())
- if name=="_debug":
- debug=True
+ params[name]=value
+
+
+
+ #set table
+ primaryKey=""
+ if not tableExt:
+ table=params.get(iCT+'table')
+ primaryKey=self.getPrimaryKey(table)
+ logging.debug("table:"+table)
+ logging.debug("primkey:"+primaryKey);
- try:
- value=urllib.unquote(q.split("=",1)[1])
- except:
- value=""
+
+ #erster durchgang suche operatoren
+ for name,value in params.items():
+
+# name=re.sub("r'+'"," ",q.split("=")[0].lower())
+# if name=="_debug":
+# debug=True
+#
+# try:
+# value=urllib.unquote(q.split("=",1)[1])
+# except:
+# value=""
#print "Hi",name[0:3],q
if name[0:3]==iCT+"op":
op=value
@@ -1422,18 +1556,18 @@ class ZSQLExtendFolder(Folder,Persistent
sortfields[field]=value
#zweiter durchgang analysiere felder
- for q in qs.split(","):
+ for name,value in params.items():
-
- name=re.sub("r'+'"," ",q.split("=")[0].lower())
-
- try:
- value=urllib.unquote(q.split("=",1)[1])
-
- except:
- value=""
-
+#
+# name=re.sub("r'+'"," ",q.split("=")[0].lower())
+#
+# try:
+# value=urllib.unquote(q.split("=",1)[1])
+#
+# except:
+# value=""
+#
punktsplit=name.split(".") #sonderfall feld mit punkten(tabelle.suchFeld.ausgewaehltesFeld,feldinoriginal), d.h. suche in anderer tabelle:
#analysiere alle anderen faelle
@@ -1501,6 +1635,7 @@ class ZSQLExtendFolder(Folder,Persistent
op=opfields[name]
else:
op="ct"
+
namealt=name
name="LOWER("+punktsplit[1]+")"
value=value.lower()
@@ -1553,7 +1688,18 @@ class ZSQLExtendFolder(Folder,Persistent
else:
op="ct"
namealt=name
- name="LOWER("+name+")"
+ ##DW hack 29-8-12 lower(oid) funktioniert nicht, generell muss oid handling noch korrigiert werden.
+
+
+
+
+ if not name==primaryKey:
+ name="LOWER("+name+")" #immer lower key nicht definiert fuer keys.
+ else:
+ op="eq" # bei keys immer eq benutzen
+
+ logging.debug("NAME: %s"%name)
+
if op=="ct":
tmp=(name+" LIKE "+sql_quote("%"+value+"%"))
elif op=="gt":
@@ -1564,8 +1710,7 @@ class ZSQLExtendFolder(Folder,Persistent
tmp=(name+"="+sql_quote(value))
elif op=="bw":
tmp=(name+" LIKE "+sql_quote(value+"%"))
- elif op=="ew":
- tmp=(name+" LIKE "+sql_quote("%"+value))
+ #tmp=(name+" LIKE "+sql_quote("%"+value))
elif op=="all":
tmps=[]
for word in value.split(" "):
@@ -1587,7 +1732,7 @@ class ZSQLExtendFolder(Folder,Persistent
op="all"
- if (value!='') and ((not tableExt) or (namealt.split('.')[0]==tableExt)): #keine leeren werde und keine auschluss
+ if (value!='') and ((not tableExt) or (namealt.split('.')[0]==tableExt)): #keine leeren werte und keine auschluss
if searchFields.has_key(namealt):
searchFields[namealt]+=lopfields.get(name,'OR')+" "+tmp
searchFieldsOnly[namealt]+=lopfields.get(name,'OR')+" "+value
@@ -1941,12 +2086,30 @@ class ZSQLBibliography(Folder,ZSQLExtend
pt.content_type="text/html"
return pt()
- def changeZSQLBibliography(self,tableName,label,description,connection_id=None,REQUEST=None):
+ def changeZSQLBibliography(self,tableName,label,description,primaryKey="",primaryKeys="",connection_id=None,REQUEST=None):
"""change it"""
self.connection_id=connection_id
self.tableName=tableName
self.label=label
self.description=description
+ self.primaryKey=primaryKey
+
+ self.primaryKeys={}
+ if primaryKeys.lstrip().rstrip()!="":
+
+ for vals in primaryKeys.split(";"):
+ splitted=vals.split(":")
+ if len(splitted)<2:
+ if REQUEST is not None:
+ return """Wrong Syntax of keystring: %s
%s is not of the form TABLE:key."""%(primaryKeys,vals)
+
+ logging.debug(splitted)
+ self.primaryKeys[splitted[0]]=splitted[1]
+
+
+
+
+
if REQUEST is not None:
return self.manage_main(self, REQUEST)
@@ -1980,44 +2143,74 @@ class ZSQLBibliography(Folder,ZSQLExtend
except:
return None
+
+ def getMetaDataManager(self):
+ return self.metadata
+
def findTagsFromMapping(self,referenceType):
"""gib hash mit label -> generic zurueck"""
- self.referencetypes=self.ZopeFind(self.standardMD)
+ self.referencetypes=self.ZopeFind(self.getMetaDataManager(),search_sub=1)
bibdata={}
retdata={}
- fieldlist=self.standardMD.fieldList
+ #fieldlist=self.standardMD.fieldList
for referenceTypeF in self.referencetypes:
#print referenceType,referenceTypeF[1].title
- if referenceTypeF[1].title == referenceType:
+ if referenceTypeF[1].title.lower() == referenceType.lower():
bibdata[referenceTypeF[1].title]=referenceTypeF[1].fields
bibdata['data']=referenceTypeF[1]
- self.fields=bibdata[referenceType]
- for field in fieldlist:
+ self.fields=bibdata[referenceTypeF[1].title]
+
+
+
+ for field in referenceTypeF[1].getFieldList():
if referenceTypeF[1].getValue(field)[0]==None:
retdata[field]=field
else:
retdata[field]=referenceTypeF[1].getValue(field)[0]
- return retdata,fieldlist
+ return retdata,referenceTypeF[1].getFieldList()
def findLabelsFromMapping(self,referenceType):
- """gib hash mit label -> generic zurueck"""
- self.referencetypes=self.ZopeFind(self.standardMD)
- bibdata={}
- retdata={}
- fieldlist=self.standardMD.fieldList
+ """gib hash mit tagname -> lalbe zurueck"""
- for referenceTypeF in self.referencetypes:
- #print referenceType,referenceTypeF[1].title
- if referenceTypeF[1].title == referenceType:
- bibdata[referenceTypeF[1].title]=referenceTypeF[1].fields
- bibdata['data']=referenceTypeF[1]
- self.fields=bibdata[referenceType]
- for field in fieldlist:
- retdata[field]=referenceTypeF[1].getValue(field)[1]
+
+
+ mapping= self.getMetaDataManager().getBibMapping(referenceType)
+
+
+
+ fields=mapping.getFields()
+
+
+
+
+ fieldlist=fields.keys()
+
+ retdata={}
+ for field in fieldlist:
+ retdata[field]=fields[field]['label']
+
+# self.referencetypes=self.ZopeFind(self.getMetaDataManager(),search_sub=1)
+#
+# bibdata={}
+# retdata={}
+# #fieldlist=self.standardMD.fieldList
+# logging.debug("XX")
+# for referenceTypeF in self.referencetypes:
+# #print referenceType,referenceTypeF[1].title
+# logging.debug("%s=%s"%(referenceTypeF[1].title,referenceType))
+# if referenceTypeF[1].title.lower() == referenceType.lower():
+#
+# bibdata[referenceTypeF[1].title]=referenceTypeF[1].fields
+# bibdata['data']=referenceTypeF[1]
+# self.fields=bibdata[referenceTypeF[1].title]
+# for field in referenceTypeF[1].getFieldList():
+# retdata[field]=referenceTypeF[1].getValue(field)[1]
+
return retdata,fieldlist
+
def createRDFTag(self,tag,content,namespace="cdli"):
"""create RDF"""
@@ -2113,14 +2306,14 @@ class ZSQLBibliography(Folder,ZSQLExtend
ret="""
"""
- for found in self.ZSQLSimpleSearch("select oid from %s limit ALL"%self.tableName):
+ for found in self.ZSQLSimpleSearch("select % from %s limit ALL"%(self.getPrimaryKey,self.tableName)):
base_url = self.absolute_url()
if fix_host:
#print "replacing ", http_host, " by ", fix_host
base_url = string.replace(base_url, http_host, fix_host, 1)
- link=base_url+"/"+"record.html?oid=%i"%found.oid
- metalink=base_url+"/"+"getMetaDataXML?oid=%i"%found.oid
+ link=base_url+"/"+"record.html?oid=%i"%getattr(found,self.getPrimaryKey())
+ metalink=base_url+"/"+"getMetaDataXML?oid=%i"%getattr(found,self.getPrimaryKey())
ret+="""\n"""%(link,metalink)