--- ZSQLExtend/ZSQLExtend.py	2005/10/11 20:05:06	1.67
+++ ZSQLExtend/ZSQLExtend.py	2006/07/27 17:31:50	1.87
@@ -17,7 +17,7 @@ import Shared.DC.ZRDB.DA
 import zLOG
 import os.path
 import os
-
+from OFS.SimpleItem import SimpleItem
 
 def analyseIntSearch(word):
     #analyse integer searches
@@ -48,11 +48,86 @@ def sql_quote(v):
 def showSQLConnectionIDs(self):
     return SQLConnectionIDs(self)
 
+class ZSQLIndex(SimpleItem):
+    """index"""
+    meta_type="ZSQLIndex"
+    
+    def __init__(self,index,id,table=''):
+        self.index=[x for x in index]
+        self.id=id
+        self.table=table
+        
+    def setIndex(self,index):
+        self.index=[x for x in index]
+        
+    def getIndex(self):
+        return self.index
+
 class ZSQLExtendFolder(Folder,Persistent, Implicit):
     """Folder"""
     meta_type="ZSQLExtendFolder"
-
-
+    
+    def ZSQLQuote(self,str):
+        """quote str for sql"""
+        return sql_quote(str)
+    
+    
+    def importXMLFile(self,table,containerTagName,file,identify=None,RESPONSE=None):
+        #TODO: finish importXMLFile
+        '''
+        Import XML file into the table
+        @param table: name of the table the xml shall be imported into
+        @param containerTagName: XML-Tag which describes a dataset
+        @param file: xmlfile handle
+        @param identify: (optional) field res. tag which identifies a entry uniquely for updating purposes.
+        @param RESPONSE: (optional)
+        '''
+        from xml.dom.pulldom import parseString
+
+        doc=parseString(file.read())
+        while 1:
+            node=doc.getEvent()
+        
+            if node is None:
+                break;
+            else:
+                if node[1].nodeName==containerTagName:
+                    doc.expandNode(node[1])
+                #fn=node[1].getAttribute("xml:id")
+                #nf=file("xtf/"+fn+".xtf",'w')
+                #nf.write("""<texts xmlns="http://emegir.info/xtf" xmlns:lem="http://emegir.info/lemma" >"""+node[1].toxml()+"</texts>")
+                #print "wrote: %s"%fn
+                
+        
+    def generateIndex(self,field,index_name,table,RESPONSE=None):
+        """erzeuge index aus feld"""
+        index={}
+        founds=self.ZSQLSimpleSearch("""SELECT %s,oid FROM %s LIMIT 2000"""%(field,table))
+
+        for found in founds:
+            tmp=getattr(found,field,None)
+            if tmp:
+                strings=tmp.split(" ")
+                for string in strings:
+                    if index.get(string):
+                        index[string].append(found.oid)
+                    else:
+                        index[string]=[found.oid]
+                    RESPONSE.write(string+"\n")
+            
+        if not hasattr(self,index_name):
+            obj=ZSQLIndex(index,index_name,table)
+            self._setObject(index_name,obj)
+                    
+        self._getOb(index_name).setIndex(index)
+        
+    def getIndex(self,index_name):
+        """getIndex"""
+        founds=self.ZopeFind(self,obj_ids=[index_name])
+        
+        return founds[0][1].getIndex()
+    
+            
     def testneu(self):
         """test"""
         relStatement="""period like '%s%%'"""
@@ -64,6 +139,7 @@ class ZSQLExtendFolder(Folder,Persistent
     def URLquote(self,txt):
         """urlquote"""
         return urllib.quote(txt)
+    
     def searchRel(self,relStatement,statement,wherePart,classes):
         """suche relative haufigkeiten"""
         ret={}
@@ -131,6 +207,8 @@ class ZSQLExtendFolder(Folder,Persistent
     def formatAscii(self,str,url=None):
         """ersetze ascii umbrueche durch <br>"""
         #url=None
+        str=str.rstrip().lstrip()
+        
         if url and str:
             
             retStr=""
@@ -163,22 +241,22 @@ class ZSQLExtendFolder(Folder,Persistent
             return ""
 
     def actualPath(self,url=None):
-	"""path"""
+        """path"""
 
         if self.REQUEST['HTTP_X_FORWARDED_SERVER']=='':
             host=self.REQUEST['HTTP_HOST']
         else:
             host=self.REQUEST['HTTP_X_FORWARDED_SERVER']
-	if not url:
-		return "http://"+host+self.REQUEST['PATH_TRANSLATED']
-    	else:
-		temp=self.REQUEST[url].split("/")
-		temp[2]=host
-		return string.join(temp,"/")
+        if not url:
+                return "http://"+host+self.REQUEST['PATH_TRANSLATED']
+        else:
+                temp=self.REQUEST[url].split("/")
+                temp[2]=host
+                return string.join(temp,"/")
 
     def getRequest(self):
-	"""request"""
-	return self.REQUEST
+        """request"""
+        return self.REQUEST
 
     def lowerEnd(self,path):
         """oinly for demo"""
@@ -195,12 +273,12 @@ class ZSQLExtendFolder(Folder,Persistent
 
     def ZSQLsearchOptions(self,fieldname=""):
         """return HTML Fragment with search options"""
-	
+        
         ret="""<select name="-op_%s">
-	<option value="bw">begins with</option>          	<!-- begins with / beginnt mit, "Wort*" -->
-	<option value="ew">ends with</option>
-   	<option value="ct" selected>contains</option>          		<!-- contains / enthaellt, "Wort" -->
-    <option value="eq">equals</option>          		<!-- equals / ist, =Wort -->
+        <option value="bw">begins with</option>                 <!-- begins with / beginnt mit, "Wort*" -->
+        <option value="ew">ends with</option>
+        <option value="ct" selected>contains</option>                   <!-- contains / enthaellt, "Wort" -->
+    <option value="eq">equals</option>                          <!-- equals / ist, =Wort -->
 </select>"""%fieldname
         return ret
 
@@ -231,7 +309,7 @@ class ZSQLExtendFolder(Folder,Persistent
         return ret
 
 
-    def ZSQLOptionsFromCRList(self,fieldname,listField, multiple='',start=None,startValue=None,size=None):
+    def ZSQLOptionsFromCRList(self,fieldname,listField, multiple='',start=None,startValue=None,size=None,selected=None):
         """generate select oprions form a cr seperated list"""
         fields=listField.split("\n")
         if size:
@@ -242,14 +320,17 @@ class ZSQLExtendFolder(Folder,Persistent
             ret="""<select name="%s" %s >
             """%(fieldname,multiple)
         if start:
-	    if start==' ':
-		start=''
+            if start==' ':
+                start=''
                 if not startValue:
                     startValue=start
                     
             ret+="""<option value="%s" >%s</option>"""%(startValue,start)
         for field in fields:
-            ret+="""<option value="%s">%s</option>"""%(field.encode('utf-8'),field.encode('utf-8'))
+            if selected and (field in selected.split("\n")):
+                 ret+="""<option selected value="%s">%s</option>"""%(field.encode('utf-8'),field.encode('utf-8'))
+            else:
+                 ret+="""<option value="%s">%s</option>"""%(field.encode('utf-8'),field.encode('utf-8'))
         ret+="""</select>"""
         return ret
 
@@ -277,11 +358,12 @@ class ZSQLExtendFolder(Folder,Persistent
             ret="""<select name="%s" %s %s>
             """%(fieldname,multiple,additionalSelect)
         if start:
-	    if start==' ':
-		start=''
-                if not startValue:
+            if start==' ':
+                        start=''
+            
+            if not startValue:
                     startValue=start
-                    
+                            
             ret+="""<option value="%s" >%s</option>"""%(startValue,start)
         for result in results:
             field=getattr(result,fieldName)
@@ -303,6 +385,7 @@ class ZSQLExtendFolder(Folder,Persistent
         qs=[]
         if storename:
             """store"""
+            storename=storename
         else:
             storename="foundCount"
             
@@ -348,38 +431,57 @@ class ZSQLExtendFolder(Folder,Persistent
         #print "INLINE:",query
         return self.ZSQLSimpleSearch(query)
     
-    
+
+    def ZSQLResetConnection(self):
+        """reset the connectione"""
+        try:
+            self.getConnectionObj().manage_close_connection()
+        except:
+            zLOG.LOG("ZSQLResetConnection",zLOG.ERROR, '%s %s'%sys.exc_info()[:2])
+        try:
+            self.getConnectionObj().manage_open_connection()
+        except:
+            zLOG.LOG("ZSQLResetConnection",zLOG.ERROR, '%s %s'%sys.exc_info()[:2])
+
     def ZSQLSimpleSearch(self,query=None,max_rows=1000000):
         """simple search"""
-        
+
         if not query:
             query=self.query
         
+        
         if (hasattr(self,"_v_searchSQL") and (self._v_searchSQL == None)) or (not hasattr(self,"_v_searchSQL")):
-            self._v_searchSQL=Shared.DC.ZRDB.DA.DA("_v_searchSQL","_v_searchSQL",self.connection_id,"var","<dtml-var var>")
+            
+            self._v_searchSQL=Shared.DC.ZRDB.DA.DA("_v_searchSQL","_v_searchSQL",self.getConnectionObj().getId(),"var","<dtml-var var>")
             
             self._v_searchSQL.max_rows_=max_rows
             try:
-                
                 return self._v_searchSQL.__call__(var=query)
             except :
+
                 if sys.exc_info()[0]=="Database Error":
                     try:
-                        getattr(self,self.connection_id).manage_open_connection()
+                        self.getConnectionObj().manage_open_connection()
                     except:
                         zLOG.LOG("ZSQLSimpleSearch",zLOG.ERROR, '%s %s'%sys.exc_info()[:2])
         else:
             try:
+
                 self._v_searchSQL.max_rows_=max_rows
                 
                 return self._v_searchSQL.__call__(var=query)
             except :
+
                 if sys.exc_info()[0]=="Database Error":
                     try:
-                        getattr(self,self.connection_id).manage_open_connection()
+                        self.getConnectionObj().manage_open_connection()
                     except:
                         zLOG.LOG("ZSQLSimpleSearch",zLOG.ERROR, '%s %s'%sys.exc_info()[:2])
 
+    def getConnectionObj(self):
+        if hasattr(self,'connection_id'):
+            return getattr(self,self.connection_id)
+       
                     
     def ZSQLSimpleSearch2(self,query=None):
         """ returrn SQLSearch"""
@@ -431,7 +533,7 @@ class ZSQLExtendFolder(Folder,Persistent
             name=re.sub("r'+'"," ",q.split("=")[0].lower())
             value=q.split("=")[1]
             value=re.sub(r'\+'," ",value)
-            value=urllib.unquote(value)	
+            value=urllib.unquote(value) 
             if name=="-table":
                 table=urllib.unquote(value)
             elif name=="-format":
@@ -457,19 +559,21 @@ class ZSQLExtendFolder(Folder,Persistent
         else:
             return True
         
-    def ZSQLChange(self,format=None,RESPONSE=None,**argv):
+    def ZSQLChange(self,format=None,RESPONSE=None,USE_FORM=None,**argv):
         """change entries"""
         #qs=self.REQUEST['QUERY_STRING']
         # very bad hack
         
         qs_temp=[]
-    
-#        for a in self.REQUEST.form.keys():
-#            qs_temp.append(a+"="+urllib.quote(str(self.REQUEST.form[a])))
+        if USE_FORM or RESPONSE:
+            for a in self.REQUEST.form.keys():
+        
+                qs_temp.append(a+"="+urllib.quote(str(self.REQUEST.form[a])))
 
         
         
         for field in argv.keys():
+                
                    if field[0]=="_":
                        fieldTmp="-"+field[1:]
                    else:
@@ -477,16 +581,16 @@ class ZSQLExtendFolder(Folder,Persistent
                        
                    qs_temp.append("%s=%s"%(fieldTmp,argv[field]))
         
-        qs=string.join(qs_temp,",")
+        
         changeList=[]
         
-        for q in qs.split(","):
-            
+        for q in qs_temp:
+       
             name=urllib.unquote(re.sub("r'+'"," ",q.split("=")[0].lower()))
             value="=".join(q.split("=")[1:])
             value=re.sub(r'\+'," ",value)
             value=urllib.unquote(value)
-        
+         
             if name=="-table":
                     table=urllib.unquote(value)
             elif name=="-identify":
@@ -494,11 +598,15 @@ class ZSQLExtendFolder(Folder,Persistent
                 identify=identify.split("=")[0]+"="+sql_quote(identify.split("=")[1])
             elif name=="-format":
                 format=urllib.unquote(value)
-            elif (not (name[0]=="-" or name[0]=="_")) and (not len(value)==0):
+            #elif (not (name[0]=="-" or name[0]=="_")) and (not len(value)==0):
+            elif (not (name[0]=="-" or name[0]=="_")):
+
                 changeList.append("\""+name+"\"="+sql_quote(urllib.unquote(value)))
                 
         changeString=string.join(changeList,",")
+
         queryString="UPDATE %s SET %s WHERE %s"%(table,changeString,identify)
+        zLOG.LOG("ZSQLExtend",zLOG.INFO,"CHANGE: "+queryString)
 
         self.ZSQLSimpleSearch(queryString)
         
@@ -509,6 +617,19 @@ class ZSQLExtendFolder(Folder,Persistent
             return True
    
 
+    def ZSQLFindIndexed(self,qs="",select="oid,*",storename=None,indexedFields=['data_line'],restrictField='id_text',**argv):
+        """find2"""
+        
+        for index in self.ZopeFind(self,obj_ids=indexedFields):
+            txt=argv.get(index[0],None)
+            if txt:
+                oids=index[1].getIndex()[txt]
+                
+        search1= self.ZSQLFind(qs=qs,select=select,storename=storename,tableExt=tableList[1],restrictField=restrictField,NoQuery='yes',NoLimit='yes',**argv)
+     
+        search2 = self.ZSQLFind(tableExt=tableList[0],qs=qs,select=select,storename=storename,restrictConnect=(tableList[0]+"."+restrictField,search1),**argv)
+        return search2
+    
     def ZSQLFind2(self,qs="",select="oid,*",storename=None,tableList=['cdli_translit','cdli_cat'],restrictField='id_text',**argv):
         """find2"""
 
@@ -520,9 +641,10 @@ class ZSQLExtendFolder(Folder,Persistent
       
     
     def ZSQLFind(self,qs="",select="oid,*",storename=None,tableExt=None,NoQuery=None,NoLimit=None,restrictField=None,restrictConnect=None,filter=None,**argv):
-        """Find"""
+        """search in database"""
 
         def delEmpty(list):
+            """"loesche leere elemente aus der liste"""
             ret=[]
             for x in list:
                 splitted=x.split("=")
@@ -530,15 +652,11 @@ class ZSQLExtendFolder(Folder,Persistent
                     ret.append(x)
             return ret
 
-
-
-        #self.REQUEST.SESSION['come_from_search']="no" # zuruecksetzen
         if qs=="":
-           
+           #kein querystring ubergeben 
            if self.REQUEST['QUERY_STRING']:
                qs=self.REQUEST['QUERY_STRING']
 
-               
                qs=string.join(qs.split("&"),",")
 
                for field in argv.keys():
@@ -570,7 +688,7 @@ class ZSQLExtendFolder(Folder,Persistent
             
         
         
-        qs=re.sub("\\+"," ",qs)# Austauschen da Leerzeichen bei http-get durch + ersetzt wird, generell sollte alles auf post umgeschrieben werden. vom search formular.
+        qs=re.sub("\\+"," ",qs)#TODO: Austauschen da Leerzeichen bei http-get durch + ersetzt wird, generell sollte alles auf post umgeschrieben werden. vom search formular.
 
         qs=string.join(delEmpty(qs.split(",")),",")
 
@@ -698,6 +816,8 @@ class ZSQLExtendFolder(Folder,Persistent
         """analysieren den QueryString"""
         
         
+        #setzte generische werte
+        
         lop="AND" # standardsuche mit and
         max="ALL" #standard alle auswaehlen
         maxstr=""
@@ -713,21 +833,22 @@ class ZSQLExtendFolder(Folder,Persistent
         limit=0
         searchFields={}
         searchFieldsOnly={}
+        queryTemplate=[]
+        
         if not select:
             select="oid,*"
-        #print "Q",nostore,qs
-        #check for op 
-
 
+        #check for op 
+        splitted=qs.split(",")
 
-        
-	splitted=qs.split(",")
         if tableExt:
             table=tableExt
 
         if restrictField:
                     select=restrictField
-            
+        
+        
+        #erster durchgang suche operatoren    
         for q in splitted:
                 
                 name=re.sub("r'+'"," ",q.split("=")[0].lower())
@@ -758,25 +879,21 @@ class ZSQLExtendFolder(Folder,Persistent
                         sortAllFields=value
                     else:
                         sortfields[field]=value
-                    
-                    #print "HI",op,field
-	#print opfieldsa
-        #now analyse the querystring
-       
+                        
+        #zweiter durchgang analysiere felder
         for q in qs.split(","):
        
             
-            #try:
-
+          
             name=re.sub("r'+'"," ",q.split("=")[0].lower())
             try:
                 value=urllib.unquote(q.split("=",1)[1])
             except:
                 value=""
             
-            #value=sql_quote(value)
-            
-
+            punktsplit=name.split(".")   #sonderfall feld mit punkten(tabelle.suchFeld.ausgewaehltesFeld,feldinoriginal), d.h. suche in anderer tabelle:                      
+         
+            #analysiere alle anderen faelle
             if name==iCT+"lop":
                 lop=value
             elif name==iCT+"table":
@@ -817,10 +934,56 @@ class ZSQLExtendFolder(Folder,Persistent
                 op=value
 
 
+            
+            elif (not name[0]==iCT) and len(punktsplit)==4:
+                if opfields.has_key(name):
+                    op=opfields[name]
+                else:
+                    op="ct"
+                namealt=name
+                name="LOWER("+punktsplit[1]+")" 
 
+                if op=="ct":
+                    tmp=(name+" LIKE "+sql_quote("%"+value+"%"))
+                elif op=="gt":
+                    tmp=(name+">"+sql_quote(value))
+                elif op=="lt":
+                    tmp=(name+"<"+sql_quote(value))
+                elif op=="eq":
+                    tmp=(name+"="+sql_quote(value))
+                elif op=="bw":
+                    tmp=(name+" LIKE "+sql_quote(value+"%"))
+                elif op=="ew":
+                    tmp=(name+" LIKE "+sql_quote("%"+value))
+                elif op=="all":
+                    tmps=[]
+                    for word in value.split(" "):
+                        tmps.append(name+" LIKE "+sql_quote("%"+word+"%"))
+                        
+                    tmp=string.join(tmps,' AND ')
+
+                elif op=="numerical":
+                    term=analyseIntSearch(value)
+                    tmp=(name+" "+term)
+                elif op=="grep":
+                    tmp=(name+" ~* "+sql_quote(value))
+                elif op=="one":
+                    tmps=[]
+                    for word in value.split(" "):
+                        tmps.append(name+" LIKE "+sql_quote("%"+word+"%"))
+                        
+                    tmp=string.join(tmps,' OR ')
+
+                op="all"
+
+
+                searchTmp="""%s in (select %s from %s where %s)"""%(punktsplit[3],punktsplit[2],punktsplit[0],tmp)
+
+                queryTemplate.append(searchTmp)
+                
             elif (not name[0]==iCT) and (not len(value)==0):
 
-		#print "OP",op,name
+                #print "OP",op,name
                 value=value.lower()
                 tmp=""
                 if opfields.has_key(name):
@@ -872,6 +1035,7 @@ class ZSQLExtendFolder(Folder,Persistent
 
 
         whereList=["("+searchFields[x]+")" for x in searchFields.keys()]
+        whereList+=queryTemplate
         
         if len(whereList)>0:
             if filter:
@@ -895,13 +1059,14 @@ class ZSQLExtendFolder(Folder,Persistent
         #print "QE",table
         #print (select,table,where,sort,maxstr,skip)
         query="SELECT %s FROM %s %s %s %s %s"%(select,table,where,sort,maxstr,skip)
-        
+
         if not nostore=="yes":
             
             self.REQUEST.SESSION['qs']=opfields
         #print "IAMHERE again:", query
 
         if storename and (not NoQuery):
+
             query2="SELECT count(*) FROM %s %s"%(table,where)
             
             #print "QUERYSTRING:",self.REQUEST.SESSION[storename]['queryString2']
@@ -959,37 +1124,34 @@ class ZSQLExtendFolder(Folder,Persistent
         else:
             return value
         
-    def ZSQLQuery(self,query):
+    def ZSQLQuery(self,query,debug=None):
         """query"""
-
+        if debug:
+            zLOG.LOG("ZSQLQuery", zLOG.INFO, query)
+            
         return self.ZSQLSimpleSearch(query)
 
     
     def ZSQLSearch(self):
         """To be done"""
-        rq=self.REQUEST['QUERY_STRING']
-
-        querys=rq.split("&")
-        
-        
+           
         formatfile=self.REQUEST['URL1'] #generisch redirect zur gleichen url
+      
+        #zerlege querystring in key value paare
+        #TODO: check if this is really necessary, use argv**
         
-        for querytemp in querys:
+        rq=self.REQUEST['QUERY_STRING']
+        querys=rq.split("&")
+        for querytemp in querys: #zerg
             query=querytemp.split("=")
             
-            
-
-            
             try:
                 if query[0].lower()=="-format":
                     formatfile=query[1]
             except:
-                """nothing"""
+                pass
 
-                
-        #print formatfile
-        
-        
+        #sichern
         self.REQUEST.SESSION['query']=string.join(self.REQUEST['QUERY_STRING'].split("&"),",")
         self.REQUEST.SESSION['come_from_search']="yes"
         
@@ -1064,7 +1226,7 @@ class ZSQLExtendFolder(Folder,Persistent
             newquery.append(query)
                 
         newquerystring=string.join(newquery,"&")
-	return "<a href='%s'>%s</a>"%(self.REQUEST['URL']+"?"+newquerystring,html)
+        return "<a href='%s'>%s</a>"%(self.REQUEST['URL']+"?"+newquerystring,html)
 
             
         
@@ -1308,16 +1470,16 @@ class ZSQLBibliography(Folder,ZSQLExtend
         retdata={}
         fieldlist=self.standardMD.fieldList
         
-	for referenceTypeF in self.referencetypes:
-		#print referenceType,referenceTypeF[1].title
-		if referenceTypeF[1].title == referenceType: 
-			bibdata[referenceTypeF[1].title]=referenceTypeF[1].fields
-			bibdata['data']=referenceTypeF[1]
-			self.fields=bibdata[referenceType]
+        for referenceTypeF in self.referencetypes:
+                #print referenceType,referenceTypeF[1].title
+                if referenceTypeF[1].title == referenceType: 
+                        bibdata[referenceTypeF[1].title]=referenceTypeF[1].fields
+                        bibdata['data']=referenceTypeF[1]
+                        self.fields=bibdata[referenceType]
                         for field in fieldlist:
-		            if referenceTypeF[1].getValue(field)[0]==None:
-				retdata[field]=field
-			    else:
+                            if referenceTypeF[1].getValue(field)[0]==None:
+                                retdata[field]=field
+                            else:
                                 retdata[field]=referenceTypeF[1].getValue(field)[0]
 
         return retdata,fieldlist
@@ -1329,12 +1491,12 @@ class ZSQLBibliography(Folder,ZSQLExtend
         retdata={}
         fieldlist=self.standardMD.fieldList
         
-	for referenceTypeF in self.referencetypes:
-		#print referenceType,referenceTypeF[1].title
-		if referenceTypeF[1].title == referenceType: 
-			bibdata[referenceTypeF[1].title]=referenceTypeF[1].fields
-			bibdata['data']=referenceTypeF[1]
-			self.fields=bibdata[referenceType]
+        for referenceTypeF in self.referencetypes:
+                #print referenceType,referenceTypeF[1].title
+                if referenceTypeF[1].title == referenceType: 
+                        bibdata[referenceTypeF[1].title]=referenceTypeF[1].fields
+                        bibdata['data']=referenceTypeF[1]
+                        self.fields=bibdata[referenceType]
                         for field in fieldlist:
                             retdata[field]=referenceTypeF[1].getValue(field)[1]
 
@@ -1429,11 +1591,11 @@ class ZSQLBibliography(Folder,ZSQLExtend
         host_port = self.REQUEST['SERVER_PORT']
         fix_host = None
         if http_host and http_host.rfind(host_port) == -1:
-            print "HTTP_HOST needs fixing!"
+            #print "HTTP_HOST needs fixing!"
             fix_host = http_host + ":" + host_port
         
         ret="""<?xml version="1.0" ?>
-	             <index>"""
+                     <index>"""
         for found in self.ZSQLSimpleSearch("select oid from %s limit ALL"%self.tableName):
             base_url = self.absolute_url()
             if fix_host: