--- cdli/cdli_files.py 2006/12/22 16:43:42 1.54 +++ cdli/cdli_files.py 2007/02/02 14:31:49 1.62 @@ -22,6 +22,79 @@ from ZPublisher.HTTPResponse import HTTP from ZPublisher.BaseRequest import RequestContainer import threading from BTrees.OOBTree import OOBTree +import logging +import transaction +import copy + +def unique(s): + """Return a list of the elements in s, but without duplicates. + + For example, unique([1,2,3,1,2,3]) is some permutation of [1,2,3], + unique("abcabc") some permutation of ["a", "b", "c"], and + unique(([1, 2], [2, 3], [1, 2])) some permutation of + [[2, 3], [1, 2]]. + + For best speed, all sequence elements should be hashable. Then + unique() will usually work in linear time. + + If not possible, the sequence elements should enjoy a total + ordering, and if list(s).sort() doesn't raise TypeError it's + assumed that they do enjoy a total ordering. Then unique() will + usually work in O(N*log2(N)) time. + + If that's not possible either, the sequence elements must support + equality-testing. Then unique() will usually work in quadratic + time. + (from the python cookbook) + """ + + n = len(s) + if n == 0: + return [] + + # Try using a dict first, as that's the fastest and will usually + # work. If it doesn't work, it will usually fail quickly, so it + # usually doesn't cost much to *try* it. It requires that all the + # sequence elements be hashable, and support equality comparison. + u = {} + try: + for x in s: + u[x] = 1 + except TypeError: + del u # move on to the next method + else: + return u.keys() + + # We can't hash all the elements. Second fastest is to sort, + # which brings the equal elements together; then duplicates are + # easy to weed out in a single pass. + # NOTE: Python's list.sort() was designed to be efficient in the + # presence of many duplicate elements. This isn't true of all + # sort functions in all languages or libraries, so this approach + # is more effective in Python than it may be elsewhere. + try: + t = list(s) + t.sort() + except TypeError: + del t # move on to the next method + else: + assert n > 0 + last = t[0] + lasti = i = 1 + while i < n: + if t[i] != last: + t[lasti] = last = t[i] + lasti += 1 + i += 1 + return t[:lasti] + + # Brute force is all that's left. + u = [] + for x in s: + if x not in u: + u.append(x) + return u + class BasketContent(SimpleItem): """classe fuer den Inhalt eines Baskets""" @@ -63,6 +136,7 @@ class uploadATFfinallyThread(Thread): self.SESSION=SESSION self.username=username self.serverport=serverport + def __call__(self): """call of the thread (equals run)""" @@ -96,11 +170,11 @@ class uploadATFfinallyThread(Thread): #add the files self.uploadATFfinallyThread(ctx,self.procedure,comment=self.comment,basketname=self.basketname,unlock=self.unlock,SESSION=self.SESSION,username=self.username) #commit the transactions - get_transaction().commit() + transaction.get().commit() conn.close() #set flag for end of this method self.end=True - print "ended" + logging.info("ended") return True def __del__(self): @@ -122,8 +196,8 @@ class uploadATFfinallyThread(Thread): #shall I only upload the changed files? if procedure=="uploadchanged": - - uploadFns=SESSION.get('changed',[])+SESSION.get('newPs',[]) + changed=[x[0] for x in SESSION.get('changed',[])] + uploadFns=changed+SESSION.get('newPs',[]) #or all elif procedure=="uploadAll": @@ -139,14 +213,21 @@ class uploadATFfinallyThread(Thread): uploadFns=[] #do first the changed files + i=0 for fn in uploadFns: + i+=1 founds=ctx2.CDLICatalog.search({'title':fn}) if len(founds)>0: SESSION['author']=str(username) self.result+="

Changing : %s"%fn founds[0].getObject().manage_addCDLIFileObject('',comment,SESSION['author'],file=os.path.join(SESSION['tmpdir'],fn),from_tmp=True) - - + if i==200: + i=0 + transaction.get().commit() + logging.info("changing: do commit") + transaction.get().commit() + logging.info("changing: last commit") + #now add the new files newPs=SESSION['newPs'] if len(newPs)>0: @@ -191,7 +272,15 @@ class uploadATFfinallyThread(Thread): return True + +class tmpStore(SimpleItem): + """simple item""" + meta_type="cdli_upload" + def __init__(self,id): + """init tmp""" + self.id=id + class uploadATFThread(Thread): """class for checking the files befor uploading""" @@ -204,13 +293,14 @@ class uploadATFThread(Thread): Thread.__init__(self) - def set(self,upload,basketId,username,serverport="8080"): + def set(self,upload,basketId,username,idTmp,serverport="8080"): """set start values for the thread""" self.result="" self.upload=upload self.basketId=basketId self.username=username self.serverport=serverport + self.idTmp=idTmp def __call__(self): """call method """ @@ -229,7 +319,7 @@ class uploadATFThread(Thread): return app.__of__(RequestContainer(REQUEST = req)) def run(self): - + idTmp=self.idTmp self.result="" #find context within ZODB from Zope import DB @@ -237,14 +327,15 @@ class uploadATFThread(Thread): root = conn.root() app = root['Application'] ctx = self.getContext(app,serverport=self.serverport) - self.uploadATFThread(ctx,self.upload,self.basketId) + logging.info("run intern") + ctx.temp_folder._setObject(idTmp,tmpStore(idTmp)) + logging.info("call thread intern") + self.uploadATFThread(ctx,self.upload,idTmp,self.basketId) #ctx.cdliRoot.cdli_main.tmpStore2[self.getName()[0:]]=self.returnValue - - get_transaction().commit() - - while self.continueVar: - pass + + + transaction.get().commit() conn.close() @@ -253,10 +344,13 @@ class uploadATFThread(Thread): """method for accessing result""" return self.result - def uploadATFThread(self,ctx,upload,basketId=0): + def uploadATFThread(self,ctx,upload,idTmp,basketId=0): """upload an atf file""" #TODO: add comments #TODO: finish uploadATF + + stObj=getattr(ctx.temp_folder,idTmp) + logging.info("start, upload thread") self.result="

I got your file, start now to split it into single atf-files!

" #make sure that id is a string and not an integer @@ -295,7 +389,7 @@ class uploadATFThread(Thread): basketNameFromId=getattr(ctx2.basketContainer,basketId).title basketLen=getattr(ctx2.basketContainer,basketId).getLastVersion().numberOfItems() - + logging.info("got the file, upload thread") self.result+="""

I got the files

< p>I am computing the differences to the exisiting files

""" @@ -340,19 +434,24 @@ class uploadATFThread(Thread): #ready, set the returnValues self.result+="

Done

" - self.returnValue={} - self.returnValue['changed']=changed - self.returnValue['errors']=errors - self.returnValue['lockerrors']=lockerrors - self.returnValue['newPs']=newPs - self.returnValue['tmpdir']=dir - self.returnValue['basketLen']=basketLen - self.returnValue['numberOfFiles']=numberOfFiles - self.returnValue['basketNameFromId']=basketNameFromId - self.returnValue['basketNameFromFile']=basketNameFromFile - self.returnValue['basketId']=basketId - self.returnValue['dir']=dir + stObj.returnValue={} + stObj.returnValue['errors']=errors + + stObj.returnValue['newPs']=newPs + stObj.returnValue['tmpdir']=dir + stObj.returnValue['basketLen']=basketLen + stObj.returnValue['numberOfFiles']=numberOfFiles + stObj.returnValue['basketNameFromId']=basketNameFromId + stObj.returnValue['basketNameFromFile']=basketNameFromFile + stObj.returnValue['basketId']=basketId + stObj.returnValue['dir']=dir + #stObj.returnValue['changed']=copy.copy(changed) + stObj.returnValue['changed']=[(x[0].getId(),x[1][0]) for x in changed] + #stObj.returnValue['lockerrors']=[x[0].getId() for x in lockerrors] + print lockerrors + stObj.returnValue['lockerrors']=[x for x in lockerrors] + self.returnValue=True #ctx2.cdli_main.setTemp('v_uploadATF_returnValue',True) @@ -519,7 +618,7 @@ class BasketObject_old(Folder): """unlock all files of the testuser for debuggin""" for object in self.contents: - if str(object.lockedBy)=="test": + if str(object.lockedBy)=="dahl": object.lockedBy="" def downloadObjectsAsOneFile(self,lock=None,procedure=None,REQUEST=None): @@ -559,8 +658,8 @@ class BasketObject_old(Folder): self.temp_folder.downloadCounter+=1 self._p_changed=1 - get_transaction().commit() - + + transaction.get().commit() for object in self.contents: @@ -578,7 +677,8 @@ class BasketObject_old(Folder): self.REQUEST.RESPONSE.write(ret) self.temp_folder.downloadCounter-=1 self._p_changed=1 - get_transaction().commit() + transaction.get().commit() + def manage_addBasket_oldObjectForm(self): @@ -1121,17 +1221,17 @@ class CDLIBasketVersion(Implicit,Persist self.temp_folder.downloadCounterBaskets+=1 self._p_changed=1 - get_transaction().commit() + transaction.get().commit() if lock: if str(self.REQUEST['AUTHENTICATED_USER'])=='Anonymous User': self.temp_folder.downloadCounterBaskets-=1 self._p_changed=1 - get_transaction().commit() + transaction.get().commit() self.temp_folder.downloadCounterBaskets-=1 self._p_changed=1 - get_transaction().commit() + transaction.get().commit() return "please login first" #check if a locked object exist in the basket. @@ -1151,7 +1251,7 @@ class CDLIBasketVersion(Implicit,Persist self.temp_folder.downloadCounterBaskets-=1 self._p_changed=1 - get_transaction().commit() + transaction.get().commit() return pt() @@ -1181,7 +1281,7 @@ class CDLIBasketVersion(Implicit,Persist self.temp_folder.downloadCounterBaskets-=1 self._p_changed=1 - get_transaction().commit() + transaction.get().commit() self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.atf" """%basket_name) self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream") @@ -1604,10 +1704,14 @@ def splitatf(fh,dir=None,ext=None): if dir: filename=os.path.join(dir,filename) nf=file(filename,"w") + logging.info("open %s"%filename) if nf: nf.write(line.replace("\n","")+"\n") - nf.close() + try: + nf.close() + except: + pass fh.close() return ret,len(os.listdir(dir)) @@ -1760,7 +1864,7 @@ class CDLIFileFolder(extVersionedFileFol self.temp_folder.downloadCounter+=1 self._p_changed=1 - get_transaction().commit() + transaction.get().commit() list=[(x.getId,x) for x in catalog()] list.sort(sortF) @@ -1780,7 +1884,7 @@ class CDLIFileFolder(extVersionedFileFol RESPONSE.write(obj.getLastVersion().getData()[0:]) self.temp_folder.downloadCounter-=1 self._p_changed=1 - get_transaction().commit() + transaction.get().commit() #os.close(tf) #RESPONSE.redirect(self.absolute_url()+"/downloadFile?fn="%tfilename) return True @@ -1862,24 +1966,64 @@ class CDLIRoot(Folder): meta_type="CDLIRoot" downloadCounterBaskets=0# counts the current basket downloads if counter > 10 no downloads are possible + def findWordRegExp(self,searchTerm): + """find all words in index which match regexp in SearchTerm""" + ret=[] + for x in self.lineIndex.iterkeys(): + if re.match(searchTerm,x): + ret.append(x) + return ret + + def searchRegExpInLineIndexDocs(self,searchTerm): + """search in inLineIndex with regexp""" + if not searchTerm: + return [] + ret=[] + words=self.findWordRegExp(searchTerm) # suche nach allen Treffern + logging.info("wd:%s"%words) + for word in words: + ret+=self.searchInLineIndexDocs(word) + + return unique(ret) + def showInLineIndex(self): """get the index for debug purposes""" print "show" for x in self.lineIndex.iterkeys(): - print "word:",x - for y in self.lineIndex[x].iterkeys(): - print "doc",y,self.lineIndex[x][y] + logging.info("word:%s"%repr(x)) + #for y in self.lineIndex[x].iterkeys(): + # print "doc",repr(y),repr(self.lineIndex[x][y]) return self.lineIndex - def searchInLineIndexDocs(self,word): + def searchInLineIndexDocs(self,word,uniq=True,regExp=False): """search occurences""" - return list(self.lineIndex.get(word.upper()).keys()) - def getLinesFromIndex(self,word,doc): + if regExp: + return self.searchRegExpInLineIndexDocs(word) + + try: + lst=list(self.lineIndex.get(word).keys()) + except: + lst=[] + if uniq: + return unique(lst) + else: + return lst + + def getLinesFromIndex(self,word,doc,regExp=False): """get lines""" - return self.lineIndex[word][doc] - + if not regExp: + return self.lineIndex.get(word)[doc] + else: # wenn regexp, suche welches word + for w in self.findWordRegExp(word): + if self.lineIndex.get(w): # ein word in im dex gefunden + try: + dc=self.lineIndex.get(word)[doc] + return dc # und ein document dann gib es zurueck + except: + pass #andernfalls weiter + def cleanInLineIndex(self): """delete InlineIndex""" for x in list(self.lineIndex.keys()): @@ -1913,7 +2057,7 @@ class CDLIRoot(Folder): self.lineIndex=li - get_transaction().commit() + transaction.get().commit() def showFile(self,fileId): @@ -1923,7 +2067,70 @@ class CDLIRoot(Folder): return "" return f[0].getObject().getLastVersionFormattedData() + + def showLineFromFile(self,fileId,lineNum,word): + """get line lineNum fromFileId""" + file=self.showFile(fileId) + #str="^%s\.[^%s\.]*%s[^\n]*\n"%(lineNum,lineNum,word) + #str="^%s\..*?%s[^\n]*\n"%(lineNum,word) + + #print str + #m=re.search(str,file,flags=re.M|re.DOTALL) + #if m: + # return m.group() + #else: + # return "" + #ret=lineNum+"." + #splitted=file.split(lineNum+".") + #if len(splitted)>1: + #for part in splitted[1:]: + #if part.find(word)>-1: + # for x in part.split("\n"): + #ret+=x + #if x.find(word)>-1: + #break + #break; + #return ret + + def showWordInFile(self,fileId,word,lineList=None): + """get lines with word fromFileId""" + + file=self.showFile(fileId) + + ret=[] + for line in file.split("\n"): + if line.find(word)>-1: + if lineList: #liste of moeglichen Zeilennummern + num=line.split(".")[0] #Zeilenummer ist alles vor dem . in der Zeile + + if num in lineList: + + ret.append(line) + else: # nimm alles ohne line check + ret.append(line) + return ret + + def tagWordInFile(self,fileId,word,lineList=None): + """get lines with word fromFileId""" + + file=self.showFile(fileId) + tagStr="""%s""" + ret=[] + for line in file.split("\n"): + if line.find(word)>-1: + if lineList: #liste of moeglichen Zeilennummern + num=line.split(".")[0] #Zeilenummer ist alles vor dem . in der Zeile + + if num in lineList: + + ret.append(line.replace(word,tagStr%word)) + else: # nimm alles ohne line check + ret.append(line.replace(word,tagStr%word)) + else: + ret.append(line) + return "
\n".join(ret) + def URLquote(self,str): """quote url""" return urllib.quote(str) @@ -1932,6 +2139,14 @@ class CDLIRoot(Folder): """unquote url""" return urllib.unquote(str) + def URLquote_plus(self,str): + """quote url""" + return urllib.quote_plus(str) + + def URLunquote_plus(self,str): + """unquote url""" + return urllib.unquote_plus(str) + def forceunlock(self): "break all locks" @@ -1942,6 +2157,18 @@ class CDLIRoot(Folder): if un and un !="": ret.append((f[0],un)) + return ret + + def forceDahl(self): + "break all locks" + ret=[] + for f in self.ZopeFind(self,obj_metatypes="CDLI file",search_sub=1): + if str(f[1].lockedBy)=="dahl": + un=f[1].forceunlock() + + if un and un !="": + ret.append((f[0],un)) + return ret def getChangesByAuthor(self,author,n=100): @@ -2009,7 +2236,14 @@ class CDLIRoot(Folder): def uploadATF(self,repeat=None,upload=None,basketId=0,RESPONSE=None): """standard ausgabe""" #self._v_uploadATF.returnValue=None + from random import randint + if (not self.REQUEST.SESSION.get('idTmp',None)): + idTmp=str(randint(0,1000000000)) + self.REQUEST.SESSION['idTmp']=idTmp + else: + idTmp=self.REQUEST.SESSION.get('idTmp',None) + threadName=repeat if not threadName or threadName=="": tmpVar=False @@ -2022,9 +2256,10 @@ class CDLIRoot(Folder): self._v_uploadATF[threadName]=thread #self._xmltrans.start() #thread=Thread(target=self._v_uploadATF) - - self._v_uploadATF[threadName].set(upload,basketId,self.REQUEST['AUTHENTICATED_USER'],serverport=self.REQUEST['SERVER_PORT']) + logging.info("set thread. extern") + self._v_uploadATF[threadName].set(upload,basketId,self.REQUEST['AUTHENTICATED_USER'],idTmp,serverport=self.REQUEST['SERVER_PORT']) #thread.start() + logging.info("start thread. extern") self._v_uploadATF[threadName].start() @@ -2046,7 +2281,7 @@ class CDLIRoot(Folder): if threadName == thread.getName(): self._v_uploadATF[threadName]=thread - if not self._v_uploadATF[threadName].returnValue: + if self._v_uploadATF.get(threadName,None) and (not self._v_uploadATF[threadName].returnValue): wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template']) @@ -2069,15 +2304,12 @@ class CDLIRoot(Folder): # # # #tmp=self.cdli_main.tmpStore2[threadName] - tmp=self._v_uploadATF[threadName].returnValue - self._v_uploadATF[threadName].continueVar=False + tmp=getattr(self.temp_folder,idTmp).returnValue + - self.REQUEST.SESSION['changed']=[x[0].getId() for x in tmp['changed']] - self.REQUEST.SESSION['lockerrors']=[x[0].getId() for x in tmp['lockerrors']] - self.REQUEST.SESSION['errors']=tmp['errors'] - self.REQUEST.SESSION['newPs']=tmp['newPs'] - self.REQUEST.SESSION['tmpdir']=tmp['dir'] + + #del(self.cdli_main.tmpStore2[threadName]) @@ -2109,8 +2341,9 @@ class CDLIRoot(Folder): self._v_uploadATF[threadName]=thread - - self._v_uploadATF[threadName].set(procedure,comment=comment,basketname=basketname,unlock=unlock,SESSION=self.REQUEST.SESSION,username=self.REQUEST['AUTHENTICATED_USER'],serverport=self.REQUEST['SERVER_PORT']) + idTmp=self.REQUEST.SESSION['idTmp'] + stObj=getattr(self.temp_folder,idTmp) + self._v_uploadATF[threadName].set(procedure,comment=comment,basketname=basketname,unlock=unlock,SESSION=stObj.returnValue,username=self.REQUEST['AUTHENTICATED_USER'],serverport=self.REQUEST['SERVER_PORT']) self._v_uploadATF[threadName].start() @@ -2143,7 +2376,7 @@ class CDLIRoot(Folder): pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self) return pt(txt='/uploadATFfinally',threadName=threadName) else: - + self.REQUEST.SESSION['idTmp']=None if RESPONSE is not None: RESPONSE.redirect(self.absolute_url()) @@ -2164,7 +2397,7 @@ class CDLIRoot(Folder): if not obj: manage_addCDLIFileFolder(root,folder,folder) fobj=getattr(root,folder) - #get_transaction().commit() + #transaction.get().commit() else: fobj=obj[0][1] @@ -2192,9 +2425,9 @@ class CDLIRoot(Folder): if count > 1000: print "committing" - get_transaction().commit() + transaction.get().commit() count=0 - get_transaction().commit() + transaction.get().commit() return "ok" @@ -2214,7 +2447,10 @@ def manage_addCDLIRoot(self, id, title=' ob=CDLIRoot() ob.id=str(id) ob.title=title - self._setObject(id, ob) + try: + self._setObject(id, ob) + except: + pass ob=self._getOb(id) checkPermission=getSecurityManager().checkPermission