--- cdli/cdli_files.py 2008/09/25 12:37:54 1.83 +++ cdli/cdli_files.py 2008/10/06 12:57:12 1.91 @@ -31,7 +31,13 @@ from BTrees.IOBTree import IOBTree import cdliSplitter from sets import Set import md5 - +from DownloadBasket import DownloadBasketFinallyThread + +def makelist(mySet): + x = list(mySet) + x.sort() + return x + def unicodify(s): """decode str (utf-8 or latin-1 representation) into unicode object""" if not s: @@ -224,7 +230,7 @@ class BasketContent(SimpleItem): def getContent(self): """get content""" - + return self.contentList def setContent(self,content): @@ -395,6 +401,7 @@ class uploadATFfinallyThread(Thread): basketId=ob.getId() basket=getattr(ctx2.basketContainer,str(basketId)) ids=os.listdir(SESSION['tmpdir']) + logging.debug("should add:"+repr(ids)) basket.addObjects(ids,deleteOld=True,username=str(username)) logging.debug("uploadatffinallythread uploadfinally done") @@ -1024,21 +1031,24 @@ class CDLIBasket(Folder,CatalogAware): def isActual(self,obj): """teste ob im basket die aktuelle version ist""" - actualNo=obj[1].getLastVersion().getVersionNumber() - storedNo=obj[0].getVersionNumber() - - founds=self.CDLICatalog.search({'title':obj[0].getId()}) - if len(founds)>0: - actualNo=founds[0].getObject().getLastVersion().getVersionNumber() + try: + logging.debug("isActual:"+repr(obj)) + actualNo=obj[1].getLastVersion().getVersionNumber() + storedNo=obj[0].getVersionNumber() - if len(founds)>0 and founds[0].getObject().aq_parent.getId()==".trash": + + actualNo=self.getFileObjectLastVersion(obj.getId()).getVersionNumber() + + #if len(founds)>0 and founds[0].getObject().aq_parent.getId()==".trash": + # return False, -1 + + if actualNo==storedNo: + return True , 0 + else: + return False, actualNo + except: return False, -1 - - if actualNo==storedNo: - return True , 0 - else: - return False, actualNo - + def history(self): """history""" @@ -1210,13 +1220,13 @@ class CDLIBasket(Folder,CatalogAware): # newContent.append((found.getObject().getLastVersion(),found.getObject())) # added+=1 - hash = md5.new(repr(self.makelist(ids))).digest() # erzeuge hash als identification + hash = md5.new(repr(makelist(ids))).hexdigest() # erzeuge hash als identification #logging.debug("JJJJJJJ:"+repr(self.makelist(ids))) logging.debug("JJJJJJJ:"+repr(hash)) - if hasattr(self.cdliRoot,'_v_tmpStore') and self.cdliRoot._v_tmpStore.has_key(hash): + if hasattr(self.cdliRoot,'v_tmpStore') and self.cdliRoot.v_tmpStore.has_key(hash): logging.debug("from store!") - newContent=Set(map(swap,self.cdliRoot._v_tmpStore[hash])) + newContent=Set(map(swap,self.cdliRoot.v_tmpStore[hash])) else: logging.debug("not from store!") @@ -1352,7 +1362,7 @@ class CDLIBasketVersion(Implicit,Persist security.declareProtected('manage','downloadObjectsAsOneFile') def downloadObjectsAsOneFile(self,lock=None,procedure=None,REQUEST=None,check="yes",current="no"): """download all selected files in one file""" - + logging.debug("doasof") if self.temp_folder.downloadCounterBaskets > 10000: return """I am sorry, currently the server has to many requests for downloads, please come back later!""" @@ -1366,25 +1376,19 @@ class CDLIBasketVersion(Implicit,Persist return self.downloadObjectsAsOneFileFinally(lock=lock,procedure=procedure,REQUEST=REQUEST,current="no") - def downloadObjectsAsOneFileFinally(self,lock=None,procedure=None,REQUEST=None,current="no"): + def downloadObjectsAsOneFileFinally(self,lock=None,procedure=None,REQUEST=None,current="no",repeat=None): """print do the download""" - + + logging.debug("HIHHHH") ret="" lockedObjects={} - self.temp_folder.downloadCounterBaskets+=1 - self._p_changed=1 - transaction.get().commit() - + + logging.debug("lock:"+repr(lock)) if lock: - + logging.debug("------lock:"+repr(lock)) if str(self.REQUEST['AUTHENTICATED_USER'])=='Anonymous User': - self.temp_folder.downloadCounterBaskets-=1 - self._p_changed=1 - transaction.get().commit() - self.temp_folder.downloadCounterBaskets-=1 - self._p_changed=1 - transaction.get().commit() + return "please login first" #check if a locked object exist in the basket. @@ -1402,53 +1406,123 @@ class CDLIBasketVersion(Implicit,Persist self.REQUEST.SESSION['lockedObjects']=lockedObjects pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','lockedObjects.zpt')).__of__(self) - self.temp_folder.downloadCounterBaskets-=1 - self._p_changed=1 - transaction.get().commit() - + return pt() elif not procedure: #keine fails gesperrt dann alle donwloaden procedure="downloadAll" + + + threadName=repeat + if not threadName or threadName=="": + thread=DownloadBasketFinallyThread() + threadName=thread.getName()[0:] + + if (not hasattr(self,'_v_downloadBasket')): + self._v_downloadBasket={} + + + self._v_downloadBasket[threadName]=thread + logging.debug("dwonloadfinally:"+repr(self)) + basketID=self.aq_parent.aq_parent.getId() + versionNumber=self.aq_parent.getId() + + if lock: + logging.debug("-----start locking") + for object in self.content.getContent(): + if object[1].lockedBy =='': + object[1].lockedBy=self.REQUEST['AUTHENTICATED_USER'] + logging.debug("-----finished locking") + + #obj.lockedBy=user + self._v_downloadBasket[threadName].set(lock,procedure,self.REQUEST['AUTHENTICATED_USER'],current,basketID,versionNumber) + + self._v_downloadBasket[threadName].start() + + + + wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template']) + if wait_template: + return wait_template[0][1]() + pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadBasketWait.zpt')).__of__(self) - for object in self.content.getContent(): - logging.error("ret:"+repr(object[0])) - logging.error(" -"+repr(procedure)) - logging.error(" -"+repr(object[1].lockedBy)) - - if (procedure=="downloadAll") or (object[1].lockedBy=='') or (object[1].lockedBy==self.REQUEST['AUTHENTICATED_USER']): - logging.error("ret1") - if current=="no": #version as they are in the basket - logging.error("ret2") - ret+=str(object[0].getData())+"\n" - elif current=="yes": - logging.error("ret3") - #search current object - founds=self.CDLICatalog.search({'title':object[1].getId().split(".")[0]}) - if len(founds)>0: - ret+=str(founds[0].getObject().getLastVersion().getData())+"\n" - - if lock and object[1].lockedBy=='': - object[1].lockedBy=self.REQUEST['AUTHENTICATED_USER'] - basket_name=self.aq_parent.title+"_V"+self.getId() + return pt(txt=self.absolute_url()+'/downloadObjectsAsOneFileFinally',threadName=threadName, + counter=self._v_downloadBasket[threadName].getCounter(), + number=self._v_downloadBasket[threadName].getNumberOfFiles()) + #_v_xmltrans.run() - #write basketname to header of atf file - ret="#basket: %s\n"%basket_name+ret + else: + #recover thread, if lost + if not hasattr(self,'_v_downloadBasket'): + self._v_downloadBasket={} + if not self._v_downloadBasket.get(threadName,None): + for thread in threading.enumerate(): + if threadName == thread.getName(): + self._v_downloadBasket[threadName]=thread + + if self._v_downloadBasket.get(threadName,None) and (self._v_downloadBasket[threadName] is not None) and (not self._v_downloadBasket[threadName].end) : - self.temp_folder.downloadCounterBaskets-=1 - self._p_changed=1 - transaction.get().commit() + wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template']) + if wait_template: + return wait_template[0][1]() + + pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadBasketWait.zpt')).__of__(self) + return pt(txt=self.absolute_url()+'/downloadObjectsAsOneFileFinally',threadName=threadName, + counter=self._v_downloadBasket[threadName].getCounter(), + number=self._v_downloadBasket[threadName].getNumberOfFiles()) + else: + + + logging.debug("FINISHED") + if not self._v_downloadBasket.get(threadName,None): + for thread in threading.enumerate(): + if threadName == thread.getName(): + self._v_downloadBasket[threadName]=thread + + #files = self._v_downloadBasket[threadName].result + files=self.basketContainer.resultHash[threadName] + lockedFiles=self.basketContainer.resultLockedHash[threadName] + logging.error(files) + # fh=file("/var/tmp/test") + #ret =fh.read() + + if (not isinstance(self.aq_parent,CDLIBasket)): + basket_name=self.aq_parent.aq_parent.title+"_V"+self.getId() + else: + basket_name=self.aq_parent.title+"_V"+self.getId() - self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.atf" """%basket_name) - self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream") - length=len(ret) - self.REQUEST.RESPONSE.setHeader("Content-Length",length) - self.REQUEST.RESPONSE.write(ret) - return True + + #write basketname to header of atf file + + + self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.atf" """%basket_name) + self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream") + #length=len(ret) + #self.REQUEST.RESPONSE.setHeader("Content-Length",length) + + ret="#basket: %s\n"%basket_name + self.REQUEST.RESPONSE.write(ret) + + for fileName in files: + try: + self.REQUEST.RESPONSE.write(file(fileName).read()) + except: + logging.error("downloadasonefile: cannot read %s"%fileName) + + + self.REQUEST.RESPONSE.write("\n# locked files\n") + for fileName in lockedFiles: + self.REQUEST.RESPONSE.write("# %s\n"%fileName) + + self.REQUEST.RESPONSE.write("# locked files end\n") + + del self.basketContainer.resultHash[threadName] + del self.basketContainer.resultLockedHash[threadName] + def numberOfItems(self): """return anzahl der elemente im basket""" return self.content.numberOfItems() @@ -1551,8 +1625,12 @@ class CDLIFileObject(CatalogAware,extVer """Return cataloguable key for ourselves.""" return str(self) + def setAuthor(self, author): + """change the author""" + self.author = author + def makeThisVersionCurrent_html(self): - """form for making this version current""" + """form for mthis version current""" pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','makeThisVersionCurrent.zpt')).__of__(self) return pt() @@ -1645,6 +1723,7 @@ def manage_addCDLIFileObject(self,id,vC= logging.debug("reindex2: %s in %s"%(repr(fob), repr(fob.default_catalog))) fob.index_object() + self.CDLIRoot.updateOrAddToFileBTree(ob) if REQUEST is not None: REQUEST['RESPONSE'].redirect(self.absolute_url()+'/manage_main') @@ -1758,6 +1837,9 @@ class CDLIFile(extVersionedFile,CatalogA except: pass + #self.cdliRoot.updateOrAddToFileBTree(self)# now update the object in the cache + + if RESPONSE: if ob.getSize()==0: self.REQUEST.SESSION['objID']=ob.getId() @@ -1967,10 +2049,13 @@ class CDLIFileFolder(extVersionedFileFol fileTree.add(found.getId) logging.debug("ADDD:"+found.getId) - def findObjectsFromList(self,enterList=None,display=False,start=None,upload=None,list=None,basketName=None,numberOfObjects=None,RESPONSE=None,REQUEST=None): + def findObjectsFromList(self,enterList=None,display=False,start=None,upload=None,list=None,basketName=None,numberOfObjects=None,RESPONSE=None,REQUEST=None,returnHash=False,hash=None): """findObjectsFromList (, TAB oder LINE separated)""" logging.debug("start: findObjectsFromList") + #logging.debug("start: findObjectsFromList"+repr(list)) + + if upload: # list from file upload txt=upload.read() @@ -2001,6 +2086,14 @@ class CDLIFileFolder(extVersionedFileFol pt=getattr(self,'filelist.html') return pt(basketName=basketName,numberOfObjects=numberOfObjects) + if hash is not None and hasattr(self.cdliRoot,'v_tmpStore') and self.cdliRoot.v_tmpStore.has_key(hash): + + logging.debug("asking for storage2") + result =self.cdliRoot.v_tmpStore[hash] + if result: + logging.debug("give result from storage2") + return hash,self.cdliRoot.v_tmpStore[hash] + if list is not None: # got already a list logging.debug(" ----List version") @@ -2024,42 +2117,55 @@ class CDLIFileFolder(extVersionedFileFol ids = fileTree & self.v_file_ids + #self.REQUEST.SESSION['fileIds']=ids#store fieldIds in session for further usage + l=makelist(fileTree)[0:] + logging.debug("l-list:"+repr(l)) + self.REQUEST.SESSION['fileIds']=l#store fieldIds in session for further usage + self.REQUEST.SESSION['searchList']=l + #self.REQUEST.SESSION['searchList']=['P000001.atf'] + - - hash = md5.new(repr(self.makelist(ids))).digest() # erzeuge hash als identification - #TODO: do I need garbage collection for _v_tmpStore ? - if hasattr(self.cdliRoot,'_v_tmpStore') and self.cdliRoot._v_tmpStore.has_key(hash): - return self.cdliRoot._v_tmpStore[hash] + hash = md5.new(repr(makelist(fileTree))).hexdigest() # erzeuge hash als identification + self.REQUEST.SESSION['hash']=hash + #TODO: do I need garbage collection for v_tmpStore ? + + #logging.debug("Hash:"+repr(hash)) +# +# if hasattr(self.cdliRoot,'v_tmpStore') and self.cdliRoot.v_tmpStore.has_key(hash): +# logging.debug("asking for storage") +# res=self.cdliRoot.v_tmpStore[hash] +# if res: +# if returnHash == True: +# return hash,res +# return res #TODO: get rid of one of these.. #ids=[x.getObject().getId() for x in ret] ret=[(self.getFileObject(x),self.getFileObjectLastVersion(x)) for x in ids] - self.REQUEST.SESSION['fileIds']=ids#store fieldIds in session for further usage - self.REQUEST.SESSION['searchList']=self.REQUEST.SESSION['fileIds'] - + + #self.REQUEST.SESSION['fileIds']=ids#store fieldIds in session for further usage + #self.REQUEST.SESSION['searchList']=self.REQUEST.SESSION['fileIds'] + if display: pt=getattr(self,'filelist.html') return pt(search=ids) else: #self.REQUEST.SESSION['hash'] = ret # store in session - if not hasattr(self,'_v_tmpStore'): - self.cdliRoot._v_tmpStore={} + if not hasattr(self,'v_tmpStore'): + self.cdliRoot.v_tmpStore={} #logging.debug("HHHHHHNEU:"+repr(self.makelist(ids))) - logging.debug("HHHHHHNEU:"+repr(hash)) - self.cdliRoot._v_tmpStore[hash] = ret # store in session + #logging.debug("HHHHHHNEU:"+repr(hash)) + self.cdliRoot.v_tmpStore[hash] = ret # store in session + if returnHash == True: + return hash,ret return ret if start: RESPONSE.redirect("filelist.html?start:int="+str(start)) - - def makelist(self,mySet): - x = list(mySet) - x.sort() - return x - + security.declareProtected('Manage','createAllFilesAsSingleFile') def createAllFilesAsSingleFile(self,RESPONSE=None): """download all files""" @@ -2187,16 +2293,72 @@ class CDLIRoot(Folder): 'graphemes':cdliSplitter.graphemeSplitter()} + def viewATF(self,id,RESPONSE): + """view an Object""" + ob = self.CDLICatalog({'title':id}) + logging.debug(ob[0].getObject().getLastVersion().absolute_url()+"/view") + if len(ob)>0: + RESPONSE.redirect(ob[0].getObject().getLastVersion().absolute_url()+"/view") + return "not found" + + def history(self,id,RESPONSE): + """view an Object""" + ob = self.CDLICatalog({'title':id}) + if len(ob)>0: + RESPONSE.redirect(ob[0].absolute_url+"/history") + return "not found" + + + def downloadLocked(self,id,RESPONSE): + """view an Object""" + ob = self.CDLICatalog({'title':id}) + if len(ob)>0: + RESPONSE.redirect(ob[0].absolute_url+"/downloadLocked") + return "not found" + + def download(self,id,RESPONSE): + """view an Object""" + ob = self.CDLICatalog({'title':id}) + if len(ob)>0: + RESPONSE.redirect(ob[0].getLastVersion().absolute_url()) + return "not found" + def addCDLIFileObjectForm(self,id,RESPONSE): + """view an Object""" + ob = self.CDLICatalog({'title':id}) + if len(ob)>0: + RESPONSE.redirect(ob[0].absolute_url+"/addCDLIFileObjectForm") + return "not found" + + def addVersionedFileObjectForm(self,id,RESPONSE): + """view an Object""" + ob = self.CDLICatalog({'title':id}) + if len(ob)>0: + RESPONSE.redirect(ob[0].absolute_url+"/addVersionedFileObjectForm") + return "not found" + + def unlock(self,id,RESPONSE): + """view an Object""" + ob = self.CDLICatalog({'title':id}) + if len(ob)>0: + RESPONSE.redirect(ob[0].absolute_url+"/unlock") + return "not found" + def getFileObject(self,fileId): + """get an object""" x=self.v_files.get(fileId) #logging.debug(x) return x def getFileObjectLastVersion(self,fileId): + """get an object""" x=self.v_files_lastVersion.get(fileId) #logging.debug(x) return x + def showFileIds(self): + """showIds""" + return self.v_file_ids + def generateFileBTree(self): """erzeuge einen Btree aus allen Files""" self.v_files = OOBTree() @@ -2210,7 +2372,25 @@ class CDLIRoot(Folder): self.v_file_ids.add(x.getId) logging.debug("add:"+x.getId+"XXX"+repr(x.getObject())) - return "done" + return True + + + def updateOrAddToFileBTree(self,obj): + """update a BTree""" + self.v_files.update({obj.getId():obj}) + self.v_files_lastVersion.update({obj.getId():obj.getLastVersion()}) + + self.v_file_ids.add(obj.getId()) + logging.debug("update:"+obj.getId()+"XXX"+repr(obj)) + + def deleteFromBTree(self,objId): + """delete an obj""" + self.v_files.pop(objId) + self.v_files_lastVersion.pop(objId) + self.v_file_ids.remove(objId) + + + def deleteFiles(self,ids): """delete files""" for id in ids: @@ -2468,6 +2648,11 @@ class CDLIRoot(Folder): return """ 2;url=%s?repeat=%s """%(self.absolute_url()+txt,threadName) + def refreshTxtBasket(self,txt="",threadName=None): + """txt fuer refresh""" + + return """ 2;url=%s?repeat=%s """%(txt,threadName) + def getResult(self,threadName=None): """result of thread""" @@ -2644,7 +2829,18 @@ class CDLIRoot(Folder): pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self) return pt(txt='/uploadATFfinally',threadName=threadName) else: + + + idTmp=self.REQUEST.SESSION['idTmp'] + stObj=getattr(self.temp_folder,idTmp) self.REQUEST.SESSION['idTmp']=None + + #update changed + logging.debug("dir:"+repr(stObj.returnValue['changed'])) + for x in stObj.returnValue['changed']: + ob=self.CDLICatalog.search({'title':x[0]}) + + self.cdliRoot.updateOrAddToFileBTree(ob[0].getObject()) if RESPONSE is not None: RESPONSE.redirect(self.absolute_url()) @@ -2659,12 +2855,15 @@ class CDLIRoot(Folder): for f in files: folder=f[0:3] f2=f[0:5] + + #check if main folder PXX already exists obj=self.ZopeFind(root,obj_ids=[folder]) logging.debug("importFiles: folder=%s f2=%s obj=%s"%(folder,f2,obj)) if ext: ext.result="

adding: %s

"%f+ext.result - if not obj: + + if not obj: # if not create it manage_addCDLIFileFolder(root,folder,folder) fobj=getattr(root,folder) #transaction.get().commit() @@ -2672,22 +2871,28 @@ class CDLIRoot(Folder): else: fobj=obj[0][1] + #check IF PYYYYY already exist obj2=fobj.ZopeFind(fobj,obj_ids=[f2]) logging.debug("importFiles: fobj=%s obj2=%s"%(fobj,obj2)) - if not obj2: + if not obj2:# if not create it manage_addCDLIFileFolder(fobj,f2,f2) fobj2=getattr(fobj,f2) else: fobj2=obj2[0][1] + # not add the file file2=os.path.join(folderName,f) id=f logging.debug("importFiles: addCDLIFile fobj2=%s, f=%s file2=%s"%(fobj2,repr(f),repr(file2))) fobj2.addFile(vC='',file=file(file2),author=author,newName=f) count+=1 - + + #now add the file to the storage + ob = getattr(fobj2,f) + self.cdliRoot.updateOrAddToFileBTree(ob) + if count%100==0: logging.debug("importfiles: committing") transaction.get().commit()