--- cdli/cdli_files.py 2008/10/15 07:48:05 1.99 +++ cdli/cdli_files.py 2008/11/05 19:53:32 1.103 @@ -1,10 +1,8 @@ """CDLI extensions of the filearchive""" from Products.versionedFile.extVersionedFile import * from Products.ZCatalog.CatalogPathAwareness import CatalogAware -from tempfile import mkstemp,mkdtemp import os.path import os -from types import * import urlparse import urllib import cgi @@ -21,7 +19,6 @@ from ZPublisher.HTTPRequest import HTTPR from ZPublisher.HTTPResponse import HTTPResponse from ZPublisher.BaseRequest import RequestContainer import threading -from BTrees.OOBTree import OOBTree, OOTreeSet import logging import transaction import copy @@ -32,7 +29,9 @@ import cdliSplitter from sets import Set import md5 from DownloadBasket import DownloadBasketFinallyThread - +from types import * +import pickle + def makelist(mySet): x = list(mySet) x.sort() @@ -224,29 +223,53 @@ def unique(s): class BasketContent(SimpleItem): """classe fuer den Inhalt eines Baskets""" + def getFileAndVersionFromId(self,pnum,versionNr): + + obj=self.cdliRoot.getFileObject(pnum) + logging.debug("obj : %s"%obj) + version=obj.getVersionNr(versionNr) + logging.debug("-------vs: %s"%version.getFileName()) + return version,obj + def __init__(self,content=[]): """content""" - self.contentList=content[0:] + + self.setContent(content[0:]) def getContent(self,filtered=True): + return self.contentList + + def getContentOld(self,filtered=True): """get content""" + logging.debug("content object: content List %s"%self.contentList) ret=[] - if filtered: - for x in self.contentList: - if not((x[0] is None) or (x[1] is None)): - ret.append(x) - return ret - - else: - return self.contentList + + return [self.getFileAndVersionFromId(x[0],x[1]) for x in self.contentList] +# +# if filtered: +# for x in self.contentList: +# if not((x[0] is None) or (x[1] is None)): +# ret.append(x) +# logging.debug("content object: content List -done filtered") +# return ret +# +# else: +# logging.debug("content object: content List -done not filtered") +# return self.contentList def allContent(self): """get all content""" return self.getContent(filtered=False) def setContent(self,content): - self.contentList=content[0:] - + contentList=[] + for x in content: + if not((x[0] is None) or (x[1] is None)): + + contentList.append((x[1].getId(),x[0].getVersionNumber())) + logging.debug("cl: %s"%contentList) + self.contentList=contentList[0:] + def numberOfItems(self): """number""" @@ -618,6 +641,10 @@ class CDLIBasketContainer(OrderedFolder) security=ClassSecurityInfo() meta_type="CDLIBasketContainer" + def getResultHash(self): + """get the result hash for debug purposes""" + return self.resultHash.keys() + def getPNumbersOfBasket(self,basketName): """get all pnumbers of a basket as a list, returns an empty list if basket not found @param basketName: name of the basket @@ -627,7 +654,7 @@ class CDLIBasketContainer(OrderedFolder) if not basketId: return [] - ob=getattr(self,basketId).getContent() + ob=getattr(self,basketId).getContent() #get the content of a basket ret=[x[0].split(".")[0] for x in ob] @@ -646,15 +673,18 @@ class CDLIBasketContainer(OrderedFolder) return "" ob=getattr(self,basketId).getLastVersion() - for object in ob.getContent(): + for pnum,versionNr in ob.getContent(): + obj=self.cdliRoot.getFileObject(pnum) + # logging.debug("obj : %s"%obj) + # version=obj.getVersionNr(versionNr) + if current=="no": #version as they are in the basket - ret+=str(object[0].getData())+"\n" + cur= obj.getVersionNr(versionNr) + ret+=str(cur.getData())+"\n" elif current=="yes": #search current object #logging.debug("current: %s"%object[1].getId().split(".")[0]) - founds=self.CDLICatalog.search({'title':object[1].getId().split(".")[0]}) - if len(founds)>0: - ret+=str(founds[0].getObject().getLastVersion().getData())+"\n" + obj.getData() return ret security.declareProtected('manage','upDateBaskets') @@ -1060,7 +1090,25 @@ class CDLIBasket(Folder,CatalogAware): return [x[1].getId() for x in self.getLastVersion().getContent()] - def isActual(self,obj): + def isActual(self,obj,nummer): + """teste ob im basket die aktuelle version ist, obj kann entweder ein CDLIFile sein oder eine + eine pnummer, die auf ein CDLIFile verweist""" + try: + #logging.debug("isActual:"+repr(obj)) + if isinstance(obj, CDLIFile): + actualNo=obj.getLastVersion().getVersionNumber() + else: + actualNo=self.cdliRoot.getFileObjectLastVersion(obj).getVersionNumber() + + if actualNo==nummer: + return True , 0 + else: + return False, actualNo + except: + logging.error( """is actual: %s (%s %s)"""%(repr(obj),sys.exc_info()[0],sys.exc_info()[1])) + logging.error(""" PARAMS: %s %s"""%(obj,nummer)) + return False, -1 + def isActualOld(self,obj): """teste ob im basket die aktuelle version ist""" try: #logging.debug("isActual:"+repr(obj)) @@ -1228,6 +1276,7 @@ class CDLIBasket(Folder,CatalogAware): def swap(x): return (x[1],x[0]) + logging.info("add to basket (%s)"%(repr(ids))) logging.info("add to basket (%s)"%(self.getId())) lastVersion=self.getLastVersion() @@ -1256,16 +1305,17 @@ class CDLIBasket(Folder,CatalogAware): hash = md5.new(repr(makelist(ids))).hexdigest() # erzeuge hash als identification #logging.debug("JJJJJJJ:"+repr(self.makelist(ids))) - - if hasattr(self.cdliRoot,'v_tmpStore') and self.cdliRoot.v_tmpStore.has_key("hash"): #TODO: muss eigentlich self.cdliRoot.v_tmpStore.has_key(hash): heissen (ohne "), erstmal so gesetzt damit der hash hier nie benutzt wird - logging.debug("from store!") - newContent=Set(map(swap,self.cdliRoot.v_tmpStore[hash])) - + retrieved = self.CDLICache.retrieve(hash) + if retrieved: + newContent=Set(map(swap,retrieved)) else: - logging.debug("not from store!") - newContent=Set([(self.getFileObjectLastVersion(x),self.getFileObject(x)) for x in ids]) - + newContent=Set([(self.getFileObjectLastVersion(x),self.getFileObject(x)) for x in ids]) + + + #remove all Elements which are not stored + if (None,None) in newContent: + newContent.remove((None,None)) content=Set(oldContent).union(newContent) added = len(content)-len(oldContent) if not username: @@ -1285,11 +1335,11 @@ class CDLIBasket(Folder,CatalogAware): ret=[] lv=self.getLastVersion() - for obj in lv.content.getContent(): + #for obj in lv.content.getContent(): #logging.info("XXXXXXXXXX %s"%repr(obj)) - ret.append((obj[1].getId(),obj[0].versionNumber)) + # ret.append((obj[1].getId(),obj[0].versionNumber)) - return ret + return lv def getContentIds(self): """print basket content""" @@ -1373,7 +1423,7 @@ class CDLIBasketVersion(Implicit,Persist objs=self.getContent() for obj in objs: - if not self.isActual(obj)[0]: + if not self.isActual(obj[0],obj[1])[0]: return True return False @@ -1400,9 +1450,16 @@ class CDLIBasketVersion(Implicit,Persist return """I am sorry, currently the server has to many requests for downloads, please come back later!""" - if (check=="yes") and self.containsNonActualFiles(): - pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadObjectAsOneFile_check.zpt')).__of__(self) - + #if (check=="yes") and self.containsNonActualFiles(): + # pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadObjectAsOneFile_check.zpt')).__of__(self) + # + # return pt(lock=lock) + + # neue Version aus Performancegruenden, es wird nicht mehr getestet, ob es nicht aktuelle Objekte gibt + # sondern lediglich gefragt. + if (check=="yes"): + pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadObjectAsOneFile_ask.zpt')).__of__(self) + return pt(lock=lock) else: @@ -1427,9 +1484,9 @@ class CDLIBasketVersion(Implicit,Persist #check if a locked object exist in the basket. lockedObjects={} for object in self.content.getContent(): - - if (not str(object[1].lockedBy)=="") and (not (str(object[1].lockedBy)==str(self.REQUEST['AUTHENTICATED_USER']))): - lockedObjects[object[1].title]=repr(object[1].lockedBy) + obj=self.getFileObject(object[0]) + if (not str(obj.lockedBy)=="") and (not (str(obj.lockedBy)==str(self.REQUEST['AUTHENTICATED_USER']))): + lockedObjects[obj.title]=repr(obj.lockedBy) keys=lockedObjects.keys() @@ -1480,8 +1537,9 @@ class CDLIBasketVersion(Implicit,Persist if lock: logging.debug("-----start locking") for object in self.content.getContent(): - if object[1].lockedBy =='': - object[1].lockedBy=self.REQUEST['AUTHENTICATED_USER'] + obj=self.ctx.getFileObject(object[0]) + if obj.lockedBy =='': + obj.lockedBy=self.REQUEST['AUTHENTICATED_USER'] logging.debug("-----finished locking") #obj.lockedBy=user @@ -1531,8 +1589,35 @@ class CDLIBasketVersion(Implicit,Persist self._v_downloadBasket[threadName]=thread #files = self._v_downloadBasket[threadName].result - files=self.basketContainer.resultHash[threadName] - lockedFiles=self.basketContainer.resultLockedHash[threadName] + # lade die files und die locked files, bei grossen Baskets muss u.U. gewartet werden + # bis das Commit aus dem Thread alles geschrieben hat, in dem Falle existiert resultHash[threadName] + # noch nicht. + o1 = file("/tmp/"+threadName,'r') + files=pickle.load(o1) + os.remove("/tmp/"+threadName) + o2 = file("/tmp/"+threadName+'_lockedFiles','r') + + lockedFiles=pickle.load(o2) + os.remove("/tmp/"+threadName+'_lockedFiles') +# try: +# files=self.basketContainer.resultHash[threadName] +# except: +# i=0 +# while (not self.basketContainer.resultHash.has_key(threadName)) and (i<100): +# logging.debug(" downloadFinally: I am waiting for thread %s to write the resultHashfile: %s"%(threadName,i)) +# time.sleep(5) +# i+=1 +# files=self.basketContainer.resultHash[threadName] +# +# try: +# lockedFiles=self.basketContainer.resultLockedHash[threadName] +# except: +# i=0 +# while (not self.basketContainer.resultLockedHash.has_key(threadName)) and (i<100): +# logging.debug(" downloadFinally: I am waiting for thread %s to write the LockedHashfile: %s"%(threadName,i)) +# time.sleep(5) +# i+=1 +# lockedFiles=self.basketContainer.resultLockedHash[threadName] # fh=file("/var/tmp/test") #ret =fh.read() @@ -1556,6 +1641,7 @@ class CDLIBasketVersion(Implicit,Persist self.REQUEST.RESPONSE.write(ret) for fileName in files: + logging.debug("download: %s"%fileName) try: self.REQUEST.RESPONSE.write(file(fileName).read()) except: @@ -1589,7 +1675,12 @@ class CDLIBasketVersion(Implicit,Persist def getContent(self): """get Basket Content""" - return self.content.getContent() + logging.debug("retrieving content A") + cnt = self.content + logging.debug("retrieving content: obj %s"%cnt) + tmp = self.content.getContent() + logging.debug("got content") + return tmp def __init__(self,id,user,comment="",basketContent=[]): @@ -1612,26 +1703,26 @@ class CDLIBasketVersion(Implicit,Persist security.declareProtected('manage','index_html') def index_html(self): """view the basket""" - + logging.debug("start index_html - Basket version") if self.REQUEST.get('change',False): ob=self.aq_parent.updateObjects(self.REQUEST['change']) self.REQUEST.RESPONSE.redirect(ob.absolute_url())#go to new basket, because changing generates a new basket - + logging.debug("start index_html - Basket version:template") pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketVersionMain.zpt')).__of__(self) return pt() def getObjUrl(self,result): """getUrl of the version of the object""" - objId=result[1].getTitle() - founds=self.CDLICatalog.search({'title':objId}) + + founds=self.CDLICatalog.search({'title':result}) if len(founds)>0: return founds[0].getObject().getLastVersion().absolute_url() else: #assume version number - splitted=objId.split("_") + splitted=result.split("_") founds=self.CDLICatalog.search({'title':splitted[1]}) - return founds[0].getObject().getLastVersion().absolute_url()+'/'+objId + return founds[0].getObject().getLastVersion().absolute_url()+'/'+result def manage_addCDLIBasketVersion(self,user,comment="",basketContent=[],RESPONSE=None): """add a version""" @@ -2134,14 +2225,12 @@ class CDLIFileFolder(extVersionedFileFol pt=getattr(self,'filelist.html') return pt(basketName=basketName,numberOfObjects=numberOfObjects) - if hash is not None and hasattr(self.cdliRoot,'v_tmpStore') and self.cdliRoot.v_tmpStore.has_key(hash): - - logging.debug("asking for storage2") - result =self.cdliRoot.v_tmpStore[hash] - if result: - logging.debug("give result from storage2") - return hash,self.cdliRoot.v_tmpStore[hash] - + + result =self.CDLICache.retrieve(hash) + if result: + logging.debug("give result from storage2") + return hash,result + if list is not None: # got already a list logging.debug(" ----List version") @@ -2200,11 +2289,11 @@ class CDLIFileFolder(extVersionedFileFol return pt(search=ids) else: #self.REQUEST.SESSION['hash'] = ret # store in session - if not hasattr(self,'v_tmpStore'): - self.cdliRoot.v_tmpStore={} + #logging.debug("HHHHHHNEU:"+repr(self.makelist(ids))) #logging.debug("HHHHHHNEU:"+repr(hash)) - self.cdliRoot.v_tmpStore[hash] = ret # store in session + self.CDLICache.store(hash,ret) + if returnHash == True: return hash,ret return ret @@ -2341,6 +2430,13 @@ class CDLIRoot(Folder): 'graphemes':cdliSplitter.graphemeSplitter()} + def unicodify(self,txt): + return unicodify(txt) + def invalidateOldCacheVersion(self): + """loescht die alte Version des Cache""" + del self.v_tmpStore + return "done" + def viewATF(self,id,RESPONSE): """view an Object""" ob = self.CDLICatalog({'title':id})