cdli/cdli_basket.py - view

File: [Repository] / cdli / cdli_basket.py
Revision 1.7: download - view: text, annotated - select for diffs - revision graph
Thu Jul 21 21:58:50 2011 UTC (12 years, 9 months ago) by dwinter
Branches: MAIN
CVS tags: HEAD

errors in basket fixed

"""CDLI extensions of the filearchive""" from Products.versionedFile.extVersionedFile import * from Products.ZCatalog.CatalogPathAwareness import CatalogAware import os.path import os import urlparse import urllib import cgi from OFS.OrderedFolder import OrderedFolder from OFS.SimpleItem import SimpleItem import time from OFS.Folder import manage_addFolder import re from AccessControl import ClassSecurityInfo from Acquisition import Implicit from Globals import Persistent from threading import Thread from ZPublisher.HTTPRequest import HTTPRequest from ZPublisher.HTTPResponse import HTTPResponse from ZPublisher.BaseRequest import RequestContainer import threading import logging import transaction import copy import codecs import sys from BTrees.IOBTree import IOBTree import cdliSplitter from sets import Set import md5 from DownloadBasket import DownloadBasketFinallyThread from types import * import pickle import tempfile from cdli_files import CDLIFile from cdli_files import splitatf,checkFile from cdli_helpers import * class BasketContent(SimpleItem): """classe fuer den Inhalt eines Baskets""" def getFileAndVersionFromId(self,pnum,versionNr): obj=self.cdliRoot.getFileObject(pnum) logging.debug("obj : %s"%obj) version=obj.getVersionNr(versionNr) logging.debug("-------vs: %s"%version.getFileName()) return version,obj def __init__(self,content=[]): """content""" self.setContent(content[0:]) def getContent(self,filtered=True): return self.contentList def getContentOld(self,filtered=True): """get content""" logging.debug("content object: content List %s"%self.contentList) ret=[] return [self.getFileAndVersionFromId(x[0],x[1]) for x in self.contentList] # # if filtered: # for x in self.contentList: # if not((x[0] is None) or (x[1] is None)): # ret.append(x) # logging.debug("content object: content List -done filtered") # return ret # # else: # logging.debug("content object: content List -done not filtered") # return self.contentList def allContent(self): """get all content""" return self.getContent(filtered=False) def setContent(self,content): contentList=[] for x in content: if not((x[0] is None) or (x[1] is None)): try: # old version contentList.append((x[1].getId(),x[0].getVersionNumber())) except: contentList.append(x) logging.debug("cl: %s"%contentList) self.contentList=contentList[0:] def numberOfItems(self): """number""" return len(self.getContent()) class uploadATFfinallyThread(Thread): """class for adding uploaded filed (temporarily stored in the staging area at /tmp""" def __init__(self): """init for uploadATFfinallyThread""" self.continueVar=True self.returnValue=None self.end=False Thread.__init__(self) def set(self,procedure,comment="",basketname='',unlock=None,SESSION=None,username=None,serverport="8080"): """set start values for the thread""" self.procedure=procedure self.comment=comment self.basketname=basketname self.unlock=unlock self.SESSION=SESSION self.username=username self.serverport=serverport def __call__(self): """call of the thread (equals run)""" self.run() return True def getContext(self, app,serverport="8080"): """get the context within the ZODB""" resp = HTTPResponse(stdout=None) env = { 'SERVER_NAME':'localhost', 'SERVER_PORT':serverport, 'REQUEST_METHOD':'GET' } req = HTTPRequest(None, env, resp) return app.__of__(RequestContainer(REQUEST = req)) def run(self): """run""" self.result="" #find context within ZODB from Zope2 import DB conn = DB.open() root = conn.root() app = root['Application'] ctx = self.getContext(app,serverport=self.serverport) #add the files self.uploadATFfinallyThread(ctx,self.procedure,comment=self.comment,basketname=self.basketname,unlock=self.unlock,SESSION=self.SESSION,username=self.username) #commit the transactions transaction.get().commit() conn.close() #set flag for end of this method self.end=True logging.info("ended") return True def __del__(self): """delete""" def getResult(self): """method for accessing result""" return self.result def uploadATFfinallyThread(self,ctx,procedure,comment="",basketname='',unlock=None,RESPONSE=None,SESSION=None,username=None): """upload the files""" #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot ctx2=ctx.cdliRoot self.result+="<h2>Start processing</h2>" #shall I only upload the changed files? logging.debug("uploadATFfinally procedure: %s"%procedure) if procedure=="uploadchanged": changed=[x[0] for x in SESSION.get('changed',[])] uploadFns=changed+SESSION.get('newPs',[]) #or all elif procedure=="uploadAll": uploadFns=[] for x in os.listdir(SESSION['tmpdir']): if not x in SESSION['lockerrors']: uploadFns.append(x) #or maybe nothing elif procedure=="noupload": return True else: uploadFns=[] #do first the changed files i=0 for fn in uploadFns: logging.debug("uploadATFfinally uploadFn=%s"%fn) i+=1 founds=ctx2.CDLICatalog.search({'title':fn}) if len(founds)>0: SESSION['author']=str(username) self.result="<p>Changing : %s"%fn+self.result logging.debug("uploadatffinallythread changing:%s"%fn+self.result) founds[0].getObject().manage_addCDLIFileObject('',comment,SESSION['author'],file=os.path.join(SESSION['tmpdir'],fn),from_tmp=True) if i%200==0: transaction.get().commit() logging.debug("uploadatffinallythread changing: do commit") transaction.get().commit() logging.debug("uploadatffinallythread changing: last commit") #now add the new files newPs=SESSION['newPs'] if len(newPs)>0: tmpDir=SESSION['tmpdir'] logging.debug("uploadatffinallythread adding start") self.result="<p>Adding files</p>"+self.result #TODO: make this configurable, at the moment base folder for the files has to be cdli_main ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self) logging.debug("uploadatffinallythread adding finished") #unlock locked files? if unlock: logging.debug("uploadatffinallythread unlocking start") self.result="<p>Unlock files</p>"+self.result unlockFns=[] for x in os.listdir(SESSION['tmpdir']): if not x in SESSION['errors']: unlockFns.append(x) logging.debug("unlocking have now what to unlock") for fn in unlockFns: #logging.info("will unlock: %s"%fn) founds=ctx2.CDLICatalog.search({'title':fn}) #logging.info("found it: %s"%repr(founds)) if len(founds)>0: #logging.info("unlock: %s"%founds[0].getObject().getId()) SESSION['author']=str(username) founds[0].getObject().lockedBy="" logging.debug("uploadatffinallythread unlocking done") #if a basketname is given, add files to the basket if not (basketname ==''): logging.debug("uploadatffinallythread add to basket %s"%basketname) self.result="<p>Add to basket</p>"+self.result basketId=ctx2.basketContainer.getBasketIdfromName(basketname) if not basketId: # create new basket logging.debug("uploadatffinallythread create basket %s"%basketname) self.result="<p>Create a new basket</p>"+self.result ob=ctx2.basketContainer.addBasket(basketname) basketId=ob.getId() basket=getattr(ctx2.basketContainer,str(basketId)) ids=os.listdir(SESSION['tmpdir']) #logging.debug("should add:"+repr(ids)) basket.addObjects(ids,deleteOld=True,username=str(username)) logging.debug("uploadatffinallythread uploadfinally done") if RESPONSE is not None: RESPONSE.redirect(self.aq_parent.absolute_url()) return True class tmpStore(SimpleItem): """simple item""" meta_type="cdli_upload" def __init__(self,id): """init tmp""" self.id=id class uploadATFThread(Thread): """class for checking the files befor uploading""" def __init__(self): """initialise""" self.continueVar=True self.returnValue=None Thread.__init__(self) def set(self,upload,basketId,username,idTmp,serverport="8080"): """set start values for the thread""" self.result="" self.upload=upload self.basketId=basketId self.username=username self.serverport=serverport self.idTmp=idTmp def __call__(self): """call method """ self.run() return True def getContext(self, app,serverport="8080"): """get the context within the ZODB""" resp = HTTPResponse(stdout=None) env = { 'SERVER_NAME':'localhost', 'SERVER_PORT':serverport, 'REQUEST_METHOD':'GET' } req = HTTPRequest(None, env, resp) return app.__of__(RequestContainer(REQUEST = req)) def run(self): idTmp=self.idTmp self.result="" #find context within ZODB from Zope2 import DB conn = DB.open() root = conn.root() app = root['Application'] ctx = self.getContext(app,serverport=self.serverport) logging.info("run intern") try: logging.info("created: %s"%idTmp) ctx.temp_folder._setObject(idTmp,tmpStore(idTmp)) except: logging.error("thread upload: %s %s"%sys.exc_info()[0:2]) logging.info("call thread intern") self.uploadATFThread(ctx,self.upload,idTmp,self.basketId) #ctx.cdliRoot.cdli_main.tmpStore2[self.getName()[0:]]=self.returnValue transaction.get().commit() conn.close() return getattr(ctx.temp_folder,idTmp) def getResult(self): """method for accessing result""" return self.result def uploadATFThread(self,ctx,upload,idTmp,basketId=0): """upload an atf file""" #TODO: add comments #TODO: finish uploadATF stObj=getattr(ctx.temp_folder,idTmp) logging.info("start, upload thread") self.result="<html><body><h2>I got your file, start now to split it into single atf-files!</h2><p>" #make sure that id is a string and not an integer basketId=str(basketId) logging.info("basketID:"+basketId) #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot ctx2=ctx.cdliRoot #get temporary file for staging the downloaded and splitted files dir=tempfile.mkdtemp() logging.info("tmpfFile:"+str(dir)) changed=[] # changed files errors=[] # files with errors lockerrors=[] # files with errors newPs=[] # new p filed psNotInCatalog=[] # files not in the catalog #split the uploadedd atf file logging.info("start splitting") basketNameFromFile, numberOfFiles=splitatf(upload,dir,ext=self) #find basketId if not set #get active abaket if basketId == '0': basketObj=ctx2.basketContainer.getActiveBasket() if basketObj: basketId=basketObj.getId() #if there is no active basket and no basketid given, id is empty, else get besketname and length if basketId == '0': basketNameFromId="" basketLen=0 else: basketNameFromId=getattr(ctx2.basketContainer,basketId).title basketLen=getattr(ctx2.basketContainer,basketId).getLastVersion().numberOfItems() logging.info("got the file, upload thread") self.result+="""<html><body><h2>I got the files</h2>< p>I am computing the differences to the exisiting files</p>""" #start to check the files for fn in os.listdir(dir): self.result="<p>process:%s</p>"%fn+self.result # check if file is in the catalog #TODO: checkCatalog is not implemented yet if ctx2.cdli_main.checkCatalog(fn): psNotInCatalog.append(fn) #check if p-file already at the server founds=ctx2.CDLICatalog.search({'title':fn}) #if not than add filename to the list of newfiles data=file(os.path.join(dir,fn)).read() status,msg=checkFile(fn,data,dir) #status=True if not status: # error errors.append((fn,msg)) else: if len(founds)==0: newPs.append(fn) #if p file alread at the server for found in founds: #analyse the differences to the actual file obj=found.getObject() if (not (str(obj.lockedBy))=='') and (not (str(obj.lockedBy)==str(self.username))): lockerrors.append((fn,str(obj.lockedBy))) else: diffs=obj.diff(data) if diffs[0]>0: changed.append((obj,diffs)) #hochladen #ready, set the returnValues self.result+="<h3>Done</h3></body></html>" stObj.returnValue={} stObj.returnValue['errors']=errors stObj.returnValue['newPs']=newPs stObj.returnValue['tmpdir']=dir stObj.returnValue['basketLen']=basketLen stObj.returnValue['numberOfFiles']=numberOfFiles stObj.returnValue['basketNameFromId']=basketNameFromId stObj.returnValue['basketNameFromFile']=basketNameFromFile stObj.returnValue['basketId']=basketId stObj.returnValue['dir']=dir #stObj.returnValue['changed']=copy.copy(changed) stObj.returnValue['changed']=[(x[0].getId(),x[1][0]) for x in changed] #stObj.returnValue['lockerrors']=[x[0].getId() for x in lockerrors] stObj.returnValue['lockerrors']=[x for x in lockerrors] self.returnValue=True #ctx2.cdli_main.setTemp('v_uploadATF_returnValue',True) class CDLIBasketContainer(OrderedFolder): """contains the baskets""" security=ClassSecurityInfo() meta_type="CDLIBasketContainer" def getResultHash(self): """get the result hash for debug purposes""" return self.resultHash.keys() def getPNumbersOfBasket(self,basketName): """get all pnumbers of a basket as a list, returns an empty list if basket not found @param basketName: name of the basket """ ret=[] basketId=self.getBasketIdfromName(basketName) if not basketId: return [] ob=getattr(self,basketId).getContent() #get the content of a basket ret=[x[0].split(".")[0] for x in ob] return ret security.declareProtected('manage','getBasketAsOneFile') def getBasketAsOneFile(self,basketName,current="no"): """returns all files of the basket combined in one file @param basketName: Name of the basket @param current: (optional) if current is set to "yes" then the most current version of all files are downloaded and not the versions of the files as stored in the basket """ ret="" basketId=self.getBasketIdfromName(basketName) if not basketId: return "" ob=getattr(self,basketId).getLastVersion() for pnum,versionNr in ob.getContent(): obj=self.cdliRoot.getFileObject(pnum) # logging.debug("obj : %s"%obj) # version=obj.getVersionNr(versionNr) if current=="no": #version as they are in the basket cur= obj.getVersionNr(versionNr) ret+=str(cur.getData())+"\n" elif current=="yes": #search current object #logging.debug("current: %s"%object[1].getId().split(".")[0]) obj.getData() return ret security.declareProtected('manage','upDateBaskets') def upDateBaskets(self): """update content in to objects""" founds=self.ZopeFind(self,obj_metatypes=['CDLIBasketVersion'],search_sub=1) for found in founds: found[1].updateBasket() security.declareProtected('manage','deleteBaskets') def deleteBaskets(self,ids=None): """delete baskets, i.e. move them into trash folder""" if ids is None: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','cdliError_html.zpt')).__of__(self) txt="Sorry, no basket selected!" return pt(txt=txt) found=self.ZopeFind(self,obj_ids=['trash']) if len(found)<1: manage_addFolder(self, 'trash') trash=self._getOb('trash') else: trash=found[0][1] if type(ids) is not ListType: ids=[ids] logging.error("XERXON:"+repr(ids)) if len(ids)==0: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','cdliError_html.zpt')).__of__(self) txt="Sorry, no basket selected!" return pt(txt=txt) cut=self.manage_cutObjects(ids) trash.manage_pasteObjects(cut) return None security.declareProtected('manage','manageBaskets') def manageBaskets(self,submit,ids=None,basket1="",basket2="",joinBasket="",subtractBasket="",REQUEST=None,RESPONSE=None): """manage baskets, delete or copy""" if submit=="delete": ret= self.deleteBaskets(ids) if ret: return ret elif submit=="join": flag,msg=self.joinBasket(joinBasket, ids) logging.info("joining %s %s"%(flag,msg)) if not flag: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','cdliError_html.zpt')).__of__(self) return pt(txt=msg) elif submit=="subtract": logging.info("BBBb %s %s"%(basket1,basket2)) flag,msg=self.subtractBasket(subtractBasket, basket1,basket2) logging.info("subtract %s %s"%(flag,msg)) if RESPONSE: RESPONSE.redirect(self.absolute_url()) security.declareProtected('View','getBasketIdfromName') def getBasketIdfromName(self,basketname): """get id from name""" for basket in self.ZopeFind(self,obj_metatypes=["CDLIBasket"]): if basket[1].title==basketname: return basket[0] else: None security.declareProtected('manage','uploadBasket_html') def uploadBasketAsync_html(self,basketId='0'): """upload an atf file, html form, jetzt aufruf der nicht asyncronen version mit ticktes""" basketId=str(basketId) if not basketId=='0': basketName=getattr(self.basketContainer,basketId).title else: basketName="" pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadBasketAsync_html.zpt')).__of__(self) return pt(basketId=basketId,basketName=basketName) def uploadBasket_html(self,basketId='0'): """upload an atf file, html form""" basketId=str(basketId) if not basketId=='0': basketName=getattr(self.basketContainer,basketId).title else: basketName="" pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadBasket_html.zpt')).__of__(self) return pt(basketId=basketId,basketName=basketName) security.declareProtected('manage','index_html') def index_html(self): """stanadard ansicht""" ext=self.ZopeFind(self,obj_ids=["index.html"]) if ext: return ext[0][1]() pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketContainerMain')).__of__(self) return pt() def getStorageFolderRoot(self): """root des storage folders""" return self.cdli_main def __init__(self,id,title): """ init basket container""" self.id=id self.title=title def getBasketsId(self): """get all baskets als klartext""" ret="" baskets=self.ZopeFind(self,obj_metatypes=['CDLIBasket']) for basket in baskets: com,user,time,values = basket[1].getContentIds() ret+= "BASKET:"+com+"\t"+user+"\t"+time+"\n" for x in values: ret+= x[0]+"\t"+x[1]+"\n" return ret def getBaskets(self,sortField='title'): """get all baskets files""" def sortName(x,y): return cmp(x[1].title.lower(),y[1].title.lower()) def sortDate(x,y): return cmp(y[1].getLastVersion().getTime(),x[1].getLastVersion().getTime()) def sortComment(x,y): try: xc=getattr(x[1],'comment','ZZZZZZZZZZZZZ').lower() except: xc='ZZZZZZZZZZZZZ'.lower() try: yc=getattr(y[1],'comment','ZZZZZZZZZZZZZ').lower() except: yc='ZZZZZZZZZZZZZ'.lower() if (xc=='') or (xc=='ZZZZZZZZZZZZZ'.lower()): try: xc=x[1].getLastVersion().getComment().lower() except: xc='ZZZZZZZZZZZZZ'.lower() if (yc=='') or (yc=='ZZZZZZZZZZZZZ'.lower()): try: yc=y[1].getLastVersion().getComment().lower() except: yc='ZZZZZZZZZZZZZ'.lower() return cmp(xc,yc) def sortAuthor(x,y): return cmp(x[1].getLastVersion().getUser().lower(),y[1].getLastVersion().getUser().lower()) baskets=self.ZopeFind(self,obj_metatypes=['CDLIBasket']) if sortField=='title': baskets.sort(sortName) elif sortField=='date': baskets.sort(sortDate) elif sortField=='author': baskets.sort(sortAuthor) elif sortField=='comment': baskets.sort(sortComment) return baskets def subtractBasket(self,newBasket,basket1,basket2): """subtract basket2 from basket1 (i.e. newbasket will contain alle elements of basket1 which are not in basket2), if basket2 contains files which are not in basket1, then theses files fill be ignored @param newbasket: name of the new basket @param basket1: basket where basket2 will be subtracted from @param basket2: see above """ logging.info("CCCCC %s %s"%(basket1,basket2)) try: newB=self.addBasket(newBasket) except: return False, "cannot create the new basket" bas2= getattr(self,basket2) bas2content=bas2.getContent() bas2ids=[x[0] for x in bas2content] bas1= getattr(self,basket1) bas1content=bas1.getContent() newBasketContent={} for id,version in bas1content: if not (id in bas2ids): newBasketContent[id]=version username=self.getActualUserName() logging.info("sbc %s"%newBasketContent) newB.addObjectsWithVersion(newBasketContent,username=username,catalog=self.CDLICatalog) return True, "" def joinBasket(self,newBasket,oldBaskets): """join two baskets @param newbasket: name of the new basket @param oldbaskets: list of baskets to be joined """ if oldBaskets is None: return False, "No Baskets selected!" try: newB=self.addBasket(newBasket) except: return False, "cannot create the new basket" newBasketContent={} for ob in oldBaskets: x= getattr(self,ob,None) if x is None: return False, "cannot find basket: %s"%ob ids=x.getContent() # hole den Inhalt for id,version in ids: if newBasketContent.has_key(id): # p number gibt's schon newBasketContent[id]=max(newBasketContent[id],version) # speichere die groessere Versionsnumber else: newBasketContent[id]=version username=self.getActualUserName() logging.info("nbc %s"%newBasketContent) newB.addObjectsWithVersion(newBasketContent,username=username,catalog=self.CDLICatalog) return True, "" def getNewId(self): """createIds""" last=getattr(self,'last',0) last +=1 while len(self.ZopeFind(self,obj_ids=[str(last)]))>0: last+=1 self.last=last return last def setActiveBasket(self,basketId,REQUEST=None): """store active basketId in a cookie""" self.REQUEST.RESPONSE.setCookie("CDLIActiveBasket",basketId,path="/") try: qs=cgi.parse_qs(REQUEST['QUERY_STRING']) del(qs['basketId']) except: qs={} if REQUEST: REQUEST.RESPONSE.redirect(REQUEST['URL1']+'?'+urllib.urlencode(qs)) def getActiveBasket(self): """get active basket from cookie""" id= self.REQUEST.cookies.get('CDLIActiveBasket',None) if id: obj=getattr(self,str(id),None) else: obj=None return obj def getActualUserName(self): """get name of the actualuser""" return str(self.REQUEST['AUTHENTICATED_USER']) security.declareProtected('manage','addBasket') def addBasket(self,newBasketName): """add a new basket""" ob=manage_addCDLIBasket(self,newBasketName) return ob def storeInBasket(self,submit,ids=None,newBasketName=None,fromFileList=None,RESPONSE=None,REQUEST=None): """store it""" if not ids: ids=self.REQUEST.SESSION['fileIds'] if (type(ids) is not ListType) and (not isinstance(ids,Set)): ids=[ids] if isinstance(ids,Set): ids=list(ids) if (submit.lower()=="store in new basket") or (submit.lower()=="new basket"): basketRet=self.addBasket(newBasketName) self.setActiveBasket(basketRet.getId()) basket=getattr(self,basketRet.getId()) elif (submit.lower()=="store in active basket") or (submit.lower()=="active basket"): basket=self.getActiveBasket() added=basket.addObjects(ids) back=self.REQUEST['HTTP_REFERER'].split("?")[0]+"?basketName="+basket.title+"&numberOfObjects="+str(added) if fromFileList: return self.cdli_main.findObjectsFromList(list=ids,basketName=basket.title,numberOfObjects=added) if RESPONSE: RESPONSE.redirect(back) return True def manage_addCDLIBasketContainerForm(self): """add the CDLIBasketContainer form""" pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIBasketContainer.zpt')).__of__(self) return pt() def manage_addCDLIBasketContainer(self,id,title,RESPONSE=None): """add the basket""" ob=CDLIBasketContainer(id,title) self._setObject(id, ob) if RESPONSE is not None: RESPONSE.redirect('manage_main') class CDLIBasket(Folder,CatalogAware): """basket""" meta_type="CDLIBasket" default_catalog="CDLIBasketCatalog" def searchInBasket(self,indexName,searchStr,regExp=False): """searchInBasket""" lst=self.searchInLineIndexDocs(indexName,searchStr,uniq=True,regExp=regExp) #TODO: fix this ret={} lv=self.getLastVersion() for obj in lv.content.getContent(): id=obj[1].getId().split(".")[0] if id in lst: ret[id]=self.showWordInFile(id,searchStr,lineList=self.getLinesFromIndex(indexName,searchStr,id,regExp=regExp),regExp=regExp,indexName=indexName) pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','searchResultsInBasket')).__of__(self) return pt(result=ret,indexName=indexName,regExp=regExp,word=searchStr) def searchInBasket_v1(self,searchStr): """search occurences of searchStr in files im basket""" ret=[] lv=self.getLastVersion() logging.info("searching") for obj in lv.content.getContent(): txt=obj[0].getData() for x in txt.split("\n"): logging.info("search %s"%x) if re.match(searchStr,x): ret.append(x) return "\n".join(ret) def getFile(self,obj): return obj[1] def getFileLastVersion(self,obj): return obj[0] def getFileNamesInLastVersion(self): """get content of the last version as list""" return [x[1].getId() for x in self.getLastVersion().getContent()] def isActual(self,obj,nummer): """teste ob im basket die aktuelle version ist, obj kann entweder ein CDLIFile sein oder eine eine pnummer, die auf ein CDLIFile verweist""" try: #logging.debug("isActual:"+repr(obj)) if isinstance(obj, CDLIFile): actualNo=obj.getLastVersion().getVersionNumber() else: actualNo=self.cdliRoot.getFileObjectLastVersion(obj).getVersionNumber() if actualNo==nummer: return True , 0 else: return False, actualNo except: logging.error( """is actual: %s (%s %s)"""%(repr(obj),sys.exc_info()[0],sys.exc_info()[1])) logging.error(""" PARAMS: %s %s"""%(obj,nummer)) return False, -1 def isActualOld(self,obj): """teste ob im basket die aktuelle version ist""" try: #logging.debug("isActual:"+repr(obj)) actualNo=obj[1].getLastVersion().getVersionNumber() storedNo=obj[0].getVersionNumber() #actualNo=self.getFileObjectLastVersion(obj.getId()).getVersionNumber() #if len(founds)>0 and founds[0].getObject().aq_parent.getId()==".trash": # return False, -1 if actualNo==storedNo: return True , 0 else: return False, actualNo except: logging.error( """is actual: %s (%s %s)"""%(repr(obj),sys.exc_info()[0],sys.exc_info()[1])) return False, -1 def history(self): """history""" ext=self.ZopeFind(self.aq_parent,obj_ids=["history_template.html"]) if ext: return getattr(self,ext[0][1].getId())() pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketHistory')).__of__(self) return pt() def getStorageFolderRoot(self): """root des storage folders""" return self.aq_parent.cdli_main def __init__(self,id,title,shortDescription="",comment=""): """init a CDLIBasket""" self.id=id self.title=title self.shortDescription=shortDescription self.comment=comment def getActualUserName(self): """get name of the actualuser""" return str(self.REQUEST['AUTHENTICATED_USER']) def getLastVersion(self): """hole letzte version""" ids=[] idsTmp= self.objectIds() for x in idsTmp: try: ids.append(int(x)) except: pass ids.sort() if len(ids)==0: return None else: ob=getattr(self,str(ids[-1])) return ob def getVersions(self): """get versions""" versions=self.ZopeFind(self,obj_metatypes=["CDLIBasketVersion"]) return versions def updateObjects(self,ids=[],RESPONSE=None,REQUEST=None): """update ids, ids not in the basket then add""" logging.debug("update") logging.debug(repr(ids)) if type(ids) is not ListType: ids=[ids] lastVersion=self.getLastVersion() oldContent=lastVersion.content.getContent() newContent=[] #keine ids dann alle update if len(ids)==0: ids=[x[0] for x in oldContent] #first copy the id not to be updated for obj in oldContent: logging.debug("obj:"+repr(obj)) try: # old format of the basket if obj[1].getId() not in ids: newContent.append(obj) except: if obj[0] not in ids: newContent.append(obj) #now add the new for id in ids: founds=self.CDLICatalog.search({'title':id}) for found in founds: if found.getObject() not in oldContent: #TODO: was passiert wenn, man eine Object dazufuegt, das schon da ist aber eine neuere version newContent.append((id,found.getObject().getLastVersion().getVersionNumber())) content=newContent user=self.getActualUserName() logging.debug(newContent) ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=newContent) obj=self._getOb(ob.getId()) logging.debug(repr(obj)) logging.debug("done") logging.debug(obj.absolute_url()) if RESPONSE: logging.debug("redirect") RESPONSE.redirect(obj.absolute_url()) return obj def addObjectsWithVersion(self,ids,deleteOld=None,username=None,catalog=None): """generate a new version of the basket with objects added, hier wird jedoch nicht die letzte Version jedes Files hinzugefuegt, s ondern ids is ein Tupel mit der Id (d.h. der p-number) und der Versionsnummer. """ logging.info("add to basket (%s)"%(self.getId())) lastVersion=self.getLastVersion() if not catalog: catalog=self.CDLICatalog if lastVersion is None: oldContent=[] else: oldContent=lastVersion.content.getContent() if deleteOld: oldContent=[] newContent=[] added=0 for id,version in ids.iteritems(): logging.info("adding %s %s"%(id,version)) id=id.split(".")[0] # title nur die pnumber ohne atf try: founds=catalog.search({'title':id}) except: founds=[] logging.info(" found %s "%(founds)) for found in founds: if found.getObject() not in oldContent: #TODO: was passiert wenn, man eine Object dazufuegt, das schon da ist aber eine neuere version newContent.append((found.getObject().getVersions()[version-1][1],found.getObject())) added+=1 content=oldContent+newContent if not username: logging.error("XXXXXXXXXXX %s"%repr(self)) user=self.getActualUserName() else: user = username ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=content) logging.info("add to basket (%s) done"%(self.getId())) return added def addObjects(self,ids,deleteOld=None,username=None): """generate a new version of the basket with objects added""" def swap(x): return (x[1],x[0]) logging.info("add to basket (%s)"%(repr(ids))) logging.info("add to basket (%s)"%(self.getId())) lastVersion=self.getLastVersion() if lastVersion is None: oldContent=[] else: oldContent=lastVersion.content.getContent() if deleteOld: oldContent=[] added=0 # for id in ids: # logging.debug("adding:"+id) # try: # founds=self.CDLICatalog.search({'title':id}) # except: # founds=[] # # for found in founds: # if found.getObject() not in oldContent: # #TODO: was passiert wenn, man eine Object dazufugt, das schon da ist aber eine neuere version # newContent.append((found.getObject().getLastVersion(),found.getObject())) # added+=1 hash = md5.new(repr(makelist(ids))).hexdigest() # erzeuge hash als identification #logging.debug("JJJJJJJ:"+repr(self.makelist(ids))) retrieved = self.CDLICache.retrieve(hash) if retrieved: logging.debug("add Objects: retrieved") newContent=Set(map(swap,retrieved)) else: logging.debug("add Objects: new") newContent=Set([(self.getFileObjectLastVersion(x),self.getFileObject(x)) for x in ids]) #remove all Elements which are not stored if (None,None) in newContent: newContent.remove((None,None)) content=Set(oldContent).union(newContent) added = len(content)-len(oldContent) if not username: user=self.getActualUserName() else: user = username #logging.debug("content:"+repr(list(content))) ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=list(content)) logging.info("add to basket (%s) done"%(self.getId())) return added def getContent(self): """print content""" ret=[] lv=self.getLastVersion() #for obj in lv.content.getContent(): #logging.info("XXXXXXXXXX %s"%repr(obj)) # ret.append((obj[1].getId(),obj[0].versionNumber)) return lv def getContentIds(self): """print basket content""" ret=[] lv=self.getLastVersion() for obj in lv.content.getContent(): ret.append((obj[0].getId(),obj[1].getId())) return lv.getComment(),lv.getUser(),lv.getTime(),ret def changeBasket(self,submit,ids=[],RESPONSE=None,REQUEST=None): """change a basket""" if submit=="update": self.updateObjects(ids,RESPONSE=RESPONSE,REQUEST=REQUEST) elif submit=="delete": self.deleteObjects(ids,RESPONSE=RESPONSE,REQUEST=REQUEST) def deleteObjects(self,ids,RESPONSE=None,REQUEST=None): """delete objects""" if type(ids) is not ListType: ids=[ids] lastVersion=self.getLastVersion() oldContent=lastVersion.content.getContent() newContent=[] for obj in oldContent: if obj[1].getId() not in ids: newContent.append(obj) user=self.getActualUserName() ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=newContent) if RESPONSE: obj=self._getOb(ob.getId()) RESPONSE.redirect(obj.absolute_url()) def manage_addCDLIBasketForm(self): """add the CDLIBasketContainer form""" pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIBasket.zpt')).__of__(self) return pt() def manage_addCDLIBasket(self,title,shortDescription="",comment="",RESPONSE=None): """add the basket""" id=str(self.getNewId()) ob=CDLIBasket(id,title,shortDescription,comment) self._setObject(id, ob) if RESPONSE is not None: RESPONSE.redirect('manage_main') else: return ob class CDLIBasketVersion(Implicit,Persistent,Folder): """version of a basket""" meta_type="CDLIBasketVersion" security=ClassSecurityInfo() def updateBasket(self): """update""" try: self._setObject('content',BasketContent(self.basketContent)) except: try: if len(self.basketContent)>0: self.content.setContent(self.basketContent) except: print "error",self.getId(),self.aq_parent.getId() self.basketContent=[] def containsNonActualFiles(self): """returns True if basket contains one or more non current files""" objs=self.getContent() for obj in objs: if not self.isActual(obj[0],obj[1])[0]: return True return False def downloadListOfPnumbers(self): """download pnumbers of the basket as list""" basket_name=self.aq_parent.title ids=self.getContent() # get the list of objects logging.error(ids) ret="\n".join([x[1].getId().split(".")[0] for x in ids]) self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.txt" """%basket_name) self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream") length=len(ret) self.REQUEST.RESPONSE.setHeader("Content-Length",length) self.REQUEST.RESPONSE.write(ret) security.declareProtected('manage','downloadObjectsAsOneFile') def downloadObjectsAsOneFile(self,lock=None,procedure=None,REQUEST=None,check="yes",current="no"): """download all selected files in one file""" if self.temp_folder.downloadCounterBaskets > 10000: return """I am sorry, currently the server has to many requests for downloads, please come back later!""" #if (check=="yes") and self.containsNonActualFiles(): # pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadObjectAsOneFile_check.zpt')).__of__(self) # # return pt(lock=lock) # neue Version aus Performancegruenden, es wird nicht mehr getestet, ob es nicht aktuelle Objekte gibt # sondern lediglich gefragt. if (check=="yes"): pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadObjectAsOneFile_ask.zpt')).__of__(self) return pt(lock=lock) else: return self.downloadObjectsAsOneFileFinally(lock=lock,procedure=procedure,REQUEST=REQUEST,current="no") def downloadObjectsAsOneFileFinally(self,lock=None,procedure=None,REQUEST=None,current="no",repeat=None): """print do the download""" ret="" lockedObjects={} if lock: logging.debug("------lock:"+repr(lock)) if str(self.REQUEST['AUTHENTICATED_USER'])=='Anonymous User': return "please login first" #check if a locked object exist in the basket. lockedObjects={} for object in self.content.getContent(): obj=self.getFileObject(object[0]) if (not str(obj.lockedBy)=="") and (not (str(obj.lockedBy)==str(self.REQUEST['AUTHENTICATED_USER']))): lockedObjects[obj.title]=repr(obj.lockedBy) keys=lockedObjects.keys() if len(keys)>0 and (not procedure): self.REQUEST.SESSION['lockedObjects']=lockedObjects pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','lockedObjects.zpt')).__of__(self) return pt() elif not procedure: #keine fails gesperrt dann alle donwloaden procedure="downloadAll" threadName=repeat if not threadName or threadName=="": thread=DownloadBasketFinallyThread() threadName=thread.getName()[0:] if (not hasattr(self,'_v_downloadBasket')): self._v_downloadBasket={} self._v_downloadBasket[threadName]=thread logging.debug("dwonloadfinally:"+repr(self)) if isinstance(self,CDLIBasketVersion): obj=self else: obj=self.aq_parent logging.debug("dwonloadfinally2:"+repr(obj)) logging.debug("dwonloadfinally2:"+repr(obj.aq_parent)) obj2=obj.aq_parent if not isinstance(obj2,CDLIBasket): obj2=obj2.aq_parent basketID=obj2.getId() versionNumber=obj.getId() logging.debug("dwonloadfinally2:"+repr(basketID)) logging.debug("dwonloadfinally2:"+repr(versionNumber)) if lock: logging.debug("-----start locking") for object in self.content.getContent(): obj=self.getFileObject(object[0]) if obj.lockedBy =='': obj.lockedBy=self.REQUEST['AUTHENTICATED_USER'] logging.debug("-----finished locking") #obj.lockedBy=user self._v_downloadBasket[threadName].set(lock,procedure,self.REQUEST['AUTHENTICATED_USER'],current,basketID,versionNumber) self._v_downloadBasket[threadName].start() wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template']) if wait_template: return wait_template[0][1]() pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadBasketWait.zpt')).__of__(self) return pt(txt=self.absolute_url()+'/downloadObjectsAsOneFileFinally',threadName=threadName, counter=self._v_downloadBasket[threadName].getCounter(), number=self._v_downloadBasket[threadName].getNumberOfFiles()) #_v_xmltrans.run() else: #recover thread, if lost if not hasattr(self,'_v_downloadBasket'): self._v_downloadBasket={} if not self._v_downloadBasket.get(threadName,None): for thread in threading.enumerate(): if threadName == thread.getName(): self._v_downloadBasket[threadName]=thread if self._v_downloadBasket.get(threadName,None) and (self._v_downloadBasket[threadName] is not None) and (not self._v_downloadBasket[threadName].end) : wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template']) if wait_template: return wait_template[0][1]() pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadBasketWait.zpt')).__of__(self) return pt(txt=self.absolute_url()+'/downloadObjectsAsOneFileFinally',threadName=threadName, counter=self._v_downloadBasket[threadName].getCounter(), number=self._v_downloadBasket[threadName].getNumberOfFiles()) else: logging.debug("FINISHED") if not self._v_downloadBasket.get(threadName,None): for thread in threading.enumerate(): if threadName == thread.getName(): self._v_downloadBasket[threadName]=thread #files = self._v_downloadBasket[threadName].result # lade die files und die locked files, bei grossen Baskets muss u.U. gewartet werden # bis das Commit aus dem Thread alles geschrieben hat, in dem Falle existiert resultHash[threadName] # noch nicht. o1 = file("/tmp/"+threadName,'r') files=pickle.load(o1) os.remove("/tmp/"+threadName) o2 = file("/tmp/"+threadName+'_lockedFiles','r') lockedFiles=pickle.load(o2) os.remove("/tmp/"+threadName+'_lockedFiles') # try: # files=self.basketContainer.resultHash[threadName] # except: # i=0 # while (not self.basketContainer.resultHash.has_key(threadName)) and (i<100): # logging.debug(" downloadFinally: I am waiting for thread %s to write the resultHashfile: %s"%(threadName,i)) # time.sleep(5) # i+=1 # files=self.basketContainer.resultHash[threadName] # # try: # lockedFiles=self.basketContainer.resultLockedHash[threadName] # except: # i=0 # while (not self.basketContainer.resultLockedHash.has_key(threadName)) and (i<100): # logging.debug(" downloadFinally: I am waiting for thread %s to write the LockedHashfile: %s"%(threadName,i)) # time.sleep(5) # i+=1 # lockedFiles=self.basketContainer.resultLockedHash[threadName] # fh=file("/var/tmp/test") #ret =fh.read() if (not isinstance(self.aq_parent,CDLIBasket)): basket_name=self.aq_parent.aq_parent.title+"_V"+self.getId() else: basket_name=self.aq_parent.title+"_V"+self.getId() #write basketname to header of atf file self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.atf" """%basket_name) self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream") #length=len(ret) #self.REQUEST.RESPONSE.setHeader("Content-Length",length) ret="#basket: %s\n"%basket_name self.REQUEST.RESPONSE.write(ret) for fileName in files: logging.debug("download: %s"%fileName) try: self.REQUEST.RESPONSE.write(file(fileName).read()) except: logging.error("downloadasonefile: cannot read %s"%fileName) self.REQUEST.RESPONSE.write("\n# locked files\n") for fileName in lockedFiles: self.REQUEST.RESPONSE.write("# %s by %s\n"%fileName) self.REQUEST.RESPONSE.write("# locked files end\n") #del self.basketContainer.resultHash[threadName] #del self.basketContainer.resultLockedHash[threadName] def numberOfItems(self): """return anzahl der elemente im basket""" return self.content.numberOfItems() def getTime(self): """getTime""" #return self.bobobase_modification_time().ISO() if hasattr(self,'time'): return time.strftime("%Y-%m-%d %H:%M:%S",self.time) elif hasattr(self,'timefixed'): return self.timefixed else: setattr(self,'timefixed',self.bobobase_modification_time().ISO()) return self.bobobase_modification_time().ISO() def getContent(self): """get Basket Content""" logging.debug("retrieving content A") cnt = self.content logging.debug("retrieving content: obj %s"%cnt) tmp = self.content.getContent() logging.debug("got content") return tmp def __init__(self,id,user,comment="",basketContent=[]): """ init a basket version""" self.id=id self.comment=comment self._setObject('content',BasketContent(basketContent)) #self.basketContent=basketContent[0:]a self.user=user self.time=time.localtime() def getUser(self): """get user""" return self.user def getComment(self): """get Comment""" return self.comment security.declareProtected('manage','index_html') def index_html(self): """view the basket""" logging.debug("start index_html - Basket version") if self.REQUEST.get('change',False): ob=self.aq_parent.updateObjects(self.REQUEST['change']) self.REQUEST.RESPONSE.redirect(ob.absolute_url())#go to new basket, because changing generates a new basket logging.debug("start index_html - Basket version:template") pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketVersionMain.zpt')).__of__(self) return pt() def getObjUrl(self,result): """getUrl of the version of the object""" logging.debug("getObjUrl:"+result) founds=self.CDLICatalog.search({'title':result}) if len(founds)>0: logging.debug("getObjUrl: founds 1") logging.debug("getObjUrl:"+repr(founds[0].getObject())) #return founds[0].getObject().getLastVersion().absolute_url() return founds[0].getObject().absolute_url() else: #assume version number logging.debug("getObjUrl: founds 2") splitted=result.split("_") founds=self.CDLICatalog.search({'title':splitted[1]}) return founds[0].getObject().getLastVersion().absolute_url()+'/'+result def manage_addCDLIBasketVersion(self,user,comment="",basketContent=[],RESPONSE=None): """add a version""" #check for already existing versions lastVersion=self.getLastVersion() if lastVersion is None: newId=str(1) else: newId=str(int(lastVersion.getId())+1) ob=CDLIBasketVersion(newId,user,comment,basketContent) self._setObject(newId, ob) if RESPONSE is not None: RESPONSE.redirect('manage_main') else: return ob