from ZODB import DB from ZEO import ClientStorage from ZPublisher.HTTPResponse import HTTPResponse from ZPublisher.HTTPRequest import HTTPRequest from ZPublisher.BaseRequest import RequestContainer from ZCatalog import ZCatalog import pickle import logging from Products.cdli.cdli_files import CDLIRoot import sys import os import os.path import transaction class uploadATFfinallyThread: """class for adding uploaded filed (temporarily stored in the staging area at /tmp""" def __init__(self): """init for uploadATFfinallyThread""" logging.getLogger().setLevel(logging.DEBUG) def set(self,procedure,uploadId,comment="",basketname='',unlock=None,username=None,serverport="29080"): """set start values for the thread""" self.result=file("/tmp/"+str(basketname)+".out","w") self.procedure=procedure self.comment=comment self.basketname=basketname self.unlock=unlock self.username=username self.serverport=serverport self.uploadId=uploadId def __call__(self): """call of the thread (equals run)""" self.run() return True def getContext(self, app,serverport="29080"): """get the context within the ZODB""" resp = HTTPResponse(stdout=None) env = { 'SERVER_NAME':'localhost', 'SERVER_PORT':serverport, 'REQUEST_METHOD':'GET' } req = HTTPRequest(None, env, resp) return app.__of__(RequestContainer(REQUEST = req)) def run(self): """run""" from App.config import getConfiguration print getConfiguration() #find context within ZODB #find context within ZODB storage=ClientStorage.ClientStorage(("localhost",8100)); db = DB(storage) conn = db.open() root = conn.root() app = root['Application'] ctx = self.getContext(app,serverport=self.serverport) #add the files self.uploadATFfinallyThread(ctx,self.procedure,comment=self.comment,basketname=self.basketname,unlock=self.unlock,username=self.username) #commit the transactions db.close() return True def __del__(self): """delete""" def getResult(self): """method for accessing result""" ret="" for x in self.result.readlines(): ret+=x return ret def uploadATFfinallyThread(self,ctx,procedure,comment="",basketname='',unlock=None,username=None): """upload the files""" #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot logging.debug(ctx) ctx2a=ctx.cdliRoot ctx2 = self.getContext(ctx2a,serverport=self.serverport) self.result.write("

Start processing

\n") tmp = pickle.load(file("/tmp/"+str(self.uploadId)+".result")) errorFn = [x[0] for x in tmp['errors']] #shall I only upload the changed files? logging.debug("uploadATFfinally procedure: %s"%procedure) if procedure.lower()=="uploadchanged": changedTmp=[x[0] for x in tmp.get('changed',[])] uploadFns=[] uploadFnsTmp=changedTmp+tmp.get('newPs',[]) for x in uploadFnsTmp: if (not x in tmp['lockerrors']) and ( not x in errorFn): uploadFns.append(x) #or all elif procedure.lower()=="uploadall": uploadFns=[] #workaround fuer memory fehler in listdir #http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir dirTmp=tmp['tmpdir'] ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n') ids.remove(dirTmp) n = len(dirTmp) if dirTmp[-1] != os.path.sep: n += 1 ids = [f[n:] for f in ids] # remove dir prefix for x in ids: if (not x in tmp['lockerrors']) and ( not x in errorFn): uploadFns.append(x) #or maybe nothing elif procedure=="noupload": return True else: uploadFns=[] #do first the changed files i=0 for fn in uploadFns: logging.debug("uploadATFfinally uploadFn=%s"%fn) i+=1 logging.debug(ctx2) catalog = ctx2['CDLICatalog'] print catalog founds=catalog.search({'title':fn}) if len(founds)>0: tmp['author']=str(username) self.result.write("

Changing : %s\n"%fn) logging.debug("uploadatffinallythread changing:%s"%fn) founds[0].getObject().manage_addCDLIFileObject('',comment,tmp['author'],file=os.path.join(tmp['tmpdir'],fn),from_tmp=True) if i%200==0: transaction.get().commit() logging.debug("uploadatffinallythread changing: do commit") self.result.flush() #now add the new files newPs=tmp['newPs'] if len(newPs)>0: tmpDir=tmp['tmpdir'] logging.debug("uploadatffinallythread adding start") self.result.write("

Adding files

\n") #TODO: make this configurable, at the moment base folder for the files has to be cdli_main print ctx2 ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self) logging.debug("uploadatffinallythread adding finished") #unlock locked files? if unlock: logging.debug("uploadatffinallythread unlocking start") self.result.write("

Unlock files

\n") unlockFns=[] #workaround fuer memory fehler in listdir #http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir dirTmp=tmp['tmpdir'] ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n') ids.remove(dirTmp) n = len(dirTmp) if dirTmp[-1] != os.path.sep: n += 1 ids = [f[n:] for f in ids] # remove dir prefix for x in ids: if not x in errorFn: unlockFns.append(x) logging.debug("unlocking have now what to unlock") for fn in unlockFns: #logging.info("will unlock: %s"%fn) founds=ctx2.CDLICatalog.search({'title':fn}) #logging.info("found it: %s"%repr(founds)) if len(founds)>0: #logging.info("unlock: %s"%founds[0].getObject().getId()) tmp['author']=str(username) founds[0].getObject().lockedBy="" logging.debug("uploadatffinallythread unlocking done") #if a basketname is given, add files to the basket if not (basketname ==''): logging.debug("uploadatffinallythread add to basket %s"%basketname) self.result.write("

Add to basket

\n") basketId=ctx2.basketContainer.getBasketIdfromName(basketname) if not basketId: # create new basket logging.debug("uploadatffinallythread create basket %s"%basketname) self.result.write("

Create a new basket

\n") ob=ctx2.basketContainer.addBasket(basketname) basketId=ob.getId() basket=getattr(ctx2.basketContainer,str(basketId)) #workaround fuer memory fehler in listdir #http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir dirTmp=tmp['tmpdir'] ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n') ids.remove(dirTmp) n = len(dirTmp) if dirTmp[-1] != os.path.sep: n += 1 ids = [f[n:] for f in ids] # remove dir prefix #ids=os.listdir(tmp['tmpdir']) #logging.debug("should add:"+repr(ids)) addIds=[] for id in ids: if not (id in errorFn): addIds.append(id) basket.addObjects(addIds,deleteOld=True,username=str(username)) transaction.get().commit() logging.debug("uploadatffinallythread uploadfinally done") return True if __name__ == "__main__": if len(sys.argv)<7: print """Usage: procedure uploadId comment basketName unlock username procedure: can be either uploadchanged or uploadall uploadId: Ticket ID von uploadATF comment: will be added to all docs basketName: name of the basket unlock: unlock or keeplock """ #set(self,procedure,uploadId,comment="",basketname='',unlock=None,username=None,serverport="8080"): upload = uploadATFfinallyThread() x=sys.argv; if sys.argv[5]=="unlock": unlock=True else: unlock=False upload.set(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],unlock=[5],username=sys.argv[6]) upload.run();