Annotation of cdli/tools/uploadATFAsyncFinally.py, revision 1.1
1.1 ! dwinter 1: from ZODB import DB
! 2: from ZEO import ClientStorage
! 3: from ZPublisher.HTTPResponse import HTTPResponse
! 4: from ZPublisher.HTTPRequest import HTTPRequest
! 5: from ZPublisher.BaseRequest import RequestContainer
! 6:
! 7: from ZCatalog import ZCatalog
! 8: import pickle
! 9: import logging
! 10: from Products.cdli.cdli_files import CDLIRoot
! 11: import sys
! 12: import os
! 13: import os.path
! 14:
! 15: class uploadATFfinallyThread:
! 16: """class for adding uploaded filed (temporarily stored in the staging area at /tmp"""
! 17:
! 18: def __init__(self):
! 19: """init for uploadATFfinallyThread"""
! 20: logging.getLogger().setLevel(logging.DEBUG)
! 21:
! 22: def set(self,procedure,uploadId,comment="",basketname='',unlock=None,username=None,serverport="8080"):
! 23: """set start values for the thread"""
! 24:
! 25: self.result=file("/tmp/"+str(basketname)+".out","w")
! 26: self.procedure=procedure
! 27: self.comment=comment
! 28: self.basketname=basketname
! 29: self.unlock=unlock
! 30: self.username=username
! 31: self.serverport=serverport
! 32: self.uploadId=uploadId
! 33:
! 34: def __call__(self):
! 35: """call of the thread (equals run)"""
! 36: self.run()
! 37: return True
! 38:
! 39: def getContext(self, app,serverport="8080"):
! 40: """get the context within the ZODB"""
! 41:
! 42: resp = HTTPResponse(stdout=None)
! 43: env = {
! 44: 'SERVER_NAME':'localhost',
! 45: 'SERVER_PORT':serverport,
! 46: 'REQUEST_METHOD':'GET'
! 47: }
! 48: req = HTTPRequest(None, env, resp)
! 49: return app.__of__(RequestContainer(REQUEST = req))
! 50:
! 51:
! 52: def run(self):
! 53: """run"""
! 54: from App.config import getConfiguration
! 55: print getConfiguration()
! 56:
! 57: #find context within ZODB
! 58: #find context within ZODB
! 59: storage=ClientStorage.ClientStorage(("localhost",8100));
! 60: db = DB(storage)
! 61: conn = db.open()
! 62:
! 63: root = conn.root()
! 64: app = root['Application']
! 65: ctx = self.getContext(app,serverport=self.serverport)
! 66:
! 67: #add the files
! 68: self.uploadATFfinallyThread(ctx,self.procedure,comment=self.comment,basketname=self.basketname,unlock=self.unlock,username=self.username)
! 69: #commit the transactions
! 70:
! 71: db.close()
! 72: return True
! 73:
! 74: def __del__(self):
! 75: """delete"""
! 76:
! 77:
! 78:
! 79: def getResult(self):
! 80: """method for accessing result"""
! 81: ret=""
! 82: for x in self.result.readlines():
! 83: ret+=x
! 84:
! 85: return ret
! 86:
! 87: def uploadATFfinallyThread(self,ctx,procedure,comment="",basketname='',unlock=None,username=None):
! 88: """upload the files"""
! 89: #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot
! 90: logging.debug(ctx)
! 91: ctx2a=ctx.cdliRoot
! 92: ctx2 = self.getContext(ctx2a,serverport=self.serverport)
! 93:
! 94:
! 95: self.result.write("<h2>Start processing</h2>\n")
! 96:
! 97: tmp = pickle.load(file("/tmp/"+str(self.uploadId)+".result"))
! 98: errorFn = [x[0] for x in tmp['errors']]
! 99: #shall I only upload the changed files?
! 100: logging.debug("uploadATFfinally procedure: %s"%procedure)
! 101: if procedure=="uploadchanged":
! 102: changedTmp=[x[0] for x in tmp.get('changed',[])]
! 103: uploadFns=[]
! 104:
! 105: uploadFnsTmp=changedTmp+tmp.get('newPs',[])
! 106: for x in uploadFnsTmp:
! 107: if (not x in tmp['lockerrors']) and ( not x in errorFn):
! 108: uploadFns.append(x)
! 109: #or all
! 110: elif procedure=="uploadAll":
! 111: uploadFns=[]
! 112: for x in os.listdir(tmp['tmpdir']):
! 113: if (not x in tmp['lockerrors']) and ( not x in errorFn):
! 114: uploadFns.append(x)
! 115:
! 116: #or maybe nothing
! 117: elif procedure=="noupload":
! 118: return True
! 119: else:
! 120: uploadFns=[]
! 121:
! 122: #do first the changed files
! 123: i=0
! 124: for fn in uploadFns:
! 125: logging.debug("uploadATFfinally uploadFn=%s"%fn)
! 126: i+=1
! 127: logging.debug(ctx2)
! 128: catalog = ctx2['CDLICatalog']
! 129: print catalog
! 130:
! 131: founds=catalog.search({'title':fn})
! 132: if len(founds)>0:
! 133: tmp['author']=str(username)
! 134: self.result.write("<p>Changing : %s\n"%fn)
! 135: logging.debug("uploadatffinallythread changing:%s"%fn)
! 136: founds[0].getObject().manage_addCDLIFileObject('',comment,tmp['author'],file=os.path.join(tmp['tmpdir'],fn),from_tmp=True)
! 137: if i%200==0:
! 138: #transaction.get().commit()
! 139: #logging.debug("uploadatffinallythread changing: do commit")
! 140: self.result.flush()
! 141:
! 142:
! 143: #now add the new files
! 144: newPs=tmp['newPs']
! 145: if len(newPs)>0:
! 146: tmpDir=tmp['tmpdir']
! 147: logging.debug("uploadatffinallythread adding start")
! 148: self.result.write("<p>Adding files</p>\n")
! 149: #TODO: make this configurable, at the moment base folder for the files has to be cdli_main
! 150: print ctx2
! 151: ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self)
! 152: logging.debug("uploadatffinallythread adding finished")
! 153:
! 154: #unlock locked files?
! 155: if unlock:
! 156: logging.debug("uploadatffinallythread unlocking start")
! 157: self.result.write("<p>Unlock files</p>\n")
! 158: unlockFns=[]
! 159: for x in os.listdir(tmp['tmpdir']):
! 160: if not x in errorFn:
! 161: unlockFns.append(x)
! 162:
! 163: logging.debug("unlocking have now what to unlock")
! 164:
! 165: for fn in unlockFns:
! 166: #logging.info("will unlock: %s"%fn)
! 167: founds=ctx2.CDLICatalog.search({'title':fn})
! 168: #logging.info("found it: %s"%repr(founds))
! 169: if len(founds)>0:
! 170: #logging.info("unlock: %s"%founds[0].getObject().getId())
! 171: tmp['author']=str(username)
! 172: founds[0].getObject().lockedBy=""
! 173:
! 174: logging.debug("uploadatffinallythread unlocking done")
! 175:
! 176: #if a basketname is given, add files to the basket
! 177: if not (basketname ==''):
! 178: logging.debug("uploadatffinallythread add to basket %s"%basketname)
! 179: self.result.write("<p>Add to basket</p>\n")
! 180: basketId=ctx2.basketContainer.getBasketIdfromName(basketname)
! 181:
! 182: if not basketId: # create new basket
! 183: logging.debug("uploadatffinallythread create basket %s"%basketname)
! 184: self.result.write("<p>Create a new basket</p>\n")
! 185: ob=ctx2.basketContainer.addBasket(basketname)
! 186: basketId=ob.getId()
! 187: basket=getattr(ctx2.basketContainer,str(basketId))
! 188:
! 189: #workaround fuer memory fehler in listdir
! 190: #http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir
! 191: dirTmp=tmp['tmpdir']
! 192: ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n')
! 193: ids.remove(dirTmp)
! 194: n = len(dirTmp)
! 195: if dirTmp[-1] != os.path.sep:
! 196: n += 1
! 197: ids = [f[n:] for f in ids] # remove dir prefix
! 198: #ids=os.listdir(tmp['tmpdir'])
! 199: #logging.debug("should add:"+repr(ids))
! 200: addIds=[]
! 201: for id in ids:
! 202: if not (id in errorFn):
! 203: addIds.append(id)
! 204:
! 205: basket.addObjects(addIds,deleteOld=True,username=str(username))
! 206:
! 207: logging.debug("uploadatffinallythread uploadfinally done")
! 208:
! 209:
! 210: return True
! 211:
! 212: if __name__ == "__main__":
! 213: if len(sys.argv)<7:
! 214: print """Usage: procedure uploadId comment basketName unlock username
! 215:
! 216: procedure: can be either uploadchanged or uploadall
! 217: uploadId: Ticket ID von uploadATF
! 218: comment: will be added to all docs
! 219: basketName: name of the basket
! 220: unlock: unlock or keeplock
! 221: """
! 222:
! 223:
! 224: #set(self,procedure,uploadId,comment="",basketname='',unlock=None,username=None,serverport="8080"):
! 225: upload = uploadATFfinallyThread()
! 226: x=sys.argv;
! 227: if sys.argv[5]=="unlock":
! 228: unlock=True
! 229: else:
! 230: unlock=False
! 231:
! 232: upload.set(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],unlock=[5],username=sys.argv[6])
! 233: upload.run();
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/cvs/cvsweb/icons/back.gif){kind=icon}
Return to uploadATFAsyncFinally.py CVS log ![[TXT]](/cvs/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvs/cvsweb/icons/dir.gif){kind=icon}
Up to [Repository] / cdli / tools