Annotation of cdli/tools/uploadATFAsyncFinally.py, revision 1.2
1.1 dwinter 1: from ZODB import DB
2: from ZEO import ClientStorage
3: from ZPublisher.HTTPResponse import HTTPResponse
4: from ZPublisher.HTTPRequest import HTTPRequest
5: from ZPublisher.BaseRequest import RequestContainer
6:
7: from ZCatalog import ZCatalog
8: import pickle
9: import logging
10: from Products.cdli.cdli_files import CDLIRoot
11: import sys
12: import os
13: import os.path
1.2 ! dwinter 14: import transaction
1.1 dwinter 15:
16: class uploadATFfinallyThread:
17: """class for adding uploaded filed (temporarily stored in the staging area at /tmp"""
18:
19: def __init__(self):
20: """init for uploadATFfinallyThread"""
21: logging.getLogger().setLevel(logging.DEBUG)
22:
1.2 ! dwinter 23: def set(self,procedure,uploadId,comment="",basketname='',unlock=None,username=None,serverport="29080"):
1.1 dwinter 24: """set start values for the thread"""
25:
26: self.result=file("/tmp/"+str(basketname)+".out","w")
27: self.procedure=procedure
28: self.comment=comment
29: self.basketname=basketname
30: self.unlock=unlock
31: self.username=username
32: self.serverport=serverport
33: self.uploadId=uploadId
34:
35: def __call__(self):
36: """call of the thread (equals run)"""
37: self.run()
38: return True
39:
1.2 ! dwinter 40: def getContext(self, app,serverport="29080"):
1.1 dwinter 41: """get the context within the ZODB"""
42:
43: resp = HTTPResponse(stdout=None)
44: env = {
45: 'SERVER_NAME':'localhost',
46: 'SERVER_PORT':serverport,
47: 'REQUEST_METHOD':'GET'
48: }
49: req = HTTPRequest(None, env, resp)
50: return app.__of__(RequestContainer(REQUEST = req))
51:
52:
53: def run(self):
54: """run"""
55: from App.config import getConfiguration
56: print getConfiguration()
57:
58: #find context within ZODB
59: #find context within ZODB
60: storage=ClientStorage.ClientStorage(("localhost",8100));
61: db = DB(storage)
62: conn = db.open()
63:
64: root = conn.root()
65: app = root['Application']
66: ctx = self.getContext(app,serverport=self.serverport)
67:
68: #add the files
69: self.uploadATFfinallyThread(ctx,self.procedure,comment=self.comment,basketname=self.basketname,unlock=self.unlock,username=self.username)
70: #commit the transactions
71:
72: db.close()
73: return True
74:
75: def __del__(self):
76: """delete"""
77:
78:
79:
80: def getResult(self):
81: """method for accessing result"""
82: ret=""
83: for x in self.result.readlines():
84: ret+=x
85:
86: return ret
87:
88: def uploadATFfinallyThread(self,ctx,procedure,comment="",basketname='',unlock=None,username=None):
89: """upload the files"""
90: #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot
91: logging.debug(ctx)
92: ctx2a=ctx.cdliRoot
93: ctx2 = self.getContext(ctx2a,serverport=self.serverport)
94:
95:
96: self.result.write("<h2>Start processing</h2>\n")
97:
98: tmp = pickle.load(file("/tmp/"+str(self.uploadId)+".result"))
99: errorFn = [x[0] for x in tmp['errors']]
100: #shall I only upload the changed files?
101: logging.debug("uploadATFfinally procedure: %s"%procedure)
1.2 ! dwinter 102: if procedure.lower()=="uploadchanged":
1.1 dwinter 103: changedTmp=[x[0] for x in tmp.get('changed',[])]
104: uploadFns=[]
105:
106: uploadFnsTmp=changedTmp+tmp.get('newPs',[])
107: for x in uploadFnsTmp:
108: if (not x in tmp['lockerrors']) and ( not x in errorFn):
109: uploadFns.append(x)
110: #or all
1.2 ! dwinter 111: elif procedure.lower()=="uploadall":
1.1 dwinter 112: uploadFns=[]
1.2 ! dwinter 113: #workaround fuer memory fehler in listdir
! 114: #http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir
! 115: dirTmp=tmp['tmpdir']
! 116: ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n')
! 117: ids.remove(dirTmp)
! 118: n = len(dirTmp)
! 119: if dirTmp[-1] != os.path.sep:
! 120: n += 1
! 121: ids = [f[n:] for f in ids] # remove dir prefix
! 122: for x in ids:
1.1 dwinter 123: if (not x in tmp['lockerrors']) and ( not x in errorFn):
124: uploadFns.append(x)
125:
126: #or maybe nothing
127: elif procedure=="noupload":
128: return True
129: else:
130: uploadFns=[]
131:
132: #do first the changed files
133: i=0
134: for fn in uploadFns:
135: logging.debug("uploadATFfinally uploadFn=%s"%fn)
136: i+=1
137: logging.debug(ctx2)
138: catalog = ctx2['CDLICatalog']
139: print catalog
140:
141: founds=catalog.search({'title':fn})
142: if len(founds)>0:
143: tmp['author']=str(username)
144: self.result.write("<p>Changing : %s\n"%fn)
145: logging.debug("uploadatffinallythread changing:%s"%fn)
146: founds[0].getObject().manage_addCDLIFileObject('',comment,tmp['author'],file=os.path.join(tmp['tmpdir'],fn),from_tmp=True)
147: if i%200==0:
1.2 ! dwinter 148: transaction.get().commit()
! 149: logging.debug("uploadatffinallythread changing: do commit")
1.1 dwinter 150: self.result.flush()
151:
152:
153: #now add the new files
154: newPs=tmp['newPs']
155: if len(newPs)>0:
156: tmpDir=tmp['tmpdir']
157: logging.debug("uploadatffinallythread adding start")
158: self.result.write("<p>Adding files</p>\n")
159: #TODO: make this configurable, at the moment base folder for the files has to be cdli_main
160: print ctx2
161: ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self)
162: logging.debug("uploadatffinallythread adding finished")
163:
164: #unlock locked files?
165: if unlock:
166: logging.debug("uploadatffinallythread unlocking start")
167: self.result.write("<p>Unlock files</p>\n")
168: unlockFns=[]
1.2 ! dwinter 169:
! 170: #workaround fuer memory fehler in listdir
! 171: #http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir
! 172: dirTmp=tmp['tmpdir']
! 173: ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n')
! 174: ids.remove(dirTmp)
! 175: n = len(dirTmp)
! 176: if dirTmp[-1] != os.path.sep:
! 177: n += 1
! 178: ids = [f[n:] for f in ids] # remove dir prefix
! 179: for x in ids:
1.1 dwinter 180: if not x in errorFn:
181: unlockFns.append(x)
182:
183: logging.debug("unlocking have now what to unlock")
184:
185: for fn in unlockFns:
186: #logging.info("will unlock: %s"%fn)
187: founds=ctx2.CDLICatalog.search({'title':fn})
188: #logging.info("found it: %s"%repr(founds))
189: if len(founds)>0:
190: #logging.info("unlock: %s"%founds[0].getObject().getId())
191: tmp['author']=str(username)
192: founds[0].getObject().lockedBy=""
193:
194: logging.debug("uploadatffinallythread unlocking done")
195:
196: #if a basketname is given, add files to the basket
197: if not (basketname ==''):
198: logging.debug("uploadatffinallythread add to basket %s"%basketname)
199: self.result.write("<p>Add to basket</p>\n")
200: basketId=ctx2.basketContainer.getBasketIdfromName(basketname)
201:
202: if not basketId: # create new basket
203: logging.debug("uploadatffinallythread create basket %s"%basketname)
204: self.result.write("<p>Create a new basket</p>\n")
205: ob=ctx2.basketContainer.addBasket(basketname)
206: basketId=ob.getId()
207: basket=getattr(ctx2.basketContainer,str(basketId))
208:
209: #workaround fuer memory fehler in listdir
210: #http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir
211: dirTmp=tmp['tmpdir']
212: ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n')
213: ids.remove(dirTmp)
214: n = len(dirTmp)
215: if dirTmp[-1] != os.path.sep:
216: n += 1
217: ids = [f[n:] for f in ids] # remove dir prefix
218: #ids=os.listdir(tmp['tmpdir'])
219: #logging.debug("should add:"+repr(ids))
220: addIds=[]
221: for id in ids:
222: if not (id in errorFn):
223: addIds.append(id)
224:
225: basket.addObjects(addIds,deleteOld=True,username=str(username))
1.2 ! dwinter 226:
! 227: transaction.get().commit()
1.1 dwinter 228: logging.debug("uploadatffinallythread uploadfinally done")
229:
230:
231: return True
232:
233: if __name__ == "__main__":
234: if len(sys.argv)<7:
235: print """Usage: procedure uploadId comment basketName unlock username
236:
237: procedure: can be either uploadchanged or uploadall
238: uploadId: Ticket ID von uploadATF
239: comment: will be added to all docs
240: basketName: name of the basket
241: unlock: unlock or keeplock
242: """
243:
244:
245: #set(self,procedure,uploadId,comment="",basketname='',unlock=None,username=None,serverport="8080"):
246: upload = uploadATFfinallyThread()
247: x=sys.argv;
248: if sys.argv[5]=="unlock":
249: unlock=True
250: else:
251: unlock=False
252:
253: upload.set(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],unlock=[5],username=sys.argv[6])
254: upload.run();
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/cvs/cvsweb/icons/back.gif){kind=icon}
Return to uploadATFAsyncFinally.py CVS log ![[TXT]](/cvs/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvs/cvsweb/icons/dir.gif){kind=icon}
Up to [Repository] / cdli / tools