1: from ZODB import DB
2: from ZEO import ClientStorage
3: from ZPublisher.HTTPResponse import HTTPResponse
4: from ZPublisher.HTTPRequest import HTTPRequest
5: from ZPublisher.BaseRequest import RequestContainer
6:
7: from ZCatalog import ZCatalog
8: import pickle
9: import logging
10: from Products.cdli.cdli_files import CDLIRoot
11: import sys
12: import os
13: import os.path
14:
15: class uploadATFfinallyThread:
16: """class for adding uploaded filed (temporarily stored in the staging area at /tmp"""
17:
18: def __init__(self):
19: """init for uploadATFfinallyThread"""
20: logging.getLogger().setLevel(logging.DEBUG)
21:
22: def set(self,procedure,uploadId,comment="",basketname='',unlock=None,username=None,serverport="8080"):
23: """set start values for the thread"""
24:
25: self.result=file("/tmp/"+str(basketname)+".out","w")
26: self.procedure=procedure
27: self.comment=comment
28: self.basketname=basketname
29: self.unlock=unlock
30: self.username=username
31: self.serverport=serverport
32: self.uploadId=uploadId
33:
34: def __call__(self):
35: """call of the thread (equals run)"""
36: self.run()
37: return True
38:
39: def getContext(self, app,serverport="8080"):
40: """get the context within the ZODB"""
41:
42: resp = HTTPResponse(stdout=None)
43: env = {
44: 'SERVER_NAME':'localhost',
45: 'SERVER_PORT':serverport,
46: 'REQUEST_METHOD':'GET'
47: }
48: req = HTTPRequest(None, env, resp)
49: return app.__of__(RequestContainer(REQUEST = req))
50:
51:
52: def run(self):
53: """run"""
54: from App.config import getConfiguration
55: print getConfiguration()
56:
57: #find context within ZODB
58: #find context within ZODB
59: storage=ClientStorage.ClientStorage(("localhost",8100));
60: db = DB(storage)
61: conn = db.open()
62:
63: root = conn.root()
64: app = root['Application']
65: ctx = self.getContext(app,serverport=self.serverport)
66:
67: #add the files
68: self.uploadATFfinallyThread(ctx,self.procedure,comment=self.comment,basketname=self.basketname,unlock=self.unlock,username=self.username)
69: #commit the transactions
70:
71: db.close()
72: return True
73:
74: def __del__(self):
75: """delete"""
76:
77:
78:
79: def getResult(self):
80: """method for accessing result"""
81: ret=""
82: for x in self.result.readlines():
83: ret+=x
84:
85: return ret
86:
87: def uploadATFfinallyThread(self,ctx,procedure,comment="",basketname='',unlock=None,username=None):
88: """upload the files"""
89: #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot
90: logging.debug(ctx)
91: ctx2a=ctx.cdliRoot
92: ctx2 = self.getContext(ctx2a,serverport=self.serverport)
93:
94:
95: self.result.write("<h2>Start processing</h2>\n")
96:
97: tmp = pickle.load(file("/tmp/"+str(self.uploadId)+".result"))
98: errorFn = [x[0] for x in tmp['errors']]
99: #shall I only upload the changed files?
100: logging.debug("uploadATFfinally procedure: %s"%procedure)
101: if procedure=="uploadchanged":
102: changedTmp=[x[0] for x in tmp.get('changed',[])]
103: uploadFns=[]
104:
105: uploadFnsTmp=changedTmp+tmp.get('newPs',[])
106: for x in uploadFnsTmp:
107: if (not x in tmp['lockerrors']) and ( not x in errorFn):
108: uploadFns.append(x)
109: #or all
110: elif procedure=="uploadAll":
111: uploadFns=[]
112: for x in os.listdir(tmp['tmpdir']):
113: if (not x in tmp['lockerrors']) and ( not x in errorFn):
114: uploadFns.append(x)
115:
116: #or maybe nothing
117: elif procedure=="noupload":
118: return True
119: else:
120: uploadFns=[]
121:
122: #do first the changed files
123: i=0
124: for fn in uploadFns:
125: logging.debug("uploadATFfinally uploadFn=%s"%fn)
126: i+=1
127: logging.debug(ctx2)
128: catalog = ctx2['CDLICatalog']
129: print catalog
130:
131: founds=catalog.search({'title':fn})
132: if len(founds)>0:
133: tmp['author']=str(username)
134: self.result.write("<p>Changing : %s\n"%fn)
135: logging.debug("uploadatffinallythread changing:%s"%fn)
136: founds[0].getObject().manage_addCDLIFileObject('',comment,tmp['author'],file=os.path.join(tmp['tmpdir'],fn),from_tmp=True)
137: if i%200==0:
138: #transaction.get().commit()
139: #logging.debug("uploadatffinallythread changing: do commit")
140: self.result.flush()
141:
142:
143: #now add the new files
144: newPs=tmp['newPs']
145: if len(newPs)>0:
146: tmpDir=tmp['tmpdir']
147: logging.debug("uploadatffinallythread adding start")
148: self.result.write("<p>Adding files</p>\n")
149: #TODO: make this configurable, at the moment base folder for the files has to be cdli_main
150: print ctx2
151: ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self)
152: logging.debug("uploadatffinallythread adding finished")
153:
154: #unlock locked files?
155: if unlock:
156: logging.debug("uploadatffinallythread unlocking start")
157: self.result.write("<p>Unlock files</p>\n")
158: unlockFns=[]
159: for x in os.listdir(tmp['tmpdir']):
160: if not x in errorFn:
161: unlockFns.append(x)
162:
163: logging.debug("unlocking have now what to unlock")
164:
165: for fn in unlockFns:
166: #logging.info("will unlock: %s"%fn)
167: founds=ctx2.CDLICatalog.search({'title':fn})
168: #logging.info("found it: %s"%repr(founds))
169: if len(founds)>0:
170: #logging.info("unlock: %s"%founds[0].getObject().getId())
171: tmp['author']=str(username)
172: founds[0].getObject().lockedBy=""
173:
174: logging.debug("uploadatffinallythread unlocking done")
175:
176: #if a basketname is given, add files to the basket
177: if not (basketname ==''):
178: logging.debug("uploadatffinallythread add to basket %s"%basketname)
179: self.result.write("<p>Add to basket</p>\n")
180: basketId=ctx2.basketContainer.getBasketIdfromName(basketname)
181:
182: if not basketId: # create new basket
183: logging.debug("uploadatffinallythread create basket %s"%basketname)
184: self.result.write("<p>Create a new basket</p>\n")
185: ob=ctx2.basketContainer.addBasket(basketname)
186: basketId=ob.getId()
187: basket=getattr(ctx2.basketContainer,str(basketId))
188:
189: #workaround fuer memory fehler in listdir
190: #http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir
191: dirTmp=tmp['tmpdir']
192: ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n')
193: ids.remove(dirTmp)
194: n = len(dirTmp)
195: if dirTmp[-1] != os.path.sep:
196: n += 1
197: ids = [f[n:] for f in ids] # remove dir prefix
198: #ids=os.listdir(tmp['tmpdir'])
199: #logging.debug("should add:"+repr(ids))
200: addIds=[]
201: for id in ids:
202: if not (id in errorFn):
203: addIds.append(id)
204:
205: basket.addObjects(addIds,deleteOld=True,username=str(username))
206:
207: logging.debug("uploadatffinallythread uploadfinally done")
208:
209:
210: return True
211:
212: if __name__ == "__main__":
213: if len(sys.argv)<7:
214: print """Usage: procedure uploadId comment basketName unlock username
215:
216: procedure: can be either uploadchanged or uploadall
217: uploadId: Ticket ID von uploadATF
218: comment: will be added to all docs
219: basketName: name of the basket
220: unlock: unlock or keeplock
221: """
222:
223:
224: #set(self,procedure,uploadId,comment="",basketname='',unlock=None,username=None,serverport="8080"):
225: upload = uploadATFfinallyThread()
226: x=sys.argv;
227: if sys.argv[5]=="unlock":
228: unlock=True
229: else:
230: unlock=False
231:
232: upload.set(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],unlock=[5],username=sys.argv[6])
233: upload.run();
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/cvs/cvsweb/icons/back.gif){kind=icon}
Return to uploadATFAsyncFinally.py CVS log ![[TXT]](/cvs/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvs/cvsweb/icons/dir.gif){kind=icon}
Up to [Repository] / cdli / tools