from ZODB import DB
from ZEO import ClientStorage
from ZPublisher.HTTPResponse import HTTPResponse
from ZPublisher.HTTPRequest import HTTPRequest
from ZPublisher.BaseRequest import RequestContainer
from ZCatalog import ZCatalog
import pickle
import logging
from Products.cdli.cdli_files import CDLIRoot
import sys
import os
import os.path
import transaction
class uploadATFfinallyThread:
"""class for adding uploaded filed (temporarily stored in the staging area at /tmp"""
def __init__(self):
"""init for uploadATFfinallyThread"""
logging.getLogger().setLevel(logging.DEBUG)
def set(self,procedure,uploadId,comment="",basketname='',unlock=None,username=None,serverport="29080"):
"""set start values for the thread"""
self.result=file("/tmp/"+str(basketname)+".out","w")
self.procedure=procedure
self.comment=comment
self.basketname=basketname
self.unlock=unlock
self.username=username
self.serverport=serverport
self.uploadId=uploadId
def __call__(self):
"""call of the thread (equals run)"""
self.run()
return True
def getContext(self, app,serverport="29080"):
"""get the context within the ZODB"""
resp = HTTPResponse(stdout=None)
env = {
'SERVER_NAME':'localhost',
'SERVER_PORT':serverport,
'REQUEST_METHOD':'GET'
}
req = HTTPRequest(None, env, resp)
return app.__of__(RequestContainer(REQUEST = req))
def run(self):
"""run"""
from App.config import getConfiguration
print getConfiguration()
#find context within ZODB
#find context within ZODB
storage=ClientStorage.ClientStorage(("localhost",8100));
db = DB(storage)
conn = db.open()
root = conn.root()
app = root['Application']
ctx = self.getContext(app,serverport=self.serverport)
#add the files
self.uploadATFfinallyThread(ctx,self.procedure,comment=self.comment,basketname=self.basketname,unlock=self.unlock,username=self.username)
#commit the transactions
db.close()
return True
def __del__(self):
"""delete"""
def getResult(self):
"""method for accessing result"""
ret=""
for x in self.result.readlines():
ret+=x
return ret
def uploadATFfinallyThread(self,ctx,procedure,comment="",basketname='',unlock=None,username=None):
"""upload the files"""
#TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot
logging.debug(ctx)
ctx2a=ctx.cdliRoot
ctx2 = self.getContext(ctx2a,serverport=self.serverport)
self.result.write("<h2>Start processing</h2>\n")
tmp = pickle.load(file("/tmp/"+str(self.uploadId)+".result"))
errorFn = [x[0] for x in tmp['errors']]
#shall I only upload the changed files?
logging.debug("uploadATFfinally procedure: %s"%procedure)
if procedure.lower()=="uploadchanged":
changedTmp=[x[0] for x in tmp.get('changed',[])]
uploadFns=[]
uploadFnsTmp=changedTmp+tmp.get('newPs',[])
for x in uploadFnsTmp:
if (not x in tmp['lockerrors']) and ( not x in errorFn):
uploadFns.append(x)
#or all
elif procedure.lower()=="uploadall":
uploadFns=[]
#workaround fuer memory fehler in listdir
#http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir
dirTmp=tmp['tmpdir']
ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n')
ids.remove(dirTmp)
n = len(dirTmp)
if dirTmp[-1] != os.path.sep:
n += 1
ids = [f[n:] for f in ids] # remove dir prefix
for x in ids:
if (not x in tmp['lockerrors']) and ( not x in errorFn):
uploadFns.append(x)
#or maybe nothing
elif procedure=="noupload":
return True
else:
uploadFns=[]
#do first the changed files
i=0
for fn in uploadFns:
logging.debug("uploadATFfinally uploadFn=%s"%fn)
i+=1
logging.debug(ctx2)
catalog = ctx2['CDLICatalog']
print catalog
founds=catalog.search({'title':fn})
if len(founds)>0:
tmp['author']=str(username)
self.result.write("<p>Changing : %s\n"%fn)
logging.debug("uploadatffinallythread changing:%s"%fn)
founds[0].getObject().manage_addCDLIFileObject('',comment,tmp['author'],file=os.path.join(tmp['tmpdir'],fn),from_tmp=True)
if i%200==0:
transaction.get().commit()
logging.debug("uploadatffinallythread changing: do commit")
self.result.flush()
#now add the new files
newPs=tmp['newPs']
if len(newPs)>0:
tmpDir=tmp['tmpdir']
logging.debug("uploadatffinallythread adding start")
self.result.write("<p>Adding files</p>\n")
#TODO: make this configurable, at the moment base folder for the files has to be cdli_main
print ctx2
ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self)
logging.debug("uploadatffinallythread adding finished")
#unlock locked files?
if unlock:
logging.debug("uploadatffinallythread unlocking start")
self.result.write("<p>Unlock files</p>\n")
unlockFns=[]
#workaround fuer memory fehler in listdir
#http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir
dirTmp=tmp['tmpdir']
ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n')
ids.remove(dirTmp)
n = len(dirTmp)
if dirTmp[-1] != os.path.sep:
n += 1
ids = [f[n:] for f in ids] # remove dir prefix
for x in ids:
if not x in errorFn:
unlockFns.append(x)
logging.debug("unlocking have now what to unlock")
for fn in unlockFns:
#logging.info("will unlock: %s"%fn)
founds=ctx2.CDLICatalog.search({'title':fn})
#logging.info("found it: %s"%repr(founds))
if len(founds)>0:
#logging.info("unlock: %s"%founds[0].getObject().getId())
tmp['author']=str(username)
founds[0].getObject().lockedBy=""
logging.debug("uploadatffinallythread unlocking done")
#if a basketname is given, add files to the basket
if not (basketname ==''):
logging.debug("uploadatffinallythread add to basket %s"%basketname)
self.result.write("<p>Add to basket</p>\n")
basketId=ctx2.basketContainer.getBasketIdfromName(basketname)
if not basketId: # create new basket
logging.debug("uploadatffinallythread create basket %s"%basketname)
self.result.write("<p>Create a new basket</p>\n")
ob=ctx2.basketContainer.addBasket(basketname)
basketId=ob.getId()
basket=getattr(ctx2.basketContainer,str(basketId))
#workaround fuer memory fehler in listdir
#http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir
dirTmp=tmp['tmpdir']
ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n')
ids.remove(dirTmp)
n = len(dirTmp)
if dirTmp[-1] != os.path.sep:
n += 1
ids = [f[n:] for f in ids] # remove dir prefix
#ids=os.listdir(tmp['tmpdir'])
#logging.debug("should add:"+repr(ids))
addIds=[]
for id in ids:
if not (id in errorFn):
addIds.append(id)
basket.addObjects(addIds,deleteOld=True,username=str(username))
transaction.get().commit()
logging.debug("uploadatffinallythread uploadfinally done")
return True
if __name__ == "__main__":
if len(sys.argv)<7:
print """Usage: procedure uploadId comment basketName unlock username
procedure: can be either uploadchanged or uploadall
uploadId: Ticket ID von uploadATF
comment: will be added to all docs
basketName: name of the basket
unlock: unlock or keeplock
"""
#set(self,procedure,uploadId,comment="",basketname='',unlock=None,username=None,serverport="8080"):
upload = uploadATFfinallyThread()
x=sys.argv;
if sys.argv[5]=="unlock":
unlock=True
else:
unlock=False
upload.set(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],unlock=[5],username=sys.argv[6])
upload.run();
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>