import sys from ZODB import DB from ZEO import ClientStorage from ZPublisher.HTTPResponse import HTTPResponse from ZPublisher.HTTPRequest import HTTPRequest from ZPublisher.BaseRequest import RequestContainer import logging import os import os.path import tempfile import codecs from Products.cdli.cdli_files import CDLIRoot import pickle class StoreObject: returnValue={} def __init__(self,storeId): self.storeId= storeId def save(self): pf = file("/tmp/"+self.storeId+".result","w") pickle.dump(self.returnValue,pf) pf.close() class uploadATFThread: """class for checking the files befor uploading""" def checkFile(self,filename,data,folder): """check the files""" # first check the file name fn=filename.split(".") # no extension if not (fn[0][0]=="P" or fn[0][0]=="S"): return False,"P/S missing in the filename" elif len(fn[0])!=7: return False,"P number has not the right length 6" elif not checkUTF8(data): return False,"not utf-8" else: return True,"" def splitatf(self,fh,dir=None,ext=None): """split it""" ret=None nf=None i=0 fileCount=0 #ROC: why split \n first and then \r??? if isinstance(fh, basestring): iter=fh.split("\n") else: iter=fh.readlines() for lineTmp in iter: #if fileCount>10: # break; lineTmp=lineTmp.replace(codecs.BOM_UTF8,'') # make sure that all BOM are removed.. for line in lineTmp.split("\r"): #logging.info("Deal with: %s"%line) i+=1 if (i%100)==0: self.result.write(str(i)+"\n") self.result.flush() #check if basket name is in the first line if line.find("#atf basket")>=0: #old convention ret=line.replace('#atf basket ','') ret=ret.split('_')[0] elif line.find("#basket:")>=0: #new convention ret=line.replace('#basket: ','') ret=ret.split('_')[0] else: if (len(line.lstrip())>0) and (line.lstrip()[0]=="&"): #newfile if nf: fileCount+=1 print fileCount nf.close() #close file #if fileCount>10: # break filename=line[1:].split("=")[0].rstrip()+".atf" if dir: filename=os.path.join(dir,filename) nf=file(filename,"w") logging.debug("open %s"%filename) if nf: nf.write(line.replace("\n","")+"\n") try: nf.close() except: pass if not isinstance(fh, basestring): fh.close() files = os.popen4('find %s' % dir)[1].read().rstrip().split('\n') files.remove(dir) return ret,len(files) def __init__(self): """initialise""" logging.getLogger().setLevel(logging.DEBUG) def set(self,uploadId,basketId,username,serverport="8080"): """set start values for the thread""" self.result=file("/tmp/"+str(uploadId)+".out","w") self.uploadId=uploadId self.basketId=basketId self.username=username self.serverport=serverport def __call__(self): """call method """ self.run() return True def getContext(self, app,serverport="8080"): """get the context within the ZODB""" resp = HTTPResponse(stdout=None) env = { 'SERVER_NAME':'localhost', 'SERVER_PORT':serverport, 'REQUEST_METHOD':'GET' } req = HTTPRequest(None, env, resp) return app.__of__(RequestContainer(REQUEST = req)) def run(self): #find context within ZODB storage=ClientStorage.ClientStorage(("localhost",8100)); db = DB(storage) conn = db.open() root = conn.root() app = root['Application'] ctx = self.getContext(app,serverport=self.serverport) logging.info("run intern") logging.info("call thread intern") self.uploadATFThread(ctx,self.uploadId,self.basketId) #ctx.cdliRoot.cdli_main.tmpStore2[self.getName()[0:]]=self.returnValue def getResult(self): """method for accessing result""" ret="" for x in self.result.readlines(): ret+=x return ret def uploadATFThread(self,ctx,uploadId,basketId=0): """upload an atf file""" #TODO: add comments #TODO: finish uploadATF logging.info("start, upload thread") self.result.write("
\n") #make sure that id is a string and not an integer basketId=str(basketId) logging.info("basketID:"+basketId) #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot ctx2=ctx.cdliRoot #get temporary file for staging the downloaded and splitted files dir=tempfile.mkdtemp() logging.info("tmpfFile:"+str(dir)) changed=[] # changed files errors=[] # files with errors lockerrors=[] # files with errors newPs=[] # new p filed psNotInCatalog=[] # files not in the catalog #split the uploadedd atf file logging.info("start splitting") basketNameFromFile,numberOfFiles=self.splitatf(file("/tmp/"+uploadId,'r'),dir) #find basketId if not set #get active abaket if basketId == '0': print ctx2 basketObj=ctx2.basketContainer.getActiveBasket() if basketObj: basketId=basketObj.getId() #if there is no active basket and no basketid given, id is empty, else get besketname and length if basketId == '0': basketNameFromId="" basketLen=0 else: basketNameFromId=getattr(ctx2.basketContainer,basketId).title basketLen=getattr(ctx2.basketContainer,basketId).getLastVersion().numberOfItems() logging.info("got the file, upload thread") self.result.write("""
process:%s
\n"%fn) logging.debug(fn) # check if file is in the catalog #TODO: checkCatalog is not implemented yet if ctx2.cdli_main.checkCatalog(fn): psNotInCatalog.append(fn) #check if p-file already at the server founds=ctx2.CDLICatalog.search({'title':fn}) #if not than add filename to the list of newfiles dataFile=file(os.path.join(dir,fn)) data=dataFile.read() dataFile.close() status,msg=self.checkFile(fn,data,dir) #status=True if not status: # error errors.append((fn,msg)) else: if len(founds)==0: newPs.append(fn) #if p file alread at the server for found in founds: #analyse the differences to the actual file obj=found.getObject() if (not (str(obj.lockedBy))=='') and (not (str(obj.lockedBy)==str(self.username))): lockerrors.append((fn,str(obj.lockedBy))) else: diffs=obj.diff(data) if diffs[0]>0: changed.append((obj,diffs)) #hochladen #ready, set the returnValues self.result.write("