Return to uploadATFAsyncFinally.py CVS log

Up to [Repository] / cdli / tools

cli tools in tools verschoben,

neue develop-verisonen von cdli_basket und files im devel ordner

from ZODB import DB
from ZEO import ClientStorage
from ZPublisher.HTTPResponse import HTTPResponse
from ZPublisher.HTTPRequest import HTTPRequest
from ZPublisher.BaseRequest import RequestContainer

from ZCatalog import ZCatalog
import pickle
import logging
from Products.cdli.cdli_files import CDLIRoot
import sys
import os
import os.path
import transaction

class uploadATFfinallyThread:
    """class for adding uploaded filed (temporarily stored in the staging area at /tmp"""
    
    def __init__(self):
        """init for uploadATFfinallyThread"""
        logging.getLogger().setLevel(logging.DEBUG)
                  
    def set(self,procedure,uploadId,comment="",basketname='',unlock=None,username=None,serverport="29080"):
        """set start values for the thread"""
        
        self.result=file("/tmp/"+str(basketname)+".out","w")
        self.procedure=procedure
        self.comment=comment
        self.basketname=basketname
        self.unlock=unlock
        self.username=username
        self.serverport=serverport
        self.uploadId=uploadId
        
    def __call__(self):
        """call of the thread (equals run)"""
        self.run()
        return True
    
    def getContext(self, app,serverport="29080"):
        """get the context within the ZODB"""
        
        resp = HTTPResponse(stdout=None)
        env = {
            'SERVER_NAME':'localhost',
            'SERVER_PORT':serverport,
            'REQUEST_METHOD':'GET'
            }
        req = HTTPRequest(None, env, resp)
        return app.__of__(RequestContainer(REQUEST = req))
          
        
    def run(self):
        """run"""
        from App.config import getConfiguration
        print getConfiguration()
        
        #find context within ZODB
       #find context within ZODB
        storage=ClientStorage.ClientStorage(("localhost",8100));
        db = DB(storage)
        conn = db.open()
      
        root = conn.root()
        app  = root['Application']
        ctx = self.getContext(app,serverport=self.serverport)

        #add the files
        self.uploadATFfinallyThread(ctx,self.procedure,comment=self.comment,basketname=self.basketname,unlock=self.unlock,username=self.username)
        #commit the transactions
        
        db.close()
        return True
    
    def __del__(self):
        """delete"""
        
        
    
    def getResult(self):
        """method for accessing result"""
        ret=""
        for x in self.result.readlines():
            ret+=x
            
        return ret
    
    def uploadATFfinallyThread(self,ctx,procedure,comment="",basketname='',unlock=None,username=None):
        """upload the files"""
        #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot
        logging.debug(ctx)
        ctx2a=ctx.cdliRoot
        ctx2 = self.getContext(ctx2a,serverport=self.serverport)

        
        self.result.write("<h2>Start processing</h2>\n")
    
        tmp = pickle.load(file("/tmp/"+str(self.uploadId)+".result"))
        errorFn = [x[0] for x in tmp['errors']]
        #shall I only upload the changed files?
        logging.debug("uploadATFfinally procedure: %s"%procedure)
        if procedure.lower()=="uploadchanged":
            changedTmp=[x[0] for x in tmp.get('changed',[])]
            uploadFns=[]
           
            uploadFnsTmp=changedTmp+tmp.get('newPs',[])
            for x in uploadFnsTmp:
                if (not x in tmp['lockerrors']) and ( not x in errorFn):
                    uploadFns.append(x)
        #or all
        elif procedure.lower()=="uploadall":
            uploadFns=[]
            #workaround fuer memory fehler in listdir 
            #http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir
            dirTmp=tmp['tmpdir']
            ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n')
            ids.remove(dirTmp)
            n = len(dirTmp)
            if dirTmp[-1] != os.path.sep:
                n += 1
                ids = [f[n:] for f in ids] # remove dir prefix
            for x in ids:
                if (not x in tmp['lockerrors']) and ( not x in errorFn):
                    uploadFns.append(x)
                    
        #or maybe nothing
        elif procedure=="noupload":
            return True
        else:
            uploadFns=[]
            
        #do first the changed files    
        i=0
        for fn in uploadFns:
            logging.debug("uploadATFfinally uploadFn=%s"%fn)
            i+=1
            logging.debug(ctx2)
            catalog = ctx2['CDLICatalog']
            print catalog
            
            founds=catalog.search({'title':fn})
            if len(founds)>0:
                tmp['author']=str(username)
                self.result.write("<p>Changing : %s\n"%fn)
                logging.debug("uploadatffinallythread changing:%s"%fn)
                founds[0].getObject().manage_addCDLIFileObject('',comment,tmp['author'],file=os.path.join(tmp['tmpdir'],fn),from_tmp=True)
            if i%200==0:
                transaction.get().commit()
                logging.debug("uploadatffinallythread changing: do commit")
                self.result.flush()
        
     
        #now add the new files        
        newPs=tmp['newPs']
        if len(newPs)>0:
            tmpDir=tmp['tmpdir']
            logging.debug("uploadatffinallythread adding start")
            self.result.write("<p>Adding files</p>\n")
            #TODO: make this configurable, at the moment base folder for the files has to be cdli_main
            print ctx2
            ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self)
            logging.debug("uploadatffinallythread adding finished")
        
        #unlock locked files?
        if unlock:
            logging.debug("uploadatffinallythread unlocking start")
            self.result.write("<p>Unlock files</p>\n")
            unlockFns=[]
            
            #workaround fuer memory fehler in listdir 
            #http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir
            dirTmp=tmp['tmpdir']
            ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n')
            ids.remove(dirTmp)
            n = len(dirTmp)
            if dirTmp[-1] != os.path.sep:
                n += 1
                ids = [f[n:] for f in ids] # remove dir prefix
            for x in ids:
                    if not x in errorFn:
                        unlockFns.append(x)
                        
            logging.debug("unlocking have now what to unlock")
                        
            for fn in unlockFns:
                #logging.info("will unlock: %s"%fn)
                founds=ctx2.CDLICatalog.search({'title':fn})
                #logging.info("found it: %s"%repr(founds))
                if len(founds)>0:
                    #logging.info("unlock: %s"%founds[0].getObject().getId())
                    tmp['author']=str(username)
                    founds[0].getObject().lockedBy=""

            logging.debug("uploadatffinallythread unlocking done")
                    
        #if a basketname is given, add files to the basket
        if not (basketname ==''):
            logging.debug("uploadatffinallythread add to basket %s"%basketname)
            self.result.write("<p>Add to basket</p>\n")
            basketId=ctx2.basketContainer.getBasketIdfromName(basketname)
            
            if not basketId: # create new basket
                logging.debug("uploadatffinallythread create basket %s"%basketname)
                self.result.write("<p>Create a new basket</p>\n")
                ob=ctx2.basketContainer.addBasket(basketname)
                basketId=ob.getId()
            basket=getattr(ctx2.basketContainer,str(basketId))
            
            #workaround fuer memory fehler in listdir 
            #http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir
            dirTmp=tmp['tmpdir']
            ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n')
            ids.remove(dirTmp)
            n = len(dirTmp)
            if dirTmp[-1] != os.path.sep:
                n += 1
                ids = [f[n:] for f in ids] # remove dir prefix
            #ids=os.listdir(tmp['tmpdir'])
            #logging.debug("should add:"+repr(ids))
            addIds=[]
            for id in ids:
                if not (id in errorFn):
                    addIds.append(id)
                
            basket.addObjects(addIds,deleteOld=True,username=str(username))    
        
        transaction.get().commit()   
        logging.debug("uploadatffinallythread uploadfinally done")

       
        return True

if __name__ == "__main__":
       if len(sys.argv)<7:
           print """Usage: procedure uploadId comment basketName unlock username
           
           procedure: can be either uploadchanged or uploadall
           uploadId: Ticket ID von uploadATF
           comment: will be added to all docs
           basketName: name of the basket
           unlock: unlock or keeplock
           """
        
            
       #set(self,procedure,uploadId,comment="",basketname='',unlock=None,username=None,serverport="8080"):
       upload = uploadATFfinallyThread()
       x=sys.argv;
       if sys.argv[5]=="unlock":
           unlock=True
       else:
           unlock=False
            
       upload.set(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],unlock=[5],username=sys.argv[6])
       upload.run();