Return to uploadATFAsyncFinally.py CVS log

Up to [Repository] / cdli / tools

cli tools in tools verschoben,

neue develop-verisonen von cdli_basket und files im devel ordner

    1: from ZODB import DB
    2: from ZEO import ClientStorage
    3: from ZPublisher.HTTPResponse import HTTPResponse
    4: from ZPublisher.HTTPRequest import HTTPRequest
    5: from ZPublisher.BaseRequest import RequestContainer
    6: 
    7: from ZCatalog import ZCatalog
    8: import pickle
    9: import logging
   10: from Products.cdli.cdli_files import CDLIRoot
   11: import sys
   12: import os
   13: import os.path
   14: import transaction
   15: 
   16: class uploadATFfinallyThread:
   17:     """class for adding uploaded filed (temporarily stored in the staging area at /tmp"""
   18:     
   19:     def __init__(self):
   20:         """init for uploadATFfinallyThread"""
   21:         logging.getLogger().setLevel(logging.DEBUG)
   22:                   
   23:     def set(self,procedure,uploadId,comment="",basketname='',unlock=None,username=None,serverport="29080"):
   24:         """set start values for the thread"""
   25:         
   26:         self.result=file("/tmp/"+str(basketname)+".out","w")
   27:         self.procedure=procedure
   28:         self.comment=comment
   29:         self.basketname=basketname
   30:         self.unlock=unlock
   31:         self.username=username
   32:         self.serverport=serverport
   33:         self.uploadId=uploadId
   34:         
   35:     def __call__(self):
   36:         """call of the thread (equals run)"""
   37:         self.run()
   38:         return True
   39:     
   40:     def getContext(self, app,serverport="29080"):
   41:         """get the context within the ZODB"""
   42:         
   43:         resp = HTTPResponse(stdout=None)
   44:         env = {
   45:             'SERVER_NAME':'localhost',
   46:             'SERVER_PORT':serverport,
   47:             'REQUEST_METHOD':'GET'
   48:             }
   49:         req = HTTPRequest(None, env, resp)
   50:         return app.__of__(RequestContainer(REQUEST = req))
   51:           
   52:         
   53:     def run(self):
   54:         """run"""
   55:         from App.config import getConfiguration
   56:         print getConfiguration()
   57:         
   58:         #find context within ZODB
   59:        #find context within ZODB
   60:         storage=ClientStorage.ClientStorage(("localhost",8100));
   61:         db = DB(storage)
   62:         conn = db.open()
   63:       
   64:         root = conn.root()
   65:         app  = root['Application']
   66:         ctx = self.getContext(app,serverport=self.serverport)
   67: 
   68:         #add the files
   69:         self.uploadATFfinallyThread(ctx,self.procedure,comment=self.comment,basketname=self.basketname,unlock=self.unlock,username=self.username)
   70:         #commit the transactions
   71:         
   72:         db.close()
   73:         return True
   74:     
   75:     def __del__(self):
   76:         """delete"""
   77:         
   78:         
   79:     
   80:     def getResult(self):
   81:         """method for accessing result"""
   82:         ret=""
   83:         for x in self.result.readlines():
   84:             ret+=x
   85:             
   86:         return ret
   87:     
   88:     def uploadATFfinallyThread(self,ctx,procedure,comment="",basketname='',unlock=None,username=None):
   89:         """upload the files"""
   90:         #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot
   91:         logging.debug(ctx)
   92:         ctx2a=ctx.cdliRoot
   93:         ctx2 = self.getContext(ctx2a,serverport=self.serverport)
   94: 
   95:         
   96:         self.result.write("<h2>Start processing</h2>\n")
   97:     
   98:         tmp = pickle.load(file("/tmp/"+str(self.uploadId)+".result"))
   99:         errorFn = [x[0] for x in tmp['errors']]
  100:         #shall I only upload the changed files?
  101:         logging.debug("uploadATFfinally procedure: %s"%procedure)
  102:         if procedure.lower()=="uploadchanged":
  103:             changedTmp=[x[0] for x in tmp.get('changed',[])]
  104:             uploadFns=[]
  105:            
  106:             uploadFnsTmp=changedTmp+tmp.get('newPs',[])
  107:             for x in uploadFnsTmp:
  108:                 if (not x in tmp['lockerrors']) and ( not x in errorFn):
  109:                     uploadFns.append(x)
  110:         #or all
  111:         elif procedure.lower()=="uploadall":
  112:             uploadFns=[]
  113:             #workaround fuer memory fehler in listdir 
  114:             #http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir
  115:             dirTmp=tmp['tmpdir']
  116:             ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n')
  117:             ids.remove(dirTmp)
  118:             n = len(dirTmp)
  119:             if dirTmp[-1] != os.path.sep:
  120:                 n += 1
  121:                 ids = [f[n:] for f in ids] # remove dir prefix
  122:             for x in ids:
  123:                 if (not x in tmp['lockerrors']) and ( not x in errorFn):
  124:                     uploadFns.append(x)
  125:                     
  126:         #or maybe nothing
  127:         elif procedure=="noupload":
  128:             return True
  129:         else:
  130:             uploadFns=[]
  131:             
  132:         #do first the changed files    
  133:         i=0
  134:         for fn in uploadFns:
  135:             logging.debug("uploadATFfinally uploadFn=%s"%fn)
  136:             i+=1
  137:             logging.debug(ctx2)
  138:             catalog = ctx2['CDLICatalog']
  139:             print catalog
  140:             
  141:             founds=catalog.search({'title':fn})
  142:             if len(founds)>0:
  143:                 tmp['author']=str(username)
  144:                 self.result.write("<p>Changing : %s\n"%fn)
  145:                 logging.debug("uploadatffinallythread changing:%s"%fn)
  146:                 founds[0].getObject().manage_addCDLIFileObject('',comment,tmp['author'],file=os.path.join(tmp['tmpdir'],fn),from_tmp=True)
  147:             if i%200==0:
  148:                 transaction.get().commit()
  149:                 logging.debug("uploadatffinallythread changing: do commit")
  150:                 self.result.flush()
  151:         
  152:      
  153:         #now add the new files        
  154:         newPs=tmp['newPs']
  155:         if len(newPs)>0:
  156:             tmpDir=tmp['tmpdir']
  157:             logging.debug("uploadatffinallythread adding start")
  158:             self.result.write("<p>Adding files</p>\n")
  159:             #TODO: make this configurable, at the moment base folder for the files has to be cdli_main
  160:             print ctx2
  161:             ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self)
  162:             logging.debug("uploadatffinallythread adding finished")
  163:         
  164:         #unlock locked files?
  165:         if unlock:
  166:             logging.debug("uploadatffinallythread unlocking start")
  167:             self.result.write("<p>Unlock files</p>\n")
  168:             unlockFns=[]
  169:             
  170:             #workaround fuer memory fehler in listdir 
  171:             #http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir
  172:             dirTmp=tmp['tmpdir']
  173:             ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n')
  174:             ids.remove(dirTmp)
  175:             n = len(dirTmp)
  176:             if dirTmp[-1] != os.path.sep:
  177:                 n += 1
  178:                 ids = [f[n:] for f in ids] # remove dir prefix
  179:             for x in ids:
  180:                     if not x in errorFn:
  181:                         unlockFns.append(x)
  182:                         
  183:             logging.debug("unlocking have now what to unlock")
  184:                         
  185:             for fn in unlockFns:
  186:                 #logging.info("will unlock: %s"%fn)
  187:                 founds=ctx2.CDLICatalog.search({'title':fn})
  188:                 #logging.info("found it: %s"%repr(founds))
  189:                 if len(founds)>0:
  190:                     #logging.info("unlock: %s"%founds[0].getObject().getId())
  191:                     tmp['author']=str(username)
  192:                     founds[0].getObject().lockedBy=""
  193: 
  194:             logging.debug("uploadatffinallythread unlocking done")
  195:                     
  196:         #if a basketname is given, add files to the basket
  197:         if not (basketname ==''):
  198:             logging.debug("uploadatffinallythread add to basket %s"%basketname)
  199:             self.result.write("<p>Add to basket</p>\n")
  200:             basketId=ctx2.basketContainer.getBasketIdfromName(basketname)
  201:             
  202:             if not basketId: # create new basket
  203:                 logging.debug("uploadatffinallythread create basket %s"%basketname)
  204:                 self.result.write("<p>Create a new basket</p>\n")
  205:                 ob=ctx2.basketContainer.addBasket(basketname)
  206:                 basketId=ob.getId()
  207:             basket=getattr(ctx2.basketContainer,str(basketId))
  208:             
  209:             #workaround fuer memory fehler in listdir 
  210:             #http://stackoverflow.com/questions/4098831/workaround-oserror-with-os-listdir
  211:             dirTmp=tmp['tmpdir']
  212:             ids = os.popen4('find %s' % dirTmp)[1].read().rstrip().split('\n')
  213:             ids.remove(dirTmp)
  214:             n = len(dirTmp)
  215:             if dirTmp[-1] != os.path.sep:
  216:                 n += 1
  217:                 ids = [f[n:] for f in ids] # remove dir prefix
  218:             #ids=os.listdir(tmp['tmpdir'])
  219:             #logging.debug("should add:"+repr(ids))
  220:             addIds=[]
  221:             for id in ids:
  222:                 if not (id in errorFn):
  223:                     addIds.append(id)
  224:                 
  225:             basket.addObjects(addIds,deleteOld=True,username=str(username))    
  226:         
  227:         transaction.get().commit()   
  228:         logging.debug("uploadatffinallythread uploadfinally done")
  229: 
  230:        
  231:         return True
  232: 
  233: if __name__ == "__main__":
  234:        if len(sys.argv)<7:
  235:            print """Usage: procedure uploadId comment basketName unlock username
  236:            
  237:            procedure: can be either uploadchanged or uploadall
  238:            uploadId: Ticket ID von uploadATF
  239:            comment: will be added to all docs
  240:            basketName: name of the basket
  241:            unlock: unlock or keeplock
  242:            """
  243:         
  244:             
  245:        #set(self,procedure,uploadId,comment="",basketname='',unlock=None,username=None,serverport="8080"):
  246:        upload = uploadATFfinallyThread()
  247:        x=sys.argv;
  248:        if sys.argv[5]=="unlock":
  249:            unlock=True
  250:        else:
  251:            unlock=False
  252:             
  253:        upload.set(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],unlock=[5],username=sys.argv[6])
  254:        upload.run();