--- cdli/cdli_files.py 2007/02/05 15:23:46 1.63 +++ cdli/cdli_files.py 2007/03/19 18:34:01 1.67 @@ -25,6 +25,7 @@ from BTrees.OOBTree import OOBTree import logging import transaction import copy +import codecs def unique(s): """Return a list of the elements in s, but without duplicates. @@ -174,8 +175,8 @@ class uploadATFfinallyThread(Thread): conn.close() #set flag for end of this method self.end=True - logging.info("ended") - return True + logging.info("ended") + return True def __del__(self): """delete""" @@ -196,7 +197,7 @@ class uploadATFfinallyThread(Thread): #shall I only upload the changed files? if procedure=="uploadchanged": - changed=[x[0] for x in SESSION.get('changed',[])] + changed=[x[0] for x in SESSION.get('changed',[])] uploadFns=changed+SESSION.get('newPs',[]) #or all @@ -213,53 +214,64 @@ class uploadATFfinallyThread(Thread): uploadFns=[] #do first the changed files - i=0 + i=0 for fn in uploadFns: - i+=1 + i+=1 founds=ctx2.CDLICatalog.search({'title':fn}) if len(founds)>0: SESSION['author']=str(username) self.result="

Changing : %s"%fn+self.result founds[0].getObject().manage_addCDLIFileObject('',comment,SESSION['author'],file=os.path.join(SESSION['tmpdir'],fn),from_tmp=True) if i==200: - i=0 - transaction.get().commit() - logging.info("changing: do commit") - transaction.get().commit() - logging.info("changing: last commit") + i=0 + transaction.get().commit() + logging.info("changing: do commit") + + transaction.get().commit() + logging.info("changing: last commit") #now add the new files newPs=SESSION['newPs'] if len(newPs)>0: tmpDir=SESSION['tmpdir'] + logging.info("adding start") self.result="

Adding files

"+self.result #TODO: make this configurable, at the moment base folder for the files has to be cdli_main ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self) - + logging.info("adding finished") #unlock locked files? if unlock: + logging.info("unlocking start") self.result="

Unlock files

"+self.result unlockFns=[] for x in os.listdir(SESSION['tmpdir']): if not x in SESSION['errors']: unlockFns.append(x) - + logging.info("unlocking have now what to unlock") + for fn in unlockFns: + #logging.info("will unlock: %s"%fn) founds=ctx2.CDLICatalog.search({'title':fn}) + #logging.info("found it: %s"%repr(founds)) if len(founds)>0: + #logging.info("unlock: %s"%founds[0].getObject().getId()) SESSION['author']=str(username) founds[0].getObject().lockedBy="" + logging.info("unlocking done") - #if a basketname is give, add files to the basket + #if a basketname is given, add files to the basket if not (basketname ==''): - self.result="

Add basket

"+self.result + logging.info("add to basket %s"%basketname) + self.result="

Add to basket

"+self.result basketId=ctx2.basketContainer.getBasketIdfromName(basketname) if not basketId: # create new basket + logging.info("create basket %s"%basketname) + self.result="

Create a new basket

"+self.result ob=ctx2.basketContainer.addBasket(basketname) basketId=ob.getId() basket=getattr(ctx2.basketContainer,str(basketId)) @@ -270,7 +282,7 @@ class uploadATFfinallyThread(Thread): RESPONSE.redirect(self.aq_parent.absolute_url()) - + logging.info("uploadfinally done") return True class tmpStore(SimpleItem): @@ -338,7 +350,7 @@ class uploadATFThread(Thread): #ctx.cdliRoot.cdli_main.tmpStore2[self.getName()[0:]]=self.returnValue - + transaction.get().commit() conn.close() @@ -561,7 +573,7 @@ class CDLIBasketContainer(OrderedFolder) ret+= "BASKET:"+com+"\t"+user+"\t"+time+"\n" for x in values: ret+= x[0]+"\t"+x[1]+"\n" - return ret + return ret def getBaskets(self,sortField='title'): """get all baskets files""" @@ -636,11 +648,11 @@ class CDLIBasketContainer(OrderedFolder) def setActiveBasket(self,basketId,REQUEST=None): """store active basketId in a cookie""" self.REQUEST.RESPONSE.setCookie("CDLIActiveBasket",basketId,path="/") - try: - qs=cgi.parse_qs(REQUEST['QUERY_STRING']) - del(qs['basketId']) - except: - qs={} + try: + qs=cgi.parse_qs(REQUEST['QUERY_STRING']) + del(qs['basketId']) + except: + qs={} if REQUEST: REQUEST.RESPONSE.redirect(REQUEST['URL1']+'?'+urllib.urlencode(qs)) @@ -832,7 +844,7 @@ class CDLIBasket(Folder,CatalogAware): def addObjects(self,ids,deleteOld=None,username=None): """generate a new version of the basket with objects added""" - + logging.info("add to basket (%s)"%(self.getId())) lastVersion=self.getLastVersion() if lastVersion is None: @@ -846,10 +858,11 @@ class CDLIBasket(Folder,CatalogAware): newContent=[] added=0 for id in ids: - try: - founds=self.CDLICatalog.search({'title':id}) - except: - founds=[] + try: + founds=self.CDLICatalog.search({'title':id}) + except: + founds=[] + for found in founds: if found.getObject() not in oldContent: #TODO: was passiert wenn, man eine Object dazufŸgt, das schon da ist aber eine neuere version @@ -863,7 +876,7 @@ class CDLIBasket(Folder,CatalogAware): user = username ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=content) - + logging.info("add to basket (%s) done"%(self.getId())) return added @@ -1157,7 +1170,7 @@ class CDLIFileObject(CatalogAware,extVer newversion=parent.manage_addCDLIFileObject('',comment,author) - newversion.data=self.data[0:] + newversion.manage_upload(self.getData()) if RESPONSE is not None: RESPONSE.redirect(self.aq_parent.absolute_url()+'/history') @@ -1437,40 +1450,42 @@ def splitatf(fh,dir=None,ext=None): i=0 for lineTmp in fh.readlines(): - for line in lineTmp.split("\r"): - if ext: - i+=1 - if (i%100)==0: - ext.result+="." - if i==10000: - i=0 - ext.result+="
" - #check if basket name is in the first line - if line.find("#atf basket")>=0: #old convention - ret=line.replace('#atf basket ','') - ret=ret.split('_')[0] - elif line.find("#basket:")>=0: #new convention - ret=line.replace('#basket: ','') - ret=ret.split('_')[0] - - else: - if (len(line.lstrip())>0) and (line.lstrip()[0]=="&"): #newfile - if nf: - nf.close() #close last file - - - filename=line[1:].split("=")[0].rstrip()+".atf" - if dir: - filename=os.path.join(dir,filename) - nf=file(filename,"w") - logging.info("open %s"%filename) - if nf: - nf.write(line.replace("\n","")+"\n") + lineTmp=lineTmp.replace(codecs.BOM_UTF8,'') # make sure that all BOM are removed.. + for line in lineTmp.split("\r"): + #logging.log("Deal with: %s"%line) + if ext: + i+=1 + if (i%100)==0: + ext.result+="." + if i==10000: + i=0 + ext.result+="
" + #check if basket name is in the first line + if line.find("#atf basket")>=0: #old convention + ret=line.replace('#atf basket ','') + ret=ret.split('_')[0] + elif line.find("#basket:")>=0: #new convention + ret=line.replace('#basket: ','') + ret=ret.split('_')[0] - try: - nf.close() + else: + if (len(line.lstrip())>0) and (line.lstrip()[0]=="&"): #newfile + if nf: + nf.close() #close last file + + + filename=line[1:].split("=")[0].rstrip()+".atf" + if dir: + filename=os.path.join(dir,filename) + nf=file(filename,"w") + logging.info("open %s"%filename) + if nf: + nf.write(line.replace("\n","")+"\n") + + try: + nf.close() except: - pass + pass fh.close() return ret,len(os.listdir(dir)) @@ -1522,6 +1537,16 @@ class CDLIFileFolder(extVersionedFileFol return ret + def getFile(self,fn): + """get the content of the file fn""" + founds=self.CDLICatalog.search({'title':fn}) + if not founds: + return [] + else: + obj=founds[0].getObject().getLastVersion() + logging.error(obj.getData()) + return obj.getData()[0:] + def checkCatalog(self,fn): """check if fn is in the catalog""" #TODO add checkCatalog @@ -1544,7 +1569,13 @@ class CDLIFileFolder(extVersionedFileFol return pt(search=list,author=author) - + def getAllPNumbers(self): + """get a list of all files (resp their p-numbers) stored""" + + ret=[x.getId for x in self.CDLICatalog()] + + return ret + def findObjectsFromList(self,enterList=None,display=False,start=None,upload=None,list=None,basketName=None,numberOfObjects=None,RESPONSE=None): """findObjectsFromList (, TAB oder LINE separated)""" @@ -1615,8 +1646,8 @@ class CDLIFileFolder(extVersionedFileFol catalog=getattr(self,self.default_catalog) #tf,tfilename=mkstemp() - if not hasattr(self.temp_folder,'downloadCounter'): - self.temp_folder.downloadCounter=0 + if not hasattr(self.temp_folder,'downloadCounter'): + self.temp_folder.downloadCounter=0 if getattr(self.temp_folder,'downloadCounter',0) > 5: return """I am sorry, currently the server has to many requests for downloads, please come back later!""" @@ -1762,7 +1793,7 @@ class CDLIRoot(Folder): return self.searchRegExpInLineIndexDocs(word) try: - lst=list(self.lineIndex.get(word).keys()) + lst=list(self.lineIndex.get(word).keys()) except: lst=[] if uniq: @@ -1832,63 +1863,63 @@ class CDLIRoot(Folder): file=self.showFile(fileId) #str="^%s\.[^%s\.]*%s[^\n]*\n"%(lineNum,lineNum,word) - #str="^%s\..*?%s[^\n]*\n"%(lineNum,word) + #str="^%s\..*?%s[^\n]*\n"%(lineNum,word) - #print str + #print str #m=re.search(str,file,flags=re.M|re.DOTALL) #if m: # return m.group() #else: - # return "" - #ret=lineNum+"." + # return "" + #ret=lineNum+"." #splitted=file.split(lineNum+".") - #if len(splitted)>1: - #for part in splitted[1:]: - #if part.find(word)>-1: - # for x in part.split("\n"): - #ret+=x - #if x.find(word)>-1: - #break - #break; - #return ret + #if len(splitted)>1: + #for part in splitted[1:]: + #if part.find(word)>-1: + # for x in part.split("\n"): + #ret+=x + #if x.find(word)>-1: + #break + #break; + #return ret def showWordInFile(self,fileId,word,lineList=None): """get lines with word fromFileId""" file=self.showFile(fileId) - ret=[] - for line in file.split("\n"): - if line.find(word)>-1: - if lineList: #liste of moeglichen Zeilennummern - num=line.split(".")[0] #Zeilenummer ist alles vor dem . in der Zeile - - if num in lineList: - - ret.append(line) - else: # nimm alles ohne line check - ret.append(line) - return ret + ret=[] + for line in file.split("\n"): + if line.find(word)>-1: + if lineList: #liste of moeglichen Zeilennummern + num=line.split(".")[0] #Zeilenummer ist alles vor dem . in der Zeile + + if num in lineList: + + ret.append(line) + else: # nimm alles ohne line check + ret.append(line) + return ret def tagWordInFile(self,fileId,word,lineList=None): """get lines with word fromFileId""" file=self.showFile(fileId) - tagStr="""%s""" - ret=[] - for line in file.split("\n"): - if line.find(word)>-1: - if lineList: #liste of moeglichen Zeilennummern - num=line.split(".")[0] #Zeilenummer ist alles vor dem . in der Zeile - - if num in lineList: - - ret.append(line.replace(word,tagStr%word)) - else: # nimm alles ohne line check - ret.append(line.replace(word,tagStr%word)) - else: - ret.append(line) - return "
\n".join(ret) + tagStr="""%s""" + ret=[] + for line in file.split("\n"): + if line.find(word)>-1: + if lineList: #liste of moeglichen Zeilennummern + num=line.split(".")[0] #Zeilenummer ist alles vor dem . in der Zeile + + if num in lineList: + + ret.append(line.replace(word,tagStr%word)) + else: # nimm alles ohne line check + ret.append(line.replace(word,tagStr%word)) + else: + ret.append(line) + return "
\n".join(ret) def URLquote(self,str): """quote url""" @@ -1922,8 +1953,8 @@ class CDLIRoot(Folder): "break all locks" ret=[] for f in self.ZopeFind(self,obj_metatypes="CDLI file",search_sub=1): - if str(f[1].lockedBy)=="dahl": - un=f[1].forceunlock() + if str(f[1].lockedBy)=="dahl": + un=f[1].forceunlock() if un and un !="": ret.append((f[0],un)) @@ -2152,7 +2183,7 @@ class CDLIRoot(Folder): obj=self.ZopeFind(root,obj_ids=[folder]) if ext: - ext.result+="

adding: %s

"%f + ext.result="

adding: %s

"%f+ext.result if not obj: manage_addCDLIFileFolder(root,folder,folder) fobj=getattr(root,folder) @@ -2180,13 +2211,13 @@ class CDLIRoot(Folder): self.CDLICatalog.catalog_object(ob) #self.CDLICatalog.manage_catalogFoundItems(obj_ids=[id],search_sub=1) #self.CDLICatalog.manage_catalogObject(self.REQUEST, self.REQUEST.RESPONSE, 'CDLICatalog', urlparse.urlparse(ob.absolute_url())[1]) - count+=1 + count+=1 - if count > 1000: - print "committing" - transaction.get().commit() - count=0 - transaction.get().commit() + if count > 1000: + print "committing" + transaction.get().commit() + count=0 + transaction.get().commit() return "ok" @@ -2207,9 +2238,9 @@ def manage_addCDLIRoot(self, id, title=' ob.id=str(id) ob.title=title try: - self._setObject(id, ob) + self._setObject(id, ob) except: - pass + pass ob=self._getOb(id) checkPermission=getSecurityManager().checkPermission