--- cdli/cdli_files.py 2009/06/24 11:13:47 1.112 +++ cdli/cdli_files.py 2011/04/27 16:19:27 1.117 @@ -90,7 +90,7 @@ class CDLIFileObject(CatalogAware,extVer def getPNumber(self): """get the pnumber""" try: - txt=re.match("&[Pp](\d*)\s*=([^\r\n]*)",self.getData()[0:]) + txt=re.match("&[PpSs](\d*)\s*=([^\r\n]*)",self.getData()[0:]) except: txt=self.getData()[0:] @@ -104,7 +104,7 @@ class CDLIFileObject(CatalogAware,extVer def getDesignation(self): """get the designation out of the file""" try: - txt=re.match("&[Pp](\d*)\s*=([^\r\n]*)",self.getData()[0:]) + txt=re.match("&[PpSs](\d*)\s*=([^\r\n]*)",self.getData()[0:]) except: txt=self.getData()[0:] @@ -306,10 +306,15 @@ def manage_addCDLIFile(self,id,title,loc def checkUTF8(data): """check utf 8""" + if not isinstance(data, str): + logging.error("checkUTF8 data is not string! (%s)"%repr(data)) + try: - data.encode('utf-8') + data.decode('utf-8') + logging.debug("checkUTF8: ok!") return True except: + logging.debug("checkUTF8: false!") return False @@ -318,8 +323,8 @@ def checkFile(filename,data,folder): # first check the file name fn=filename.split(".") # no extension - if not fn[0][0]=="P": - return False,"P missing in the filename" + if not (fn[0][0]=="P" or fn[0][0]=="S"): + return False,"P/S missing in the filename" elif len(fn[0])!=7: return False,"P number has not the right length 6" elif not checkUTF8(data): @@ -335,7 +340,7 @@ def splitatf(fh,dir=None,ext=None): i=0 #ROC: why split \n first and then \r??? - if (type(fh) is StringType) or (type(fh) is UnicodeType): + if isinstance(fh, basestring): iter=fh.split("\n") else: iter=fh.readlines() @@ -369,7 +374,7 @@ def splitatf(fh,dir=None,ext=None): if dir: filename=os.path.join(dir,filename) nf=file(filename,"w") - logging.info("open %s"%filename) + logging.debug("open %s"%filename) if nf: nf.write(line.replace("\n","")+"\n") @@ -378,8 +383,9 @@ def splitatf(fh,dir=None,ext=None): except: pass - if not((type(fh) is StringType) or (type(fh) is UnicodeType)): + if not isinstance(fh, basestring): fh.close() + return ret,len(os.listdir(dir)) @@ -801,13 +807,34 @@ class CDLIRoot(Folder): if isinstance(fileId,CDLIFileObject): # support for old baskets return fileId x=self.v_files.get(fileId) - + logging.debug("obj: "+repr(x)) + if x==None: + folder=fileId[0:3] + f2=fileId[0:5] + fObj = getattr(self.cdliRoot.cdli_main,folder); + f2Obj = getattr(fObj,f2) + + o = getattr(f2Obj,fileId) + logging.debug(o); + self.updateOrAddToFileBTree(o) + return o return x def getFileObjectLastVersion(self,fileId): """get an object""" x=self.v_files_lastVersion.get(fileId) - #logging.debug("lastVersion: "+repr(x)) + logging.debug("lastVersion: "+repr(x)) + if x==None: + folder=fileId[0:3] + f2=fileId[0:5] + fObj = getattr(self.cdliRoot.cdli_main,folder); + f2Obj = getattr(fObj,f2) + + o =getattr(f2Obj,fileId) + logging.debug(o); + return o.getLastVersion() + + return x def showFileIds(self): @@ -836,8 +863,17 @@ class CDLIRoot(Folder): self.v_files_lastVersion.update({obj.getId():obj.getLastVersion()}) self.v_file_ids.add(obj.getId()) - logging.debug("update:"+obj.getId()+"XXX"+repr(obj)) + #change everthing around to make it persistent... + tmp = self.v_files + self.v_files=tmp + tmp2=self.v_file_ids + self.v_file_ids=tmp2 + + self.CDLICache.cleanCache() #be sure that the cache is clean + logging.debug("update:"+obj.getId()+"XXX"+repr(obj)) + + def deleteFromBTree(self,objId): """delete an obj""" self.v_files.pop(objId) @@ -858,8 +894,9 @@ class CDLIRoot(Folder): - def searchText(self, query, index='graphemes'): - """searches query in the fulltext index and returns a list of file ids/P-numbers""" + def searchText(self, query, index='graphemes', resultFilter=None): + """searches query in the fulltext index and returns a list of file IDs/P-numbers + resultFilter is matched against the beginning of the file ID""" # see also: http://www.plope.com/Books/2_7Edition/SearchingZCatalog.stx#2-13 logging.debug("searchtext for '%s' in index %s"%(query,index)) #import Products.ZCTextIndex.QueryParser @@ -869,8 +906,11 @@ class CDLIRoot(Folder): idx = getattr(self, self.file_catalog) # do search resultset = idx.search(query_request=idxQuery,sort_index='textid') - # put only the P-Number in the result - results = [res.getId[:7] for res in resultset] + # put only the P-Number in the result + if resultFilter is None: + results = [res.getId[:7] for res in resultset] + else: + results = [res.getId[:7] for res in resultset if res.getId.startswith(resultFilter)] logging.debug("searchtext: found %d texts"%len(results)) return results @@ -970,7 +1010,7 @@ class CDLIRoot(Folder): if line.lstrip().startswith('#lem:'): continue # ignore p-num line - if line.startswith('&P'): + if line.startswith('&P') or line.startswith('&S'): continue # ignore version lines if line.startswith('#version'): @@ -1061,9 +1101,6 @@ class CDLIRoot(Folder): lv.author=user lv.versionComment="XXXXXXX" - - - gen def forceunlock(self,REQUEST=None,user=None,fid=None): "break all locks" if fid is not None: @@ -1287,9 +1324,6 @@ class CDLIRoot(Folder): def uploadATFfinally(self,procedure='',comment="",basketname='',unlock=None,repeat=None,RESPONSE=None): """nowupload the files""" - - - threadName=repeat if not threadName or threadName=="": thread=uploadATFfinallyThread() @@ -1349,7 +1383,11 @@ class CDLIRoot(Folder): try: self.cdliRoot.updateOrAddToFileBTree(ob[0].getObject()) except: + logging.error("uploadATFfinally - cannot update Object %s Error: %s %s"%(ob[1],sys.exc_info()[0],sys.exc_info()[1])) + for x in stObj.returnValue['newPs']: + obj=self.getFileObject(x) #updates the object in the cache + logging.debug("Got:"+repr(obj)) if RESPONSE is not None: RESPONSE.redirect(self.absolute_url()) @@ -1396,17 +1434,21 @@ class CDLIRoot(Folder): id=f logging.debug("importFiles: addCDLIFile fobj2=%s, f=%s file2=%s"%(fobj2,repr(f),repr(file2))) fobj2.addFile(vC='',file=file(file2),author=author,newName=f) + logging.debug("importfiles: fobj2.add") count+=1 #now add the file to the storage ob = getattr(fobj2,f) - self.cdliRoot.updateOrAddToFileBTree(ob) - + logging.debug("importfiles: btree_start") + #self.cdliRoot.updateOrAddToFileBTree(ob) + logging.debug("importfiles: btree_end") if count%100==0: logging.debug("importfiles: committing") transaction.get().commit() + logging.debug("importfiles: committing") transaction.get().commit() + logging.debug("importfiles: committing done") return "ok"