version 1.110, 2009/06/21 11:28:37
|
version 1.117, 2011/04/27 16:19:27
|
Line 90 class CDLIFileObject(CatalogAware,extVer
|
Line 90 class CDLIFileObject(CatalogAware,extVer
|
def getPNumber(self): |
def getPNumber(self): |
"""get the pnumber""" |
"""get the pnumber""" |
try: |
try: |
txt=re.match("&[Pp](\d*)\s*=([^\r\n]*)",self.getData()[0:]) |
txt=re.match("&[PpSs](\d*)\s*=([^\r\n]*)",self.getData()[0:]) |
except: |
except: |
txt=self.getData()[0:] |
txt=self.getData()[0:] |
|
|
Line 104 class CDLIFileObject(CatalogAware,extVer
|
Line 104 class CDLIFileObject(CatalogAware,extVer
|
def getDesignation(self): |
def getDesignation(self): |
"""get the designation out of the file""" |
"""get the designation out of the file""" |
try: |
try: |
txt=re.match("&[Pp](\d*)\s*=([^\r\n]*)",self.getData()[0:]) |
txt=re.match("&[PpSs](\d*)\s*=([^\r\n]*)",self.getData()[0:]) |
except: |
except: |
txt=self.getData()[0:] |
txt=self.getData()[0:] |
|
|
Line 306 def manage_addCDLIFile(self,id,title,loc
|
Line 306 def manage_addCDLIFile(self,id,title,loc
|
|
|
def checkUTF8(data): |
def checkUTF8(data): |
"""check utf 8""" |
"""check utf 8""" |
|
if not isinstance(data, str): |
|
logging.error("checkUTF8 data is not string! (%s)"%repr(data)) |
|
|
try: |
try: |
data.encode('utf-8') |
data.decode('utf-8') |
|
logging.debug("checkUTF8: ok!") |
return True |
return True |
except: |
except: |
|
logging.debug("checkUTF8: false!") |
return False |
return False |
|
|
|
|
Line 318 def checkFile(filename,data,folder):
|
Line 323 def checkFile(filename,data,folder):
|
# first check the file name |
# first check the file name |
fn=filename.split(".") # no extension |
fn=filename.split(".") # no extension |
|
|
if not fn[0][0]=="P": |
if not (fn[0][0]=="P" or fn[0][0]=="S"): |
return False,"P missing in the filename" |
return False,"P/S missing in the filename" |
elif len(fn[0])!=7: |
elif len(fn[0])!=7: |
return False,"P number has not the right length 6" |
return False,"P number has not the right length 6" |
elif not checkUTF8(data): |
elif not checkUTF8(data): |
Line 335 def splitatf(fh,dir=None,ext=None):
|
Line 340 def splitatf(fh,dir=None,ext=None):
|
i=0 |
i=0 |
|
|
#ROC: why split \n first and then \r??? |
#ROC: why split \n first and then \r??? |
if (type(fh) is StringType) or (type(fh) is UnicodeType): |
if isinstance(fh, basestring): |
iter=fh.split("\n") |
iter=fh.split("\n") |
else: |
else: |
iter=fh.readlines() |
iter=fh.readlines() |
Line 343 def splitatf(fh,dir=None,ext=None):
|
Line 348 def splitatf(fh,dir=None,ext=None):
|
for lineTmp in iter: |
for lineTmp in iter: |
lineTmp=lineTmp.replace(codecs.BOM_UTF8,'') # make sure that all BOM are removed.. |
lineTmp=lineTmp.replace(codecs.BOM_UTF8,'') # make sure that all BOM are removed.. |
for line in lineTmp.split("\r"): |
for line in lineTmp.split("\r"): |
#logging.log("Deal with: %s"%line) |
#logging.info("Deal with: %s"%line) |
if ext: |
if ext: |
i+=1 |
i+=1 |
if (i%100)==0: |
if (i%100)==0: |
Line 369 def splitatf(fh,dir=None,ext=None):
|
Line 374 def splitatf(fh,dir=None,ext=None):
|
if dir: |
if dir: |
filename=os.path.join(dir,filename) |
filename=os.path.join(dir,filename) |
nf=file(filename,"w") |
nf=file(filename,"w") |
logging.info("open %s"%filename) |
logging.debug("open %s"%filename) |
if nf: |
if nf: |
nf.write(line.replace("\n","")+"\n") |
nf.write(line.replace("\n","")+"\n") |
|
|
Line 378 def splitatf(fh,dir=None,ext=None):
|
Line 383 def splitatf(fh,dir=None,ext=None):
|
except: |
except: |
pass |
pass |
|
|
if not((type(fh) is StringType) or (type(fh) is UnicodeType)): |
if not isinstance(fh, basestring): |
fh.close() |
fh.close() |
|
|
return ret,len(os.listdir(dir)) |
return ret,len(os.listdir(dir)) |
|
|
|
|
Line 794 class CDLIRoot(Folder):
|
Line 800 class CDLIRoot(Folder):
|
RESPONSE.redirect(ob[0].absolute_url+"/unlock") |
RESPONSE.redirect(ob[0].absolute_url+"/unlock") |
return "not found" |
return "not found" |
|
|
|
|
def getFileObject(self,fileId): |
def getFileObject(self,fileId): |
"""get an object""" |
"""get an object""" |
|
logging.debug("getFileObj:"+repr(fileId)) |
if isinstance(fileId,CDLIFileObject): # support for old baskets |
if isinstance(fileId,CDLIFileObject): # support for old baskets |
return fileId |
return fileId |
x=self.v_files.get(fileId) |
x=self.v_files.get(fileId) |
#logging.debug("getFileObj:"+repr(fileId)) |
logging.debug("obj: "+repr(x)) |
|
if x==None: |
|
folder=fileId[0:3] |
|
f2=fileId[0:5] |
|
fObj = getattr(self.cdliRoot.cdli_main,folder); |
|
f2Obj = getattr(fObj,f2) |
|
|
|
o = getattr(f2Obj,fileId) |
|
logging.debug(o); |
|
self.updateOrAddToFileBTree(o) |
|
return o |
return x |
return x |
|
|
def getFileObjectLastVersion(self,fileId): |
def getFileObjectLastVersion(self,fileId): |
"""get an object""" |
"""get an object""" |
x=self.v_files_lastVersion.get(fileId) |
x=self.v_files_lastVersion.get(fileId) |
#logging.debug("lastVersion: "+repr(x)) |
logging.debug("lastVersion: "+repr(x)) |
|
if x==None: |
|
folder=fileId[0:3] |
|
f2=fileId[0:5] |
|
fObj = getattr(self.cdliRoot.cdli_main,folder); |
|
f2Obj = getattr(fObj,f2) |
|
|
|
o =getattr(f2Obj,fileId) |
|
logging.debug(o); |
|
return o.getLastVersion() |
|
|
|
|
return x |
return x |
|
|
def showFileIds(self): |
def showFileIds(self): |
Line 834 class CDLIRoot(Folder):
|
Line 863 class CDLIRoot(Folder):
|
self.v_files_lastVersion.update({obj.getId():obj.getLastVersion()}) |
self.v_files_lastVersion.update({obj.getId():obj.getLastVersion()}) |
|
|
self.v_file_ids.add(obj.getId()) |
self.v_file_ids.add(obj.getId()) |
|
#change everthing around to make it persistent... |
|
tmp = self.v_files |
|
self.v_files=tmp |
|
|
|
tmp2=self.v_file_ids |
|
self.v_file_ids=tmp2 |
|
|
|
self.CDLICache.cleanCache() #be sure that the cache is clean |
logging.debug("update:"+obj.getId()+"XXX"+repr(obj)) |
logging.debug("update:"+obj.getId()+"XXX"+repr(obj)) |
|
|
|
|
def deleteFromBTree(self,objId): |
def deleteFromBTree(self,objId): |
"""delete an obj""" |
"""delete an obj""" |
self.v_files.pop(objId) |
self.v_files.pop(objId) |
Line 856 class CDLIRoot(Folder):
|
Line 894 class CDLIRoot(Folder):
|
|
|
|
|
|
|
def searchText(self, query, index='graphemes'): |
def searchText(self, query, index='graphemes', resultFilter=None): |
"""searches query in the fulltext index and returns a list of file ids/P-numbers""" |
"""searches query in the fulltext index and returns a list of file IDs/P-numbers |
|
resultFilter is matched against the beginning of the file ID""" |
# see also: http://www.plope.com/Books/2_7Edition/SearchingZCatalog.stx#2-13 |
# see also: http://www.plope.com/Books/2_7Edition/SearchingZCatalog.stx#2-13 |
logging.debug("searchtext for '%s' in index %s"%(query,index)) |
logging.debug("searchtext for '%s' in index %s"%(query,index)) |
#import Products.ZCTextIndex.QueryParser |
#import Products.ZCTextIndex.QueryParser |
Line 868 class CDLIRoot(Folder):
|
Line 907 class CDLIRoot(Folder):
|
# do search |
# do search |
resultset = idx.search(query_request=idxQuery,sort_index='textid') |
resultset = idx.search(query_request=idxQuery,sort_index='textid') |
# put only the P-Number in the result |
# put only the P-Number in the result |
|
if resultFilter is None: |
results = [res.getId[:7] for res in resultset] |
results = [res.getId[:7] for res in resultset] |
|
else: |
|
results = [res.getId[:7] for res in resultset if res.getId.startswith(resultFilter)] |
logging.debug("searchtext: found %d texts"%len(results)) |
logging.debug("searchtext: found %d texts"%len(results)) |
return results |
return results |
|
|
Line 968 class CDLIRoot(Folder):
|
Line 1010 class CDLIRoot(Folder):
|
if line.lstrip().startswith('#lem:'): |
if line.lstrip().startswith('#lem:'): |
continue |
continue |
# ignore p-num line |
# ignore p-num line |
if line.startswith('&P'): |
if line.startswith('&P') or line.startswith('&S'): |
continue |
continue |
# ignore version lines |
# ignore version lines |
if line.startswith('#version'): |
if line.startswith('#version'): |
Line 1059 class CDLIRoot(Folder):
|
Line 1101 class CDLIRoot(Folder):
|
lv.author=user |
lv.author=user |
lv.versionComment="XXXXXXX" |
lv.versionComment="XXXXXXX" |
|
|
|
def forceunlock(self,REQUEST=None,user=None,fid=None): |
|
|
gen |
|
def forceunlock(self,REQUEST=None,user=None): |
|
"break all locks" |
"break all locks" |
|
if fid is not None: |
|
self.getFileObject(fid).forceunlock() |
|
return fid |
ret=[] |
ret=[] |
|
|
for f in self.ZopeFind(self,obj_metatypes="CDLI file",search_sub=1): |
for f in self.ZopeFind(self,obj_metatypes="CDLI file",search_sub=1): |
Line 1084 class CDLIRoot(Folder):
|
Line 1126 class CDLIRoot(Folder):
|
"""hole alle gesperrten files""" |
"""hole alle gesperrten files""" |
ret={} |
ret={} |
|
|
for f in self.ZopeFind(self,obj_metatypes="CDLI file",search_sub=1): |
for nm,f in self.v_files.items(): |
lb = f[1].lockedBy |
lb = str(f.lockedBy) |
add=False |
add=False |
if (lb is not None) and (lb!=""): |
if (lb is not None) and (lb!=""): |
|
|
Line 1097 class CDLIRoot(Folder):
|
Line 1139 class CDLIRoot(Folder):
|
if add==True: |
if add==True: |
if not ret.has_key(lb): |
if not ret.has_key(lb): |
ret[lb]=[] |
ret[lb]=[] |
ret[lb].append(f[0]) |
ret[lb].append(nm) |
|
|
|
|
if REQUEST is not None: |
if REQUEST is not None: |
Line 1282 class CDLIRoot(Folder):
|
Line 1324 class CDLIRoot(Folder):
|
|
|
def uploadATFfinally(self,procedure='',comment="",basketname='',unlock=None,repeat=None,RESPONSE=None): |
def uploadATFfinally(self,procedure='',comment="",basketname='',unlock=None,repeat=None,RESPONSE=None): |
"""nowupload the files""" |
"""nowupload the files""" |
|
|
|
|
|
|
threadName=repeat |
threadName=repeat |
if not threadName or threadName=="": |
if not threadName or threadName=="": |
thread=uploadATFfinallyThread() |
thread=uploadATFfinallyThread() |
Line 1344 class CDLIRoot(Folder):
|
Line 1383 class CDLIRoot(Folder):
|
try: |
try: |
self.cdliRoot.updateOrAddToFileBTree(ob[0].getObject()) |
self.cdliRoot.updateOrAddToFileBTree(ob[0].getObject()) |
except: |
except: |
|
|
logging.error("uploadATFfinally - cannot update Object %s Error: %s %s"%(ob[1],sys.exc_info()[0],sys.exc_info()[1])) |
logging.error("uploadATFfinally - cannot update Object %s Error: %s %s"%(ob[1],sys.exc_info()[0],sys.exc_info()[1])) |
|
for x in stObj.returnValue['newPs']: |
|
obj=self.getFileObject(x) #updates the object in the cache |
|
logging.debug("Got:"+repr(obj)) |
if RESPONSE is not None: |
if RESPONSE is not None: |
RESPONSE.redirect(self.absolute_url()) |
RESPONSE.redirect(self.absolute_url()) |
|
|
Line 1391 class CDLIRoot(Folder):
|
Line 1434 class CDLIRoot(Folder):
|
id=f |
id=f |
logging.debug("importFiles: addCDLIFile fobj2=%s, f=%s file2=%s"%(fobj2,repr(f),repr(file2))) |
logging.debug("importFiles: addCDLIFile fobj2=%s, f=%s file2=%s"%(fobj2,repr(f),repr(file2))) |
fobj2.addFile(vC='',file=file(file2),author=author,newName=f) |
fobj2.addFile(vC='',file=file(file2),author=author,newName=f) |
|
logging.debug("importfiles: fobj2.add") |
count+=1 |
count+=1 |
|
|
#now add the file to the storage |
#now add the file to the storage |
ob = getattr(fobj2,f) |
ob = getattr(fobj2,f) |
self.cdliRoot.updateOrAddToFileBTree(ob) |
logging.debug("importfiles: btree_start") |
|
#self.cdliRoot.updateOrAddToFileBTree(ob) |
|
logging.debug("importfiles: btree_end") |
if count%100==0: |
if count%100==0: |
logging.debug("importfiles: committing") |
logging.debug("importfiles: committing") |
transaction.get().commit() |
transaction.get().commit() |
|
|
|
logging.debug("importfiles: committing") |
transaction.get().commit() |
transaction.get().commit() |
|
logging.debug("importfiles: committing done") |
return "ok" |
return "ok" |
|
|
|
|