version 1.106, 2009/06/04 05:56:16
|
version 1.119, 2011/07/20 09:33:35
|
Line 25 import copy
|
Line 25 import copy
|
import codecs |
import codecs |
import sys |
import sys |
from BTrees.IOBTree import IOBTree |
from BTrees.IOBTree import IOBTree |
|
from BTrees.OOBTree import OOBTree |
import cdliSplitter |
import cdliSplitter |
from sets import Set |
from sets import Set |
import md5 |
import md5 |
Line 89 class CDLIFileObject(CatalogAware,extVer
|
Line 90 class CDLIFileObject(CatalogAware,extVer
|
def getPNumber(self): |
def getPNumber(self): |
"""get the pnumber""" |
"""get the pnumber""" |
try: |
try: |
txt=re.match("&[Pp](\d*)\s*=([^\r\n]*)",self.getData()[0:]) |
txt=re.match("&[PpSs](\d*)\s*=([^\r\n]*)",self.getData()[0:]) |
except: |
except: |
txt=self.getData()[0:] |
txt=self.getData()[0:] |
|
|
Line 103 class CDLIFileObject(CatalogAware,extVer
|
Line 104 class CDLIFileObject(CatalogAware,extVer
|
def getDesignation(self): |
def getDesignation(self): |
"""get the designation out of the file""" |
"""get the designation out of the file""" |
try: |
try: |
txt=re.match("&[Pp](\d*)\s*=([^\r\n]*)",self.getData()[0:]) |
txt=re.match("&[PpSs](\d*)\s*=([^\r\n]*)",self.getData()[0:]) |
except: |
except: |
txt=self.getData()[0:] |
txt=self.getData()[0:] |
|
|
Line 305 def manage_addCDLIFile(self,id,title,loc
|
Line 306 def manage_addCDLIFile(self,id,title,loc
|
|
|
def checkUTF8(data): |
def checkUTF8(data): |
"""check utf 8""" |
"""check utf 8""" |
|
if not isinstance(data, str): |
|
logging.error("checkUTF8 data is not string! (%s)"%repr(data)) |
|
|
try: |
try: |
data.encode('utf-8') |
data.decode('utf-8') |
|
logging.debug("checkUTF8: ok!") |
return True |
return True |
except: |
except: |
|
logging.debug("checkUTF8: false!") |
return False |
return False |
|
|
|
|
Line 317 def checkFile(filename,data,folder):
|
Line 323 def checkFile(filename,data,folder):
|
# first check the file name |
# first check the file name |
fn=filename.split(".") # no extension |
fn=filename.split(".") # no extension |
|
|
if not fn[0][0]=="P": |
if not (fn[0][0]=="P" or fn[0][0]=="S"): |
return False,"P missing in the filename" |
return False,"P/S missing in the filename" |
elif len(fn[0])!=7: |
elif len(fn[0])!=7: |
return False,"P number has not the right length 6" |
return False,"P number has not the right length 6" |
elif not checkUTF8(data): |
elif not checkUTF8(data): |
Line 334 def splitatf(fh,dir=None,ext=None):
|
Line 340 def splitatf(fh,dir=None,ext=None):
|
i=0 |
i=0 |
|
|
#ROC: why split \n first and then \r??? |
#ROC: why split \n first and then \r??? |
if (type(fh) is StringType) or (type(fh) is UnicodeType): |
if isinstance(fh, basestring): |
iter=fh.split("\n") |
iter=fh.split("\n") |
else: |
else: |
iter=fh.readlines() |
iter=fh.readlines() |
Line 342 def splitatf(fh,dir=None,ext=None):
|
Line 348 def splitatf(fh,dir=None,ext=None):
|
for lineTmp in iter: |
for lineTmp in iter: |
lineTmp=lineTmp.replace(codecs.BOM_UTF8,'') # make sure that all BOM are removed.. |
lineTmp=lineTmp.replace(codecs.BOM_UTF8,'') # make sure that all BOM are removed.. |
for line in lineTmp.split("\r"): |
for line in lineTmp.split("\r"): |
#logging.log("Deal with: %s"%line) |
#logging.info("Deal with: %s"%line) |
if ext: |
if ext: |
i+=1 |
i+=1 |
if (i%100)==0: |
if (i%100)==0: |
Line 368 def splitatf(fh,dir=None,ext=None):
|
Line 374 def splitatf(fh,dir=None,ext=None):
|
if dir: |
if dir: |
filename=os.path.join(dir,filename) |
filename=os.path.join(dir,filename) |
nf=file(filename,"w") |
nf=file(filename,"w") |
logging.info("open %s"%filename) |
logging.debug("open %s"%filename) |
if nf: |
if nf: |
nf.write(line.replace("\n","")+"\n") |
nf.write(line.replace("\n","")+"\n") |
|
|
Line 377 def splitatf(fh,dir=None,ext=None):
|
Line 383 def splitatf(fh,dir=None,ext=None):
|
except: |
except: |
pass |
pass |
|
|
if not((type(fh) is StringType) or (type(fh) is UnicodeType)): |
if not isinstance(fh, basestring): |
fh.close() |
fh.close() |
|
|
return ret,len(os.listdir(dir)) |
return ret,len(os.listdir(dir)) |
|
|
|
|
Line 720 class CDLIRoot(Folder):
|
Line 727 class CDLIRoot(Folder):
|
splitter = {'words':cdliSplitter.wordSplitter(), |
splitter = {'words':cdliSplitter.wordSplitter(), |
'graphemes':cdliSplitter.graphemeSplitter()} |
'graphemes':cdliSplitter.graphemeSplitter()} |
|
|
|
def redirect(self,RESPONSE,url): |
|
"""mache ein redirect mit einem angehaengten time stamp um ein reload zu erzwingen""" |
|
|
|
timeStamp=time.time() |
|
|
|
if url.find("?")>-1: #giebt es schon parameter |
|
addStr="&time=%s" |
|
else: |
|
addStr="?time=%s" |
|
|
|
RESPONSE.setHeader('Last-Modified',email.Utils.formatdate().split("-")[0]+'GMT') |
|
logging.error(email.Utils.formatdate()+' GMT') |
|
RESPONSE.redirect(url+addStr%timeStamp) |
|
|
def unicodify(self,txt): |
def unicodify(self,txt): |
return unicodify(txt) |
return unicodify(txt) |
Line 755 class CDLIRoot(Folder):
|
Line 775 class CDLIRoot(Folder):
|
"""view an Object""" |
"""view an Object""" |
ob = self.CDLICatalog({'title':id}) |
ob = self.CDLICatalog({'title':id}) |
if len(ob)>0: |
if len(ob)>0: |
RESPONSE.redirect(ob[0].getLastVersion().absolute_url()) |
logging.info("objekt:"+repr(ob[0])) |
|
#RESPONSE.redirect(ob[0].getLastVersion().absolute_url()) |
|
RESPONSE.redirect(ob[0].absolute_url+"/download") |
return "not found" |
return "not found" |
def addCDLIFileObjectForm(self,id,RESPONSE): |
def addCDLIFileObjectForm(self,id,RESPONSE): |
"""view an Object""" |
"""view an Object""" |
Line 778 class CDLIRoot(Folder):
|
Line 800 class CDLIRoot(Folder):
|
RESPONSE.redirect(ob[0].absolute_url+"/unlock") |
RESPONSE.redirect(ob[0].absolute_url+"/unlock") |
return "not found" |
return "not found" |
|
|
|
|
def getFileObject(self,fileId): |
def getFileObject(self,fileId): |
"""get an object""" |
"""get an object""" |
|
logging.debug("getFileObj:"+repr(fileId)) |
|
if isinstance(fileId,CDLIFileObject): # support for old baskets |
|
return fileId |
x=self.v_files.get(fileId) |
x=self.v_files.get(fileId) |
#logging.debug(x) |
logging.debug("obj: "+repr(x)) |
|
if x is None: |
|
logging.debug("fileId"+repr(fileId)) |
|
folder=fileId[0:3] |
|
f2=fileId[0:5] |
|
fObj = getattr(self.cdliRoot.cdli_main,folder); |
|
f2Obj = getattr(fObj,f2) |
|
|
|
o = getattr(f2Obj,fileId) |
|
logging.debug(o); |
|
self.updateOrAddToFileBTree(o) |
|
return o |
return x |
return x |
|
|
def getFileObjectLastVersion(self,fileId): |
def getFileObjectLastVersion(self,fileId): |
"""get an object""" |
"""get an object""" |
x=self.v_files_lastVersion.get(fileId) |
x=self.v_files_lastVersion.get(fileId) |
#logging.debug("lastVersion: "+repr(x)) |
logging.debug("lastVersion: "+repr(x)) |
|
if x==None: |
|
folder=fileId[0:3] |
|
f2=fileId[0:5] |
|
fObj = getattr(self.cdliRoot.cdli_main,folder); |
|
f2Obj = getattr(fObj,f2) |
|
|
|
o =getattr(f2Obj,fileId) |
|
logging.debug(o); |
|
return o.getLastVersion() |
|
|
|
|
return x |
return x |
|
|
def showFileIds(self): |
def showFileIds(self): |
Line 816 class CDLIRoot(Folder):
|
Line 864 class CDLIRoot(Folder):
|
self.v_files_lastVersion.update({obj.getId():obj.getLastVersion()}) |
self.v_files_lastVersion.update({obj.getId():obj.getLastVersion()}) |
|
|
self.v_file_ids.add(obj.getId()) |
self.v_file_ids.add(obj.getId()) |
|
#change everthing around to make it persistent... |
|
tmp = self.v_files |
|
self.v_files=tmp |
|
|
|
tmp2=self.v_file_ids |
|
self.v_file_ids=tmp2 |
|
|
|
self.CDLICache.cleanCache() #be sure that the cache is clean |
logging.debug("update:"+obj.getId()+"XXX"+repr(obj)) |
logging.debug("update:"+obj.getId()+"XXX"+repr(obj)) |
|
|
|
|
def deleteFromBTree(self,objId): |
def deleteFromBTree(self,objId): |
"""delete an obj""" |
"""delete an obj""" |
self.v_files.pop(objId) |
self.v_files.pop(objId) |
Line 838 class CDLIRoot(Folder):
|
Line 895 class CDLIRoot(Folder):
|
|
|
|
|
|
|
def searchText(self, query, index='graphemes'): |
def searchText(self, query, index='graphemes', resultFilter=None): |
"""searches query in the fulltext index and returns a list of file ids/P-numbers""" |
"""searches query in the fulltext index and returns a list of file IDs/P-numbers |
|
resultFilter is matched against the beginning of the file ID""" |
# see also: http://www.plope.com/Books/2_7Edition/SearchingZCatalog.stx#2-13 |
# see also: http://www.plope.com/Books/2_7Edition/SearchingZCatalog.stx#2-13 |
logging.debug("searchtext for '%s' in index %s"%(query,index)) |
logging.debug("searchtext for '%s' in index %s"%(query,index)) |
#import Products.ZCTextIndex.QueryParser |
#import Products.ZCTextIndex.QueryParser |
Line 850 class CDLIRoot(Folder):
|
Line 908 class CDLIRoot(Folder):
|
# do search |
# do search |
resultset = idx.search(query_request=idxQuery,sort_index='textid') |
resultset = idx.search(query_request=idxQuery,sort_index='textid') |
# put only the P-Number in the result |
# put only the P-Number in the result |
|
if resultFilter is None: |
results = [res.getId[:7] for res in resultset] |
results = [res.getId[:7] for res in resultset] |
|
else: |
|
results = [res.getId[:7] for res in resultset if res.getId.startswith(resultFilter)] |
logging.debug("searchtext: found %d texts"%len(results)) |
logging.debug("searchtext: found %d texts"%len(results)) |
return results |
return results |
|
|
Line 950 class CDLIRoot(Folder):
|
Line 1011 class CDLIRoot(Folder):
|
if line.lstrip().startswith('#lem:'): |
if line.lstrip().startswith('#lem:'): |
continue |
continue |
# ignore p-num line |
# ignore p-num line |
if line.startswith('&P'): |
if line.startswith('&P') or line.startswith('&S'): |
continue |
continue |
# ignore version lines |
# ignore version lines |
if line.startswith('#version'): |
if line.startswith('#version'): |
Line 1029 class CDLIRoot(Folder):
|
Line 1090 class CDLIRoot(Folder):
|
|
|
def changeUserForPObjectFromFile(self,user,fname): |
def changeUserForPObjectFromFile(self,user,fname): |
"""aendere user fuer alle p in fiele""" |
"""aendere user fuer alle p in fiele""" |
pns = file(fname,"r") |
pns = file(os.path.join(package_home(globals()),'inputs',fname),"r") |
for p in pns.readline(): |
for p in pns.readlines(): |
f = getattr(self, self.file_catalog).search({'textid':pnum}) |
p=p.lstrip().rstrip() |
pObj=f.getObject() |
logging.info(str(p+".atf")) |
|
pObj=self.getFileObject(p+".atf") |
|
if pObj is not None: |
|
logging.info(pObj) |
lv=pObj.getContentObject() |
lv=pObj.getContentObject() |
logging.info("author:"+lv.author) |
logging.info("author:"+lv.author) |
|
lv.author=user |
|
lv.versionComment="XXXXXXX" |
|
|
|
def forceunlock(self,REQUEST=None,user=None,fid=None): |
|
|
|
|
|
|
def forceunlock(self,REQUEST=None,user=None): |
|
"break all locks" |
"break all locks" |
|
if fid is not None: |
|
self.getFileObject(fid).forceunlock() |
|
return fid |
ret=[] |
ret=[] |
|
|
for f in self.ZopeFind(self,obj_metatypes="CDLI file",search_sub=1): |
for f in self.ZopeFind(self,obj_metatypes="CDLI file",search_sub=1): |
Line 1062 class CDLIRoot(Folder):
|
Line 1127 class CDLIRoot(Folder):
|
"""hole alle gesperrten files""" |
"""hole alle gesperrten files""" |
ret={} |
ret={} |
|
|
for f in self.ZopeFind(self,obj_metatypes="CDLI file",search_sub=1): |
for nm,f in self.v_files.items(): |
lb = f[1].lockedBy |
lb = str(f.lockedBy) |
add=False |
add=False |
if (lb is not None) and (lb!=""): |
if (lb is not None) and (lb!=""): |
|
|
Line 1075 class CDLIRoot(Folder):
|
Line 1140 class CDLIRoot(Folder):
|
if add==True: |
if add==True: |
if not ret.has_key(lb): |
if not ret.has_key(lb): |
ret[lb]=[] |
ret[lb]=[] |
ret[lb].append(f[0]) |
ret[lb].append(nm) |
|
|
|
|
if REQUEST is not None: |
if REQUEST is not None: |
Line 1180 class CDLIRoot(Folder):
|
Line 1245 class CDLIRoot(Folder):
|
|
|
return generateXMLReturn(stObj.returnValue) |
return generateXMLReturn(stObj.returnValue) |
|
|
|
def uploadATFAsync(self,repeat=None,upload=None,basketId=0,RESPONSE=None): |
|
"""upload an atf file / basket file""" |
|
#self._v_uploadATF.returnValue=None |
|
|
|
|
|
idTmp=str(randint(0,1000000000)) |
|
|
|
if upload is None: |
|
return "ERROR NO FILE!" |
|
|
|
tmpFile = File("/tmp/idTmp","w") |
|
# sicher basket in file |
|
for x in upload.read(): |
|
tmpFile.write(x) |
|
|
|
tmpFile.close(); |
|
uploadATFAsync(baskerId,idTmp,basketId,self.REQUEST['AUTHENTICATED_USER'],idTmp,serverport=self.REQUEST['SERVER_PORT']) |
|
|
|
return idTMP |
|
|
|
def viewTicketAsync(self,ticketNr=1): |
|
"""viewticket""" |
|
|
|
tmp = pickle.load(file("/tmp/"+str(ticketNr)+".result")) |
|
|
|
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadCheckAsync.zpt')).__of__(self) |
|
|
|
return pt(changed=tmp['changed'],lockerrors=tmp['lockerrors'],errors=tmp['errors'],dir=tmp['dir'],newPs=tmp['newPs'],basketLen=tmp['basketLen'],numberOfFiles=tmp['numberOfFiles'], |
|
basketNameFromId=tmp['basketNameFromId'],basketNameFromFile=tmp['basketNameFromFile'],basketId=tmp['basketId']) |
|
|
|
|
|
|
def uploadATF(self,repeat=None,upload=None,basketId=0,RESPONSE=None): |
def uploadATF(self,repeat=None,upload=None,basketId=0,RESPONSE=None): |
"""upload an atf file / basket file""" |
"""upload an atf file / basket file""" |
#self._v_uploadATF.returnValue=None |
#self._v_uploadATF.returnValue=None |
Line 1251 class CDLIRoot(Folder):
|
Line 1348 class CDLIRoot(Folder):
|
return pt(changed=tmp['changed'],lockerrors=tmp['lockerrors'],errors=tmp['errors'],dir=tmp['dir'],newPs=tmp['newPs'],basketLen=tmp['basketLen'],numberOfFiles=tmp['numberOfFiles'], |
return pt(changed=tmp['changed'],lockerrors=tmp['lockerrors'],errors=tmp['errors'],dir=tmp['dir'],newPs=tmp['newPs'],basketLen=tmp['basketLen'],numberOfFiles=tmp['numberOfFiles'], |
basketNameFromId=tmp['basketNameFromId'],basketNameFromFile=tmp['basketNameFromFile'],basketId=tmp['basketId']) |
basketNameFromId=tmp['basketNameFromId'],basketNameFromFile=tmp['basketNameFromFile'],basketId=tmp['basketId']) |
|
|
|
|
def redoUpload(self,threadName): |
def redoUpload(self,threadName): |
"""redo the upload""" |
"""redo the upload""" |
tmp=self.cdli_main.tmpStore2[threadName] |
tmp=self.cdli_main.tmpStore2[threadName] |
Line 1260 class CDLIRoot(Folder):
|
Line 1358 class CDLIRoot(Folder):
|
|
|
def uploadATFfinally(self,procedure='',comment="",basketname='',unlock=None,repeat=None,RESPONSE=None): |
def uploadATFfinally(self,procedure='',comment="",basketname='',unlock=None,repeat=None,RESPONSE=None): |
"""nowupload the files""" |
"""nowupload the files""" |
|
|
|
|
|
|
threadName=repeat |
threadName=repeat |
if not threadName or threadName=="": |
if not threadName or threadName=="": |
thread=uploadATFfinallyThread() |
thread=uploadATFfinallyThread() |
Line 1322 class CDLIRoot(Folder):
|
Line 1417 class CDLIRoot(Folder):
|
try: |
try: |
self.cdliRoot.updateOrAddToFileBTree(ob[0].getObject()) |
self.cdliRoot.updateOrAddToFileBTree(ob[0].getObject()) |
except: |
except: |
|
|
logging.error("uploadATFfinally - cannot update Object %s Error: %s %s"%(ob[1],sys.exc_info()[0],sys.exc_info()[1])) |
logging.error("uploadATFfinally - cannot update Object %s Error: %s %s"%(ob[1],sys.exc_info()[0],sys.exc_info()[1])) |
|
for x in stObj.returnValue['newPs']: |
|
obj=self.getFileObject(x) #updates the object in the cache |
|
logging.debug("Got:"+repr(obj)) |
if RESPONSE is not None: |
if RESPONSE is not None: |
RESPONSE.redirect(self.absolute_url()) |
RESPONSE.redirect(self.absolute_url()) |
|
|
Line 1342 class CDLIRoot(Folder):
|
Line 1441 class CDLIRoot(Folder):
|
obj=self.ZopeFind(root,obj_ids=[folder]) |
obj=self.ZopeFind(root,obj_ids=[folder]) |
logging.debug("importFiles: folder=%s f2=%s obj=%s"%(folder,f2,obj)) |
logging.debug("importFiles: folder=%s f2=%s obj=%s"%(folder,f2,obj)) |
if ext: |
if ext: |
|
|
|
if type(ext.result) is types.FileType: |
|
ext.result.write("<p>adding: %s </p>\n"%f) |
|
else: |
ext.result="<p>adding: %s </p>"%f+ext.result |
ext.result="<p>adding: %s </p>"%f+ext.result |
|
|
|
|
Line 1369 class CDLIRoot(Folder):
|
Line 1472 class CDLIRoot(Folder):
|
id=f |
id=f |
logging.debug("importFiles: addCDLIFile fobj2=%s, f=%s file2=%s"%(fobj2,repr(f),repr(file2))) |
logging.debug("importFiles: addCDLIFile fobj2=%s, f=%s file2=%s"%(fobj2,repr(f),repr(file2))) |
fobj2.addFile(vC='',file=file(file2),author=author,newName=f) |
fobj2.addFile(vC='',file=file(file2),author=author,newName=f) |
|
logging.debug("importfiles: fobj2.add") |
count+=1 |
count+=1 |
|
|
#now add the file to the storage |
#now add the file to the storage |
ob = getattr(fobj2,f) |
ob = getattr(fobj2,f) |
self.cdliRoot.updateOrAddToFileBTree(ob) |
logging.debug("importfiles: btree_start") |
|
#self.cdliRoot.updateOrAddToFileBTree(ob) |
|
logging.debug("importfiles: btree_end") |
if count%100==0: |
if count%100==0: |
logging.debug("importfiles: committing") |
logging.debug("importfiles: committing") |
transaction.get().commit() |
transaction.get().commit() |
|
|
|
logging.debug("importfiles: committing") |
transaction.get().commit() |
transaction.get().commit() |
|
logging.debug("importfiles: committing done") |
return "ok" |
return "ok" |
|
|
|
|