--- ECHO_content/VLPExtension.py 2008/08/05 16:17:46 1.70 +++ ECHO_content/VLPExtension.py 2009/05/06 07:53:46 1.90 @@ -2,16 +2,10 @@ This module contains extensions which where originally made for the VLP. """ from OFS.Cache import Cacheable -from Products.ECHO_content.ECHO_collection import * from Products.PageTemplates.PageTemplateFile import PageTemplateFile from Products.PageTemplates.PageTemplate import PageTemplate from Products.PageTemplates.ZopePageTemplate import ZopePageTemplate from OFS.Image import File -try: - from Products.zogiLib import zogiLib -except: - print "Zogilib not installed, VLP_resource will not work" - import xml.dom.minidom import urllib import xmlrpclib @@ -22,6 +16,14 @@ from stat import * from types import * from Globals import package_home import transaction +import Acquisition + +from Products.ECHO_content.ECHO_collection import * +from vlp_xmlhelpers import * +try: + from Products.zogiLib import zogiLib +except: + print "Zogilib not installed, VLP_resource will not work" def lemmatize(str): @@ -52,13 +54,14 @@ def checkXMLfrag(str): xmltempl = """ %s""" + try: - dom=xml.dom.minidom.parseString(xmltempl%str) + dom=xml.dom.minidom.parseString(xmltempl%utf8ify(str)) return str except: str=str.replace("& ","& ") - dom=xml.dom.minidom.parseString(xmltempl%str) + dom=xml.dom.minidom.parseString(xmltempl%utf8ify(str)) return str @@ -159,6 +162,10 @@ def manage_addSendMailForm(self, id, toA return '' +class VLP_collectionFolder(Folder): + """klasse fŸr folder innheralb der collection""" + meta_type="VLP_collectionFolder" + class VLP_essay(Folder): """classe fr VLP essays""" @@ -246,10 +253,14 @@ class VLP_essay(Folder): pages = self.objectItems(['DTML Document','File']) pages.sort(sortFind) + #logging.debug("pageList=%s"%repr(pages)) + return pages def getPageXMLfrag(self, pageNum=None, pageId=None, pageObj=None): """returns XML fragment of page with id""" + xml = None + if pageNum is not None: pl = self.getPageList() pageObj = pl[int(pageNum)-1][1] @@ -259,16 +270,24 @@ class VLP_essay(Folder): if pageObj is not None: txt = "" - data = pageObj.data - while data is not None: - txt += data - if isinstance(data, str): - data = None - else: - data = data.next + if pageObj.meta_type == 'File': + data = pageObj.data + #logging.debug("data=%s txt=%s"%(repr(data),txt)) + while data is not None: + if isinstance(data, str): + txt += data + data = None + else: + txt += data.data + data = data.next + else: + # not a File object... perhaps its executable + txt += pageObj() + xml = checkXMLfrag(txt) - return xml + + return unicodify(xml) def getPage(self,pagenum): @@ -464,10 +483,76 @@ def manage_addVLP_subCollection(self,id, if RESPONSE is not None: RESPONSE.redirect('manage_main') +from zope.publisher.interfaces import IPublishTraverse +from zope.interface import implements +from zope.publisher.interfaces import NotFound +from zope.app import zapi +from zope.component import queryMultiAdapter +try: + from ZPublisher.BaseRequest import DefaultPublishTraverse +except: + pass -class VLP_collection(ECHO_collection): +class VLP_collection(ECHO_collection,Acquisition.Explicit): """VLP spezifische Erweiterung der Collection""" + + + implements(IPublishTraverse) + def findObjFromLitName(self,fileName): + if not fileName[0:3]=="lit": + logging.error("getOrCreateFolder wrong filename: %s"%fileName) + return None + + + restName="%08d"%int(fileName[3:]) + + fs=(restName[0:2],restName[0:4],restName[0:6]) + + current =self + for f in fs: + obj = getattr(current,f,None) + if not obj: #subfolder existiert nicht + return None + current=obj + + return getattr(current,fileName,None) + + def publishTraverse(self,request,name): + """change the traversal if literature object""" + + actual_url=request['ACTUAL_URL'] + lastActual=actual_url.split("/")[-1] + logging.debug("pub_trav:"+actual_url) + + if name.startswith("lit"): #umleitung wenn lit aufgerufen wirk + # umleitung auf den eigentlichen folder nur wenn direkt der Folder litXXX oder dessen index_html methode aufgerufen wird + if lastActual.startswith("lit") or (lastActual=="index_html") or (lastActual=="index_meta") or (lastActual=="getTitle") or (lastActual=="getMetaDataLink") or (lastActual=="hasTranscription") or (lastActual=="transcription"): + #obj=self. ZopeFind(self,obj_ids=[name],search_sub=1) + ob=self.findObjFromLitName(name) + if not ob: + return "LIt not found" + else: + + logging.debug("request:"+repr(ob)) + request.response.setStatus(200) + #if (lastActual=="index_meta"): + # logging.debug("index_meta requested") + # return ob.index_meta + return ob + + else: ## andern falls transversiere zum parent. + obj = self.aq_parent + return obj + else: # mache gar nichts falls nicht lit aufgerufen wird + + + tr=DefaultPublishTraverse(self, request) + ob= tr.publishTraverse(request, name) + + return ob + #raise NotFound(self.context, name, request) + #return repr(request) meta_type="VLP_collection" manage_options=ECHO_collection.manage_options+( @@ -475,9 +560,15 @@ class VLP_collection(ECHO_collection): {'label':'Update Library','action':'updateCollection'}, {'label':'Update Metadata','action':'updateCollectionMD'}, ) - - - + def getAllRessources(self): + """getallressources""" + res=self.ZopeFind(self,obj_metatypes=['VLP_resource'],search_sub=1) + ret=[] + for r in res: + ret.append([r[0],r[1].absolute_url()]) + + return ret + def generateSubCollections(self,errorsTXT="",forceUpdate=False,RESPONSE=None): """erzeuge subcollectionen""" logging.debug("generateSubCollections") @@ -489,23 +580,25 @@ class VLP_collection(ECHO_collection): for found in founds: try: litid = str(found.reference) - foundCol=self.ZopeFind(self,obj_ids=[litid]) + foundCol=self.findObjFromLitName(litid) + #foundCol=self.ZopeFind(self,obj_ids=[litid]) if foundCol: - col = foundCol[0][1] - logging.debug("generateSubCollections: subcollection %s exists (%s)"%(col.getId(),found.reference)) - if (col.title != found.titlerefdisplay) or (col.label != found.titlerefdisplay): + col = foundCol + logging.debug("generateSubCollections: subcollection %s exists (%s)"%(repr(col),found.reference)) + logging.debug(repr([col.title,found.titlerefdisplay,col.label,found.titlerefdisplay])) + if (unicodify(col.title) != unicodify(found.titlerefdisplay)) or (unicodify(col.label) != unicodify(found.titlerefdisplay)): # subcollection seems to have changed logging.debug("generateSubCollections: subcollection has changed, recreating!") - self.manage_delObjects([foundCol[0][0]]) - manage_addVLP_subCollection(self,litid,found.titlerefdisplay,found.titlerefdisplay) + col.aq_parent.manage_delObjects([col.getId()]) + manage_addVLP_subCollection(self.getOrCreateFolderForFile(litid),litid,found.titlerefdisplay,found.titlerefdisplay) else: logging.debug("generateSubCollections: creating new subcollection %s"%found.reference) - manage_addVLP_subCollection(self,litid,found.titlerefdisplay,found.titlerefdisplay) + manage_addVLP_subCollection(self.getOrCreateFolderForFile(litid),litid,found.titlerefdisplay,found.titlerefdisplay) #teste ob es Images auf dem Server gibt mit gleichem Namen (frontmatter) if os.path.exists(os.path.join(self.vlp_basis,litid)): logging.debug("generateSubCollections: found frontmatter in %s"%litid) - obj=getattr(self,litid) + obj=self.findObjFromLitName(litid) if not self.ZopeFind(obj,obj_ids=[litid]): metalink=self.REQUEST['URL1']+"/"+litid+"/"+litid+"/index_meta" newObj=VLP_resource(litid,'',metalink,litid,litid,litid,'generated','book','','','','','','') @@ -516,16 +609,18 @@ class VLP_collection(ECHO_collection): if RESPONSE is not None: self.REQUEST.RESPONSE.write("

%s

\n"%litid) + logging.debug("

%s

\n"%litid) except: error=sys.exc_info()[0:2] logging.error("generateSubCollections: ERROR in %s"%litid) if RESPONSE is not None: + RESPONSE.write("

(ERROR updateII(%s): %s %s TXT: %s)\n

"%(litid,error[0],error[1],sys.exc_info()[2])) RESPONSE.write("

(ERROR (%s): %s %s)

\n"%(litid,error[0],error[1])) - #logging.error("generateSubCollections: %s %s"%error) - #self.REQUEST.RESPONSE.write("(ERROR (%s): %s %s)\n"%(litid,repr(error[0]),repr(error[1]))) + logging.error("generateSubCollections: %s %s"%error) + self.REQUEST.RESPONSE.write("(ERROR (%s): %s %s)\n"%(litid,repr(error[0]),repr(error[1]))) errorsTXT+="

ERROR: No subcollection of %s (ERROR: %s %s)

\n"%(litid,error[0],error[1]) - #errorsTXT+="

No subcollection of %s "%litid+"(ERROR: %s %s)

"%error + errorsTXT+="

No subcollection of %s "%litid+"(ERROR: %s %s)

"%error return errorsTXT @@ -540,29 +635,80 @@ class VLP_collection(ECHO_collection): if RESPONSE is not None: RESPONSE.redirect('manage_main') - def updateCollection(self,forceUpdate=False,RESPONSE=None): - """liest verzeichnisse aus dem pfad und legt sie dann als objekte in den ordner""" + + def getOrCreateFolderForFile(self,fileName): + + if not fileName[0:3]=="lit": + logging.error("getOrCreateFolder wrong filename: %s"%fileName) + return None + + try: + restName="%08d"%int(fileName[3:]) + except: + logging.error("getOrCreateFolder wrong filename: %s"%fileName) + return None + + fs=(restName[0:2],restName[0:4],restName[0:6]) + + current =self + for f in fs: + obj = getattr(current,f,None) + if not obj: + newObj=VLP_collectionFolder(f) + current._setObject(f,newObj) + obj = getattr(current,f) + current=obj + + return current + + def changeOrAddResources(self,forceUpdate=False,RESPONSE=None): + """changes ord updates all resources found in the file system""" + errorsTXT="

Errors

" if RESPONSE is not None: RESPONSE.setHeader('Content-Type','text/html') RESPONSE.write("") files=os.listdir(self.vlp_basis) + i=0 for fileName in files: + if i<100: + i+=1 + else: + i=0 + j=0 + while j<5: + try: + transaction.get().commit() + j=6 + except: + logging.error("Commit: %s"%j) + j+=1 + + RESPONSE.write("

committed

") if fileName[0:3]=="lit": metalink=self.REQUEST['URL1']+"/"+fileName+"/index_meta" + + folder=self.getOrCreateFolderForFile(fileName) #get the folder where fileName lives or should live + if not folder: #folder gave an error + continue try: - if not hasattr(self,fileName): + if not hasattr(folder,fileName): # create new resource logging.debug("updateCollection: new %s"%fileName) if RESPONSE is not None: RESPONSE.write("

new: %s

\n"%fileName) + logging.debug("new: %s \n"%fileName) newObj=VLP_resource(fileName,'',metalink,fileName,fileName,fileName,'generated','book','','','','','','') - self._setObject(fileName,newObj) + + folder._setObject(fileName,newObj) if RESPONSE is not None: RESPONSE.write("

got: %s "%fileName) - genObj=getattr(self,fileName) + + genObj=getattr(folder,fileName) + logging.debug("got: %s "%fileName) + logging.debug("updateCollection: inspecting %s"%fileName) # create index if hasattr(genObj,'createIndexFile'): @@ -574,7 +720,7 @@ class VLP_collection(ECHO_collection): except: error=sys.exc_info()[0:2] if RESPONSE is not None: - RESPONSE.write("(ERROR (%s): %s %s) "%(fileName,error[0],error[1])) + RESPONSE.write("(ERROR Update(%s): %s %s %s) "%(fileName,error[0],error[1],sys.exc_info()[2])) errorsTXT+="

ERROR creating index for %s "%fileName+"(ERROR: %s %s)

\n"%error # update pageSizeSum for subcollections @@ -593,10 +739,16 @@ class VLP_collection(ECHO_collection): error=sys.exc_info()[0:2] logging.error("updateCollection: ERROR in %s (%s %s)"%(fileName,error[0],error[1])) if RESPONSE is not None: - RESPONSE.write("

(ERROR (%s): %s %s)

\n"%(fileName,error[0],error[1])) + RESPONSE.write("(ERROR updateII(%s): %s %s %s) "%(fileName,error[0],error[1],sys.exc_info()[2])) errorsTXT+="

File not created:%s (ERROR: %s %s)

\n"%(fileName,error[0],error[1]) - - # update subcollections + return errorsTXT + + def updateCollection(self,forceUpdate=True,RESPONSE=None): + """liest verzeichnisse aus dem pfad und legt sie dann als objekte in den ordner""" + + # update subcollections + errorsTXT="" + errorsTXT+=self.changeOrAddResources(forceUpdate, RESPONSE); errorsTXT+=self.generateSubCollections(forceUpdate=forceUpdate,RESPONSE=RESPONSE) errorsTXT+="\n" if RESPONSE is not None: @@ -658,9 +810,14 @@ class VLP_resource(ECHO_resource,Cacheab meta_type="VLP_resource" + def _p_resolveConflict(self,oldstate,savedstate,newstate): + logging.debug("updateCollection: Have to resolve conflict!") + return newstate + #vlp_basis="/mpiwg/online/permanent/vlp" + referencetypes=[] manage_options=ECHO_resource.manage_options+Cacheable.manage_options+( @@ -670,17 +827,68 @@ class VLP_resource(ECHO_resource,Cacheab {'label':'Generate Library Template','action':'generateLibraryTemplateHTML'}, ) + def getMetaDataLink(self): + """link auf die metatdaten""" + return self.absolute_url()+"/index_meta" + + def getTitle(self): + """Title der Ressource""" + title= self.ZSQLSimpleSearch("""select title from vl_literature where reference= \'%s\' """ %self.getId())[0].title + logging.debug(title) + return title + def getImagePath(self): """Pfad zu den Images""" + path=None if os.path.isdir(os.path.join(self.vlp_basis,self.resourceID,'pageimg')): - return os.path.join(self.vlp_basis,self.resourceID,'pageimg') + path= os.path.join(self.vlp_basis,self.resourceID,'pageimg') elif os.path.isdir(os.path.join(self.vlp_basis,self.resourceID,'pages')): - return os.path.join(self.vlp_basis,self.resourceID,'pages') + path = os.path.join(self.vlp_basis,self.resourceID,'pages') + # workaround in some cases there seem to be no pictures in pages + + if os.path.isdir(os.path.join(self.vlp_basis,self.resourceID,'pagesHi')): + + + pagesHi = os.listdir(os.path.join(self.vlp_basis,self.resourceID,'pagesHi')) + if path is None: + return os.path.join(self.vlp_basis,self.resourceID,'pagesHi') + + pages = os.listdir(path) + + if len (pagesHi) > len(pages): + + countHi=0 + for p in pagesHi: + dat,ext = os.path.splitext(p) + if ext.lower() in [".tiff",".tif",".jpg"]: + countHi+=1 + count=0 + for p in pages: + dat,ext = os.path.splitext(p) + if ext.lower() in [".tiff",".tif",".jpg"]: + count+=1 + + if countHi > count: + path=os.path.join(self.vlp_basis,self.resourceID,'pagesHi') + + logging.error("pages < pagesHi: %s"%self.resourceID) + if path is None: + + + if len (os.listdir(self.vlp_basis,self.resourceID,'pagesHi')) > len(os.listdir(path)): + + + + path=os.path.join(self.vlp_basis,self.resourceID,'pagesHi') + logging.error("pages < pagesHi: %s"%self.resourceID) else: - return "no images " - + + logging.error("no images: %s"%self.resourceID) + return None + + return path def transcription(self): @@ -695,6 +903,34 @@ class VLP_resource(ECHO_resource,Cacheab + def identifyMediaType(self): + """identifiziert den Medientype aus der VLP Datenbank""" + founds=self.ZSQLSimpleSearch('select * from vl_literature where reference=\''+self.resourceID+'\' order by id') + if (not founds) or (len(founds)==0): + self.mediaType="generic" # steht nicht in der Datenbank, dann nimm generisch an + return self.mediaType + + + if founds[0].referencetype =="Audio": + self.mediaType ="audio" + elif founds[0].referencetype =="Film": + self.mediaType ="video" + else: + self.mediaType ="image" # nimm an, dass alle anderen Eintrage in der Datenbank auf images verweisen (Buecher oder Fotografien). + + return self.mediaType + + def getMediaType(self,cached=False): + """give the Media-type back, if chached False, then get it from the database""" + + if (not hasattr(self,'mediaType')) or (self.mediaType=="") or (not cached): + self.identifyMediaType() + + return self.mediaType + + + + def copyTranscriptionFromDB(self): """checks if transcription exits in DB""" founds=self.ZSQLSimpleSearch('select * from vl_transcript where source=\''+self.resourceID+'\' order by id') @@ -713,7 +949,7 @@ class VLP_resource(ECHO_resource,Cacheab ret+="" re2=ret[0:] - re3=re2.decode('latin-1').encode('utf-8') + re3=self.unicodify(re2) #re3=re2 ft=self.ZopeFind(self,obj_metatypes=['ECHO_fullText']) @@ -734,6 +970,19 @@ class VLP_resource(ECHO_resource,Cacheab else: return "no: %s"%self.getId() + def hasTranscription(self): + """transcription""" + + if not hasattr(self,'fulltext'): + self.copyTranscriptionFromDB() + if not hasattr(self,'fulltext'): #still not + self._setObject('fulltext',None) + + if self.fulltext is None: + return False + else: + return True + def resetMetaLink(self,all="no",RESPONSE=None): """resets metalink to standard resp. in case of server change it sets to the new server url""" if all=="yes": @@ -749,21 +998,33 @@ class VLP_resource(ECHO_resource,Cacheab def show(self): """show the rendered file""" - if self.REQUEST.get('p',None): - #self.REQUEST.set('pn',self.file2page(self.REQUEST.get('p',None))) - params = "pn=%s"%self.file2page(self.REQUEST.get('p',None)) - mk = self.REQUEST.get('mk',None) - if mk: - params += "&mk=%s"%mk - self.REQUEST.RESPONSE.redirect(self.REQUEST['URL']+'?'+params) - - if hasattr(self,'libraryTemplate.html'): - return getattr(self,'libraryTemplate.html')() + mt = self.getMediaType() - pt=PageTemplateFile(os.path.join(package_home(globals()),'vlp','library_template.zpt')).__of__(self) - pt.content_type="text/html" + if (mt in ['video','audio']): + pt = self.getVLPTemplate('libraryMovieTemplate') + else: #zur zeit werden alle anderen type auf das libraryTemplate abgebildet. + + if self.REQUEST.get('p',None): #make sure that parameter p exists + #self.REQUEST.set('pn',self.file2page(self.REQUEST.get('p',None))) + params = "pn=%s"%self.file2page(self.REQUEST.get('p',None)) + mk = self.REQUEST.get('mk',None) + if mk: + params += "&mk=%s"%mk + self.REQUEST.RESPONSE.redirect(self.REQUEST['URL']+'?'+params) + + pt = self.getVLPTemplate('libraryTemplate') + return pt() + def getVLPTemplate(self,name): + html=name+".html" + if hasattr(self,html): + return getattr(self,html) + + pt=PageTemplateFile(os.path.join(package_home(globals()),'vlp',name+'.zpt')).__of__(self) + pt.content_type="text/html" + return pt + def index_html(self): """index_html""" #return self.REQUEST @@ -790,21 +1051,31 @@ class VLP_resource(ECHO_resource,Cacheab RESPONSE.redirect('manage_main') - def index_meta(self): + def index_meta(self,RESPONSE=None): """index_meta""" - + pt=PageTemplateFile(os.path.join(package_home(globals()),'vlp','index_meta.zpt')).__of__(self) - pt.content_type="text/html" - return pt() - + pt.content_type="text/xml" + x= pt.pt_render() + #logging.debug("index_meta 4:"+x) + if RESPONSE: + RESPONSE.setHeader("Content-Type","text/xml") + return x def file2page(self,p): """converts filename to pagenumber""" - dom=xml.dom.minidom.parse(ECHO_helpers.urlopen(getattr(self,'index.xml').absolute_url())) + + #dom=xml.dom.minidom.parse(ECHO_helpers.urlopen(getattr(self,'index.xml').absolute_url())) + index=getattr(self,'index.xml') + + txt=index.pt_render() + dom=xml.dom.minidom.parseString(txt) for page in dom.getElementsByTagName('page'): # pageummer mit lo oder hi sind aequivalent if (page.getAttribute('file')==p) or ("%slo"%page.getAttribute('file')==p) or ("%shi"%page.getAttribute('file')==p): - return page.getAttribute('nr') + nr=page.getAttribute('nr') + + return nr return 0 @@ -857,13 +1128,14 @@ class VLP_resource(ECHO_resource,Cacheab ret="%s: %s"%(str[0],name) return ret - try: - ret=os.listdir(os.path.join(self.vlp_basis,self.resourceID,'pageimg')) - self.imagePath='pageimg' - except: - ret=os.listdir(os.path.join(self.vlp_basis,self.resourceID,'pages')) - self.imagePath='pages' + self.imagePathFull=self.getImagePath(); + self.imagePath=self.imagePathFull.split("/")[-1] + if (self.imagePath is None): + return None + temp=[] + ret = os.listdir(self.imagePathFull) + ret.sort() for x in ret: if (not (x[0] in ('.',':'))) and (not x[0:4] in excludeNames): @@ -982,7 +1254,10 @@ class VLP_resource(ECHO_resource,Cacheab zt=getattr(self,'index.xml') if not forceUpdate: # check if directory is newer (self.imagePath sould be set by now) - imgdir = os.path.join(self.vlp_basis, self.resourceID, self.imagePath) + path = self.getImagePath() + if (path is None): + return "CREATE INDEX FILE NOT POSSIBLE" + imgdir = os.path.join(path) dirtime = os.path.getmtime(imgdir) zttime = zt.bobobase_modification_time() logging.debug("createindexfile: dir %s of %s index of %s"%(imgdir,dirtime,zttime))