# HG changeset patch # User casties # Date 1311605448 -7200 # Node ID 9f9d9be26e53acd338477ebf3d86895c46feab4f first checkin in Mercurial (see history in SVN) diff -r 000000000000 -r 9f9d9be26e53 MetaData.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/MetaData.py Mon Jul 25 16:50:48 2011 +0200 @@ -0,0 +1,419 @@ +from OFS.Folder import Folder +from Products.PageTemplates.PageTemplateFile import PageTemplateFile +from Globals import package_home +from AccessControl import ClassSecurityInfo +import os.path +import urllib +import logging +import urlparse + +# TODO: which xml toolkit? +import amara +import xml.sax.saxutils +import xml.dom.minidom +import xml.etree.ElementTree as ET + + +# TODO: do we need this? +#from Products.OSA_system2 import OSAS_helpers +#from Products.OSA_system2.OSAS_metadata import OSAS_Metadata,OSAS_MetadataMapping + +from OSAS_metadata import OSAS_Metadata, OSAS_MetadataMapping + + +from SrvTxtUtils import getHttpData, getText + + +# TODO: get rid of this +def getTextFromNode(nodelist): + """gibt text aus nodelist""" + rc = "" + for node in nodelist: + if node.nodeType == node.TEXT_NODE: + rc = rc + node.data + return rc + + +def normalizeBibtype(bt): + """returns normalised bib type for looking up mappings""" + bt = bt.strip().replace(' ', '-').lower() + return bt + +def toString(list): + ret=u"" + + for l in list: + ret+=unicode(l) + + return ret + +def dcMetaDataToHash(mdSet): + """Convenience Function for creates a hash from the DCMetadataset + @param mdSet: String containing DCMetadata informmation + currently only in the format getDCMetadata of this module""" + + NSS = { + 'rdf': 'http://www.w3.org/1999/02/22-rdf-syntax-ns#', + 'dc': 'http://dublincore.org/documents/dcmi-namespace/', + 'owl':"http://www.w3.org/2002/07/owl#", + 'rdfs':"http://www.w3.org/2000/01/rdf-schema#" + } + ret={} + import StringIO + import sys + buffer= StringIO.StringIO(mdSet) + try: + md = amara.parse(buffer,prefixes=NSS) + except: + logging.error("Error: %s (%s)"%(sys.exc_info()[0],sys.exc_info()[1])) + + ret["error"]=mdSet + return ret + + ret["title"] = toString(md.xml_xpath("//dc:title/text()")) + ret["creator"] =toString(md.xml_xpath("//dc:creator/text()")) + ret["date"] = toString(md.xml_xpath("//dc:date/text()")) + + return ret + +def getBibdataFromDom(dom): + """returns dict with all elements from bib-tag""" + bibinfo = {} + bib = dom.find(".//meta/bib") + if bib is not None: + # put type in @type + type = bib.get('type') + bibinfo['@type'] = normalizedBibtype(type) + # put all subelements in dict + for e in bib: + bibinfo[e.tag] = getText(e) + + return bibinfo + + + + +class MetaData(OSAS_Metadata): + """provides basic methods for managing metadata structures""" + meta_type='MetaData' + security=ClassSecurityInfo() + manage_options = Folder.manage_options+( + {'label':'Main Config','action':'changeMetadataForm'}, + {'label':'Import XML Schema','action':'importMetaDataExportXML'}, + {'label':'Select Fields for Display','action':'indicateDisplayFieldsForm'}, + ) + + def __init__(self,id,shortDescription='',description='',fields=''): + """initialize a new instance""" + self.id = id + self.shortDescription =shortDescription #label fuer link auf add page + self.description=description #description of the method for link page + self.fieldList=fields.split(",")[0:] + self.metaDataServerUrl="" # muss mit change metadata gesetzt werden + + + def correctPath(self,path,remove=None,prefix=None,cut=0): + """convinience method um einen pfad zu veraendern""" + + if remove is not None: + path=path.replace(remove,'') + if prefix is not None: + path=os.path.join(prefix,path) + + if cut>0: + splitted=path.split("/") + path="/".join(splitted[0:len(splitted)-cut]) + return path + + def importMetaDataExportXML(self,importFile=None,RESPONSE=None): + """imports metadata from the metadataexportxml file""" + + if importFile is None: + pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','importMetaDataExportXML.zpt')).__of__(self) + return pt() + + dom=xml.dom.minidom.parse(importFile) + self.createMappingFromDom(dom.getElementsByTagName("metadataExport")[0]) + + if RESPONSE is not None: + RESPONSE.redirect('manage_main') + + + def createMappingFromDom(self,metadatanode,metadata=None): + """erzeuge ein Mapping aus dem der metadatanode des xmlformats, metadata ist ein metadataobject""" + + if metadata is None: + metadata=self + + nodes=metadatanode.childNodes + + #erster schritt: anlegen der fieldlist + for node in nodes: + logging.debug("node: %s"%node.tagName) + if node.tagName=="set": + set=node + if set.getAttribute('name')=='generic': + list=[] + for entry in set.getElementsByTagName('entry'): + list.append(entry.getAttribute('genericName')) + metadata.fieldList=list[0:] + + else: + id=set.getAttribute('name').encode('utf-8') + list=[] + argList={} + for entry in set.getElementsByTagName('entry'): + genericName=entry.getAttribute('genericName') + tag=entry.getAttribute('tag') + label=entry.getAttribute('label') + description=getTextFromNode(entry.childNodes) #TODO: clean + argList[genericName]=(tag,label,description) + metadata._setObject(id,MetaDataMapping(id,id,argList)) + + elif node.tagName=="metadata": + mn=node + name=mn.getAttribute('name').encode('utf-8') + metadata._setObject(name,MetaData(name,name)) + mdObj=getattr(metadata,name) + mdObj.createMappingFromDom(mn) + + + def getMDFromPathOrUrl(self,path): + parsedurl = urlparse.urlparse(path) + if parsedurl[0] != "": + # has schema (e.g. http) + url=path + else: + # path only + if path.endswith("index.meta"): + url =self.metaDataServerUrl%path + else: + url=os.path.join(self.metaDataServerUrl%path,'index.meta') + + #logging.debug("get Metadata: %s"%url) + md = getHttpData(url) + return md + + def getDCFormatted(self,path): + """get the metadata as dc set""" + logging.debug("getDCFormatted(path=%s)"%path) + namespace={ 'mpiwg': "http://www.mpiwg-berlin.mpg.de/ns/mpiwg"} + namespaceUsed=False + + md = self.getMDFromPathOrUrl(path) + logging.debug("MD in XML"+md) + im = amara.parse(md, prefixes=namespace) + + typePaths=im.xml_xpath('//bib/@type') + archimedes=False + + if len(typePaths)<1: + typePaths=im.xml_xpath('//meta/archimedes') # sinderfall fuer veraltete index.meta files vom typ archimedes + if len(typePaths)>0: + type = "archimedes" + archimedes=True + else: + typePaths=im.xml_xpath('//mpiwg:bib/@type') + if len(typePaths)<1: + return "" + else: + namespaceUsed=True + + type=unicode(typePaths[0]) + else: + type=unicode(typePaths[0]) + logging.info("got type:"+type) + try: + mapping=getattr(self.main.meta.bib,type.lower(),None) + except: + logging.error("getMetaDataFromServer no mapping for type: %s"%type) + return "" + + try: + dcMapping=getattr(self.main.meta.bib,"dc",None) + except: + logging.error("getMetaDataFromServer no dc in meta/bib") + return "" + + mds=mapping.generateMappingHash() # Hole das Mapping generisches Feld --> Feld im entsprechenden Typ + dcMds=dcMapping.generateMappingHash() + + mdHash=[] + logging.debug("Value: %s"%repr(mds)) + + for key,valueTriple in mds.items(): + value=valueTriple[0] + logging.debug("Value: %s"%repr(value)) + logging.debug("Key: %s"%repr(key)) + if value!="": + if not archimedes: + if namespaceUsed: + try: + v = im.xml_xpath('//mpiwg:bib/mpiwg:%s/text()'%value) + except: + logging.error('cannot do: //mpiwg:bib/mpiwg:%s/text()'%value) + else: + v = im.xml_xpath('//bib/%s/text()'%value) + else: + v = im.xml_xpath('//archimedes/%s/text()'%value) + if len(v) > 0: + dc=dcMds[key][0] + + if (dc !="") and (value !=""): + logging.debug("%s--> : %s"%(repr(value),dc)) + mdHash.append([dc,unicode(v[0])]) + + ret=""" """ + ret+="%s"%type + for md in mdHash: + + ret+="""%s"""%(md[0],xml.sax.saxutils.escape(md[1]),md[0]) + ret+="" + return ret + + + def getStdMappedHash(self, bibdata): + """returns dict with metadata from bibdata mapped according to standard mapping""" + mdHash={} + bibtype = bibdata['@type'] + # get mapping from main/meta/bib + try: + mapping=getattr(self.main.meta.bib, bibtype.lower()) + except: + logging.error("getStdMappedHash: no mapping for type: %s"%bibtype) + return mdHash + + mds = mapping.generateMappingHash() # Hole das Mapping generisches Feld --> Feld im entsprechenden Typ + + for field in mds.keys(): + # get mapped field name + mf = mds[field][0] + if not mf: + continue + logging.debug("mapping: %s = %s"%(field,mf)) + mdHash[field] = bibdata.get(mf, '') + + return mdHash + + + def getFormatted(self, template, path=None, dom=None, bibdata=None): + """returns string with document data formatted according to template. + gets data from server or dom or pre-parsed bibdata.""" + logging.debug("getFormatted(template=%s)"%(template)) + + if dom is None and bibdata is None: + # get from server + md = self.getMDFromPathOrUrl(path.replace("/mpiwg/online","")) + #logging.debug("md:"+md) + #dom = amara.parse(md) + dom = ET.fromstring(md) + + # get contents of bib tag + if bibdata is None: + bibdata = getBibdataFromDom(dom) + + bibtype = bibdata['@type'] + + # get template + tp=getattr(self,"%s_%s"%(template, bibtype.lower()), None) + if tp is None: + logging.warning("getFormatted: no template for: %s_%s"%(template, bibtype)) + # try generic + tp=getattr(self,"%s_generic"%(template), None) + if tp is None: + logging.error("getFormatted: no generic template either: %s"%(template)) + return "" + + # put mapped data in mdHash + mdHash = self.getStdMappedHash(bibdata) + + return tp(stdmd=mdHash, md=bibdata) + + + def getFormattedMetaData(self, path=None, dom=None, bibdata=None): + """get the metadafrom server""" + logging.debug("getFormattedMetaData(path=%s)"%path) + return self.getFormatted('metadata_template', path=path, dom=dom, bibdata=bibdata) + + def getFormattedMetaDataShort(self, path=None, dom=None, bibdata=None): + """get the metadafrom server""" + logging.debug("getFormattedMetaDataShort(path=%s)"%path) + return self.getFormatted('metadata_template', path=path, dom=dom, bibdata=bibdata) + + def getFormattedMetaDataExtended(self,path=None, dom=None, bibdata=None): + """get the metadafrom server""" + logging.debug("getFormattedMetaDataExtended(path=%s)"%path) + return self.getFormatted('metadata_extended_template', path=path, dom=dom, bibdata=bibdata) + + def getFormattedLabel(self,path=None, dom=None, bibdata=None): + """get the metadafrom server""" + logging.debug("getFormattedLabel(%s)"%path) + return self.getFormatted('label_template', path=path, dom=dom, bibdata=bibdata) + + def getFormattedMetaDataShortFromServer(self,path): + """get the metadafrom server""" + logging.debug("getFormattedMetaDataShortFromServer(path=%s)"%path) + return self.getFormatted('metadata_template', path) + + def getFormattedMetaDataExtendedFromServer(self,path): + """get the metadafrom server""" + logging.debug("getFormattedMetaDataExtendedFromServer(path=%s)"%path) + return self.getFormatted('metadata_extended_template', path) + + def getFormattedLabelFromServer(self,path): + """get the metadafrom server""" + logging.debug("getFormattedLabelFromServer(%s)"%path) + return self.getFormatted('label_template', path) + + + security.declarePublic('changeMetadataForm') + def changeMetadataForm(self): + """Main configuration""" + pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','changeMetadata.zpt')).__of__(self) + return pt() + + security.declarePublic('changeMetadata') + def changeMetadata(self,shortDescription,description,fields,metaDataServerUrl,RESPONSE=None): + """Change Metadata""" + self.shortDescription=shortDescription + self.description=description + self.fieldList=fields.split(",")[0:] + self.metaDataServerUrl=metaDataServerUrl + if RESPONSE is not None: + RESPONSE.redirect('manage_main') + + +def manage_addMetaDataForm(self): + """interface for adding the OSAS_add_Metadata""" + pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addMetadataForm.zpt')).__of__(self) + return pt() + +def manage_addMetaData(self,id,shortDescription,description,fields,RESPONSE=None): + """a metadata objekt""" + newObj=MetaData(id,shortDescription,description,fields) + self.Destination()._setObject(id,newObj) + if RESPONSE is not None: + RESPONSE.redirect('manage_main') + +class MetaDataMapping(OSAS_MetadataMapping): + meta_type="MetadataMapping" + +def manage_addMetaDataMappingForm(self): + """interface for adding the OSAS_root""" + pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addMetadataMappingForm.zpt')).__of__(self) + + return pt() + +def manage_addMetaDataMapping(self,idOfObject,titleOfObject,RESPONSE=None): + """add the OSAS_root""" + + argList={} + for arg in self.fieldList: + if not (arg in ['idOfObject','titleOfObject']): + argList[arg]=(self.REQUEST.form[arg],self.REQUEST.form['label_'+arg],self.REQUEST.form['explanation_'+arg],self.REQUEST.form['status_'+arg],self.REQUEST.form['values_'+arg]) + + newObj=MetaDataMapping(idOfObject,titleOfObject,argList) + self._setObject(idOfObject,newObj) + if RESPONSE is not None: + RESPONSE.redirect('manage_main') + diff -r 000000000000 -r 9f9d9be26e53 MetaDataClient.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/MetaDataClient.py Mon Jul 25 16:50:48 2011 +0200 @@ -0,0 +1,48 @@ +import xmlrpclib +import sys +import base64 +import bz2 +import re +import logging +class MetaDataClient: + + def __init__(self,serverUrl): + self.serverUrl=serverUrl + try: + self.server=xmlrpclib.ServerProxy(serverUrl) + except: + logging.error("MetaDataClient: writeMetaDataFile: (%s %s %s)"%sys.exc_info()) + + def writeMetaDataFile(self,path,newMetaXML,compressed=None): + #path hast to be an url to metadatafile or the path on the storage starting with /permanent or /experimental + # the url hast to contain the path to metadata file, the path has to start with /permanent or /experimental + + path = self.correctPath(path) + if compressed: + newMetaXML=self.compress(newMetaXML); + + ret=self.server.writeMetaDataFile(path,newMetaXML); + logging.debug("MetaDataClient: writeMetaDataFile:"+repr(ret)) + + return ret + def compress(self,txt): + return base64.encodestring(bz2.compress(string)) + + def correctPath(self,path): + #take only the path of the url which starts with /permanent or /experimental + + rs= re.search("/permanent/(.*)", path); + if rs is not None: + txt="permanent/"+rs.group(1) + else: + rs= re.search("/experimental/(.*)", path); + if rs is not None: + txt="experimental"+rs.group(1) + else: + return None + + return txt + + + + \ No newline at end of file diff -r 000000000000 -r 9f9d9be26e53 MetaDataMapping.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/MetaDataMapping.py Mon Jul 25 16:50:48 2011 +0200 @@ -0,0 +1,65 @@ +from OFS.Folder import Folder +from Products.PageTemplates.PageTemplateFile import PageTemplateFile +from Globals import package_home +from AccessControl import ClassSecurityInfo +import os.path +import urllib +import logging +import urlparse + +# TODO: which xml toolkit? +import amara +import xml.sax.saxutils +import xml.dom.minidom +import xml.etree.ElementTree as ET + +# TODO: do we need this? +#from Products.OSA_system2 import OSAS_helpers +#from Products.OSA_system2.OSAS_metadata import OSAS_Metadata,OSAS_MetadataMapping +from OSAS_metadata import OSAS_MetadataMapping + + +class MetaDataMapping(OSAS_MetadataMapping): + """Metadata mapping object representing a fixed set of sub-elements.""" + + meta_type="MetadataMapping" + manage_options = SimpleItem.manage_options+( + {'label':'Main Config','action':'changeMetadataMappingForm'}, + ) + + fields = {} + """dict containing element descriptions""" + + fieldList = [] + """list of element names in preferred order""" + + def __init__(self,id,title,fields): + """init + @param id: id + @param title: title fuer zope navigation + @param fields: dictionary mit Namen der zugelassenen Metadaten als key und Tripel als Werte (human readable, tag version,explanation + """ + self.id=id + self.title=title + for fieldName in fields.keys(): + self.fields[fieldName] = fields[fieldName] + + + + + + +manage_addMetaDataMappingForm = PageTemplateFile('zpt/addMetadataMappingForm', globals()) + +def manage_addMetaDataMapping(self,idOfObject,titleOfObject,RESPONSE=None): + """add the MetapdataMapping""" + argList={} + for arg in self.fieldList: + if not (arg in ['idOfObject','titleOfObject']): + argList[arg]=(self.REQUEST.form[arg],self.REQUEST.form['label_'+arg],self.REQUEST.form['explanation_'+arg],self.REQUEST.form['status_'+arg],self.REQUEST.form['values_'+arg]) + + newObj=MetaDataMapping(idOfObject,titleOfObject,argList) + self._setObject(idOfObject,newObj) + if RESPONSE is not None: + RESPONSE.redirect('manage_main') + diff -r 000000000000 -r 9f9d9be26e53 OSAS_metadata.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/OSAS_metadata.py Mon Jul 25 16:50:48 2011 +0200 @@ -0,0 +1,658 @@ +""" Classes for managing metadata""" + +from OFS.SimpleItem import SimpleItem +from Globals import InitializeClass,package_home +from OFS.Folder import Folder +from AccessControl import ClassSecurityInfo +from Products.PageTemplates.PageTemplateFile import PageTemplateFile +import os.path +import sys +import xml.dom.minidom +import xml.dom.pulldom +#TODO: get rid of this +from Products.OSA_system2 import OSAS_helpers +import logging + +#ersetzt logging +def logger(txt,method,txt2): + """logging""" + logging.info(txt+ txt2) + + +import string +try: + from xml.xpath import Evaluate +except: + from Ft.Xml.XPath import Evaluate +import xmlrpclib + +from types import * + +class OSAS_MetadataMapping(SimpleItem): + """Einfaches Mapping Object""" + + meta_type="OSAS_MetadataMapping__neu" + + def readFieldsFromParent(self): + """read all elements from root""" + + return self.aq_parent.fieldList + + def __init__(self,id,title,arglist): + """init + @param id: id + @param title: title fuer zope navigation + @param arglist: dictionary mit Namen der zugelassenen generische Metadaten als key und Tripel als Werte (human readable, tag version,explanation + """ + self.id=id + self.title=title + for fieldName in arglist.keys(): + setattr(self,"md_"+fieldName,arglist[fieldName]) + + + manage_options = SimpleItem.manage_options+( + {'label':'Main Config','action':'changeMetadataMappingForm'}, + ) + + + def showSetXML(self,RESPONSE=None): + """prints out the mapping as XML""" + ret=""""""%self.title + for fieldName in self.readFieldsFromParent(): + entry=getattr(self,"md_"+fieldName) + if entry[2]=="": # no explanation of this field + ret+=""""""%(fieldName,entry[0],entry[1]) + else: + ret+="""%s"""%(fieldName,entry[0],entry[1],entry[2]) + ret+="" + + if not RESPONSE: + return ret + else: + self.REQUEST.RESPONSE.setHeader('Content-Type','text/xml') + return ret + + def getValue(self,fieldName): + """get md value + @param fieldName: Bezeichnung des gesuchten Metadatums + @retunr: Value des Metadatums""" + + ret= getattr(self,"md_"+fieldName,(None,None,None,None,None)) + if len(ret)!= 4: # alte MD haben keine info ueber optional/required und listen werte + ret=ret+("","") + return ret + + + def isEmptyValue(self,fieldName): + """teste ob fielname in diesem Metadatenschema definiert ist""" + field=getattr(self,"md_"+fieldName,'') + if field[1]=='': + return 0 + else: + return 1 + + def generateMappingHash(self): + """erzeugen des dictionaries: generisches Feld -> Definition in diesem Schema""" + hash={} + for field in self.fieldList: + hash[field]=getattr(self,"md_"+field,'') + return hash + + + + def changeMetadataMappingForm(self): + """change""" + pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','changeMetadataMapping.zpt')).__of__(self) + return pt() + + def changeMetadataMapping(self,titleOfObject,RESPONSE=None): + """change""" + + self.title=titleOfObject + arglist=self.REQUEST.form + + for fieldName in self.readFieldsFromParent(): + setattr(self,"md_"+fieldName,(arglist[fieldName],arglist['label_'+fieldName],arglist['explanation_'+fieldName],arglist['status_'+fieldName],arglist['values_'+fieldName])) + + + if RESPONSE is not None: + RESPONSE.redirect('manage_main') + + manage_workspace=changeMetadataMappingForm + +def manage_addMetadataMappingForm(self): + """interface for adding the OSAS_root""" + pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addMetadataMappingForm.zpt')).__of__(self) + + return pt() + +def manage_addMetadataMapping(self,idOfObject,titleOfObject,RESPONSE=None): + """add the OSAS_root""" + + argList={} + for arg in self.fieldList: + if not (arg in ['idOfObject','titleOfObject']): + argList[arg]=(self.REQUEST.form[arg],self.REQUEST.form['label_'+arg],self.REQUEST.form['explanation_'+arg],self.REQUEST.form['status_'+arg],self.REQUEST.form['values_'+arg]) + + newObj=OSAS_MetadataMapping(idOfObject,titleOfObject,argList) + self._setObject(idOfObject,newObj) + if RESPONSE is not None: + RESPONSE.redirect('manage_main') + + + +class OSAS_Metadata(Folder): + """Foldertype enthaelt methoden zur Halbautomatischen Erstellung von Metadatenfiles""" + security=ClassSecurityInfo() + + def __init__(self,id,shortDescription,description,fields): + """initialize a new instance""" + self.id = id + self.shortDescription =shortDescription #label fuer link auf add page + self.description=description #description of the method for link page + self.fieldList=fields.split(",")[0:] + + meta_type='OSAS_Metadata__neu' + + manage_options = Folder.manage_options+( + {'label':'Main Config','action':'changeMetadataForm'}, + {'label':'Import XML Schema','action':'importXMLSchemaForm'}, + {'label':'Select Fields for Display','action':'indicateDisplayFieldsForm'}, + ) + + def showGenericXML(self,RESPONSE=None): + """show generic fields as XML""" + ret=""""""%"generic" + for field in self.fieldList: + ret+=""""""%field + + ret+="" + + if not RESPONSE: + return ret + else: + self.REQUEST.RESPONSE.setHeader('Content-Type','text/xml') + return ret + + + def showOverviewXML(self,RESPONSE=None,wrapped=False): + """gives an overview over the Metadata stored in this folder""" + ret="" + if wrapped: + ret+="""""" + ret+=""""""%self.getId() + ret+=self.showGenericXML() + for entry in self.ZopeFind(self,obj_metatypes=['OSAS_MetadataMapping__neu']): + ret+=entry[1].showSetXML() + + for entry in self.ZopeFind(self,obj_metatypes=['OSAS_Metadata__neu']): + ret+=entry[1].showOverviewXML() + + ret+="" + + if wrapped: + ret+="""""" + if not RESPONSE: + return ret + else: + RESPONSE.setHeader('Content-Type','text/xml') + return ret + + def generateMappingList(self): + """Erzeuge Mapping""" + mapping={} + + for dict in self.__dict__: + #print dict + obj=getattr(self,dict) + if hasattr(obj,'meta_type'): + logging.debug("generateMappungList read type:"+repr(obj.meta_type)) + if ((obj.meta_type=="OSAS_MetadataMapping__neu") or (obj.meta_type=="MetadataMapping")): #metadatamapping is the newer type + logging.debug("generateMappungListadded:"+repr(obj.getId())) + mapping[obj.getId()]=obj.generateMappingHash() + + return mapping + + def generateMappingForType(self,type,clean="yes"): + """erzeuge spezifisches Mapping""" + + hash=self.generateMappingList() + logging.debug("generateMappingForType:"+type) + for key in hash.keys(): + logging.debug("generateMappingForType comparing:"+key.lower()+type.lower()) + if (key.lower() == type.lower()): + if clean=="yes": + temp={} + for x in hash[key].keys(): + if not hash[key][x]=="": + temp[x]=hash[key][x] + return temp + else: + + return hash[key] + + return {} + + def getFieldList(self): + """erzeuge string aus fields""" + try: + return string.join(self.fieldList,",") + except: + return "" + + security.declarePublic('getFields') + def getFields(self): + """ausgabe der Felder""" + return self.fieldList + + def getTypeTitle(self,id): + """Title von ID""" + try: + obj=getattr(self,id) + return obj.title + except: + return id + + def getType(self,type): + """gib metadataobject type zurueck""" + + for obj in self.ZopeFind(self,obj_metatypes=['OSAS_MetadataMapping__neu']): + if obj[0]==type: + return obj + return (self.id,self) + + + + def getStoredTypes(self): + """Gebe gespeicherte typen zurueck""" + + types=[] + + for obj in self.ZopeFind(self,obj_metatypes=['OSAS_MetadataMapping__neu']): + if obj[1].title=="": + title=obj[1].id + else: + title=obj[1].title + types.append((obj[1].id, title, obj[1])) + + return types + + def indicateDisplayFieldsForm(self): + """form zur Makierung der Felder die in der Browserumgebung angezeigt werden""" + pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','indicateDisplayFieldsForm.zpt')).__of__(self) + return pt() + + def indicateDisplayFields(self,displayFields,RESPONSE=None): + """set Displayfields + @param displayFields: Liste von Felder die im Browserenvironment angezeigt werden + """ + self.displayFields=OSAS_helpers.toList(displayFields) + if RESPONSE is not None: + RESPONSE.redirect('manage_main') + + def getDisplayFieldsAsStr(self,indexMeta): + ret=[] + try: + if indexMeta and not (indexMeta==""): + dom=xml.dom.pulldom.parseString(indexMeta) + + for (event,node) in dom: + + if event == xml.dom.pulldom.START_ELEMENT and node.tagName=="bib": + dom.expandNode(node) + + try: + type=node.getAttribute('type') + mapping=getattr(self,type).generateMappingHash() + except: + type='generic' + mapping=getattr(self,type).generateMappingHash() + + for field in self.displayFields: + try: + ret.append(OSAS_helpers.getText(node.getElementsByTagName(mapping[field][0])[0].childNodes)) + except: + """nothing""" + + return "; ".join(ret) + else: + return "" + except: + return "" + def getDisplayFieldsAsStrOLD(self,indexMeta): + """Gebe display fields als string zurueck + @param path: Pfad zum Object + """ + ret=[] + try: + dom=xml.dom.minidom.parseString(indexMeta) + except: + logger("OSAS_metadata (getDisplayFieldsAsStr)",logging.INFO,"Cannot parse: %s"%indexMeta) + try: + bib = dom.getElementsByTagName("meta")[0].getElementsByTagName("bib")[0] + except: + return "" + try: + type=bib.getAttribute('type') + mapping=getattr(self,type).generateMappingHash() + except: + type='generic' + mapping=getattr(self,type).generateMappingHash() + + for field in self.displayFields: + try: + ret.append(OSAS_helpers.getText(bib.getElementsByTagName(mapping[field][0])[0].childNodes)) + except: + """nothing""" + + return "; ".join(ret) + + security.declarePublic('changeMetadataForm') + def changeMetadataForm(self): + """Main configuration""" + pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','changeMetadata.zpt')).__of__(self) + return pt() + + security.declarePublic('changeMetadata') + def changeMetadata(self,shortDescription,description,fields,RESPONSE=None): + """Change Metadata""" + self.shortDescription=shortDescription + self.description=description + self.fieldList=fields.split(",")[0:] + if RESPONSE is not None: + RESPONSE.redirect('manage_main') + + security.declarePublic('index_html') + + def importXMLSchemaForm(self): + """form""" + pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','importXMLSchemaForm.zpt')).__of__(self) + return pt() + + + def importXMLSchema(self,file,RESPONSE=None): + """import xmlschema, Metadatenschema wird eingelesen und entsprechende Metadatenmappings angelegt.""" + + dom=xml.dom.minidom.parse(file) + sets=dom.getElementsByTagName('set') + #erster schritt: anlegen der fieldlist + for set in sets: + if set.getAttribute('name')=='generic': + list=[] + for entry in set.getElementsByTagName('entry'): + list.append(entry.getAttribute('genericName')) + self.fieldList=list[0:] + + #zweiter schritt: anlegen der mapping + for set in sets: + id=set.getAttribute('name').encode('utf-8') + list=[] + argList={} + for entry in set.getElementsByTagName('entry'): + genericName=entry.getAttribute('genericName') + tag=entry.getAttribute('tag') + label=entry.getAttribute('label') + description=OSAS_helpers.getText(entry.childNodes) + argList[genericName]=(tag,label,description) + self._setObject(id,OSAS_MetadataMapping(id,id,argList)) + if RESPONSE: + RESPONSE.write("Wrote: %s"%id) + + + def createMetadataFragment(self,type,path,prefix="",presets={}): + """fragment""" + self.REQUEST.SESSION['MDF_type']=type + self.REQUEST.SESSION['MDF_path']=path + self.REQUEST.SESSION['MDF_prefix']=prefix + self.REQUEST.SESSION['MDF_presets']=presets + + pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addMetadataForm_fragment.zpt')).__of__(self) + return pt() + + def createMetadataForm(self,type="",path=""): + """createMetadataForm""" + self.REQUEST.SESSION['MDF_type']=type + self.REQUEST.SESSION['MDF_path']=path + + + pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addMetadataForm_template.zpt')).__of__(self) + return pt() + + def readMetadata(self,MDF_path,MDF_type,MDF_prefix="",MDF_xpathStart="", MDF_addPath=None,MDF_identifyFields=None,newdoc=None,actualNode=None): + + if type(MDF_path)==ListType: + MDF_path=MDF_path[0] + + indexFile=os.path.join(MDF_path,'index.meta') + + server=xmlrpclib.Server(self.serverUrl) + + documentStr=server.getFile(indexFile) + + if documentStr: + newdoc=xml.dom.minidom.parseString(documentStr) + dom=newdoc.documentElement + actualNode=dom + else: + return {} + + if self.containerTag=="": + containerTag="doc" + else: + containerTag=self.containerTag + + if MDF_xpathStart=="": + dom=newdoc.documentElement + actualNode=dom + else: + #try to find xpath + if MDF_identifyFields: + query=[] + for field in MDF_identifyFields.keys(): + + query.append("""(%s="%s")"""%(field,MDF_identifyFields[field])) + querystr=" and ".join(query) + xpathStr=MDF_xpathStart+"[%s]"%querystr + + else: + xpathStr=MDF_xpathStart + + + xpathNodes=Evaluate(xpathStr,actualNode) + + + if len(xpathNodes)>0: + actualNode=xpathNodes[0] + else: + return {} + + ret={} + for node in actualNode.childNodes: + name=node.tagName + text=OSAS_helpers.getText(node.childNodes) + ret[name]=text + + return ret + + def writeMetadata(self,MDF_path,MDF_type,form,MDF_prefix="",MDF_xpathStart="", MDF_addPath=None,MDF_identifyFields=None,newdoc=None,actualNode=None): + """writeMetadata""" + #MDF_path="/tmp/index.meta" + if type(MDF_path)==ListType: + MDF_path=MDF_path[0] + + + indexFile=os.path.join(MDF_path,'index.meta') + + server=xmlrpclib.Server(self.serverUrl) + if newdoc: + if not actualNode: actualNode=newdoc + dom=newdoc + else: + documentStr=server.getFile(indexFile) + + if documentStr: + + newdoc=xml.dom.minidom.parseString(documentStr) + dom=newdoc.documentElement + actualNode=dom + + + else: + + impl=xml.dom.minidom.getDOMImplementation() + newdoc=None + + + if self.containerTag=="": + containerTag="doc" + else: + containerTag=self.containerTag + + create=None + if MDF_xpathStart=="": + if not newdoc: + newdoc=impl.createDocument(None,containerTag,None) + dom=newdoc.documentElement + actualNode=dom + else: + + #try to find xpath + if MDF_identifyFields: + query=[] + for field in MDF_identifyFields: + + query.append("""(%s="%s")"""%(field,form[MDF_prefix+"MD_"+field])) + querystr=" and ".join(query) + xpathStr=MDF_xpathStart+"[%s]"%querystr + + else: + xpathStr=MDF_xpathStart + + + xpathNodes=Evaluate(xpathStr,actualNode) + + + + + if len(xpathNodes)>0: + actualNode=xpathNodes[0] + + else: + #existiert nicht dann neue erzeugen + + if len(Evaluate(MDF_xpathStart,dom))>0: + + create=True + + splitted=MDF_xpathStart.split("/") + base="" + for element in splitted: + + if not (element=="") and not (element==containerTag): + base="/".join([base,element]) + + if not newdoc: + newdoc=impl.createDocument(None,element,None) + actualNode=newdoc.documentElement + dom=actualNode + else: + changed=None + + if not (MDF_addPath==base): + + + for childNode in actualNode.childNodes: + if getattr(childNode,'tagName','')==element: + actualNode=childNode + changed=1 + + if (os.path.normpath("/".join(["",containerTag,base]))==MDF_xpathStart) and create: + actualNode=actualNode.parentNode + changed=None + + if not changed: + namenode=newdoc.createElement(element) + + actualNode.appendChild(namenode) + actualNode=namenode + + + + + for name in self.REQUEST.form.keys(): + length=len(MDF_prefix) + if MDF_type and not (MDF_type == ""): + actualNode.setAttribute("type",MDF_type) + if name[0:3+length]==MDF_prefix+"MD_": + tagName=name[3+length:] + + #CHECK if element exists + for childNode in actualNode.childNodes: + if getattr(childNode,'tagName','')==tagName: + actualNode.removeChild(childNode).unlink() + + namenode=newdoc.createElement(tagName) + namenodetext=newdoc.createTextNode(self.REQUEST.form[name]) + namenode.appendChild(namenodetext) + actualNode.appendChild(namenode) + + ret=newdoc.toxml(encoding='utf-8') + zLOG.LOG("OSAS_metadata (writeMetadata)",zLOG.INFO,"write: %s"%ret) + + + server.writeMetaDataFile(indexFile,ret) + + + return newdoc,actualNode + + def writeMetadataFile(self,MDF_path,MDF_type,MDF_xpathStart="",newdoc=None,actualNode=None): + """writeMetaFile""" + + return self.writeMetadata(MDF_path,MDF_type,self.REQUEST.form,MDF_xpathStart=MDF_xpathStart,newdoc=newdoc,actualNode=actualNode) + + + def isEmptyValue(self,fieldName): + """im generischen fall stets falsch""" + return 1 + + def getValue(self,fieldName): + """im generischen fall gleich fieldname""" + return fieldName,fieldName,"","","" + + def getList(self,list): + """return list""" + + if list=="": + return None + listsplit=[i.rstrip() for i in list.split("\n")] + return listsplit + + def showHelp(self,refType,genericTag): + """helptext""" + for reference in self.ZopeFind(self): + if reference[1].title==refType: + text=getattr(reference[1],'md_'+genericTag)[2] + return text + return "NO EXPLANATION" + + def showHelpTag(self,url,reftype,item): + """gen javascript for showhelp""" + url2=url+'/showHelp?refType=%s&genericTag=%s'%(reftype,item) + ret="""javascript:wd=window.open(\'%s\',\'Help\',\'width=300,height=250\');void(\'\');wd.focus();"""%url2 + return ret + + +def manage_addMetadataForm(self): + """interface for adding the OSAS_add_Metadata""" + pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addMetadataForm.zpt')).__of__(self) + return pt() + +def manage_addMetadata(self,id,shortDescription,description,fields,RESPONSE=None): + """add the OSAS_root""" + newObj=OSAS_Metadata(id,shortDescription,description,fields) + self.Destination()._setObject(id,newObj) + if RESPONSE is not None: + RESPONSE.redirect('manage_main') + + + diff -r 000000000000 -r 9f9d9be26e53 SrvTxtUtils.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/SrvTxtUtils.py Mon Jul 25 16:50:48 2011 +0200 @@ -0,0 +1,83 @@ +"""Utility methods for handling XML, reading HTTP, etc""" + +import sys +import urllib +import urllib2 +import logging + + +srvTxtUtilsVersion = "1.0" + +def getInt(number, default=0): + """returns always an int (0 in case of problems)""" + try: + return int(number) + except: + return int(default) + +def getText(node): + """returns all text content of a node and its subnodes""" + if node is None: + return "" + # ElementTree: + text = node.text or "" + for e in node: + text += gettext(e) + if e.tail: + text += e.tail + + # 4Suite: + #nodelist=node.childNodes + #text = "" + #for n in nodelist: + # if n.nodeType == node.TEXT_NODE: + # text = text + n.data + + return text + + + +def getHttpData(url, data=None, num_tries=3, timeout=10): + """returns result from url+data HTTP request""" + # we do GET (by appending data to url) + if isinstance(data, str) or isinstance(data, unicode): + # if data is string then append + url = "%s?%s"%(url,data) + elif isinstance(data, dict) or isinstance(data, list) or isinstance(data, tuple): + # urlencode + url = "%s?%s"%(url,urllib.urlencode(data)) + + response = None + errmsg = None + for cnt in range(num_tries): + try: + logging.debug("getHttpData(#%s %ss) url=%s"%(cnt+1,timeout,url)) + if sys.version_info < (2, 6): + # set timeout on socket -- ugly :-( + import socket + socket.setdefaulttimeout(float(timeout)) + response = urllib2.urlopen(url) + else: + # timeout as parameter + response = urllib2.urlopen(url,timeout=float(timeout)) + # check result? + break + except urllib2.HTTPError, e: + logging.error("getHttpData: HTTP error(%s): %s"%(e.code,e)) + errmsg = str(e) + # stop trying + break + except urllib2.URLError, e: + logging.error("getHttpData: URLLIB error(%s): %s"%(e.reason,e)) + errmsg = str(e) + # stop trying + #break + + if response is not None: + data = response.read() + response.close() + return data + + raise IOError("ERROR fetching HTTP data from %s: %s"%(url,errmsg)) + #return None + diff -r 000000000000 -r 9f9d9be26e53 __init__.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/__init__.py Mon Jul 25 16:50:48 2011 +0200 @@ -0,0 +1,21 @@ +import MetaData +import MetaDataMapping + +def initialize(context): + """initialize Metadata""" + + context.registerClass( + MetaData.MetaData, + constructors = ( + MetaData.manage_addMetaDataForm, + MetaData.manage_addMetaData + ) + ) + + context.registerClass( + MetaDataMapping.MetaDataMapping, + constructors = ( + MetaDataMapping.manage_addMetaDataMappingForm, + MetaDataMapping.manage_addMetaDataMapping + ) + ) \ No newline at end of file diff -r 000000000000 -r 9f9d9be26e53 zpt/addMetadataForm.zpt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/zpt/addMetadataForm.zpt Mon Jul 25 16:50:48 2011 +0200 @@ -0,0 +1,18 @@ + + +
+ ID:

+ short description:

+ Description:

+ Generic Metafields + +

+ Name of the container Tag (optional)

+

+
+ + diff -r 000000000000 -r 9f9d9be26e53 zpt/addMetadataMappingForm.zpt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/zpt/addMetadataMappingForm.zpt Mon Jul 25 16:50:48 2011 +0200 @@ -0,0 +1,57 @@ + + +

Add a Metadata Mapping

+
+ ID:

+ Title (=Reference Type):

+ + + + + + + + + + + + + + + + + + +
+ Tag-Name + + Label + + Explanation + + Optional + + Valuelist +
+ + + + + + + + + + + + + +
+

+
+ + diff -r 000000000000 -r 9f9d9be26e53 zpt/changeMetadata.zpt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/zpt/changeMetadata.zpt Mon Jul 25 16:50:48 2011 +0200 @@ -0,0 +1,24 @@ + + +
+ ShortDescription:

+ Description:

+ Generic Metafields + + +

+ + Container Tag

+ + Server für xml files (e.g. url des Textservlets von digilib als python formaated String mit %s wo der Pfad eingesetzt werden soll.) +
+

+ +

+
+ + diff -r 000000000000 -r 9f9d9be26e53 zpt/importMetaDataExportXML.zpt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/zpt/importMetaDataExportXML.zpt Mon Jul 25 16:50:48 2011 +0200 @@ -0,0 +1,10 @@ + + +
+ +
+upload: + +
+ +