--- documentViewer/MpdlXmlTextServer.py 2011/07/15 19:45:52 1.238.2.3 +++ documentViewer/MpdlXmlTextServer.py 2011/08/05 13:31:27 1.238.2.9 @@ -1,4 +1,3 @@ - from OFS.SimpleItem import SimpleItem from Products.PageTemplates.PageTemplateFile import PageTemplateFile @@ -12,28 +11,8 @@ import xml.etree.ElementTree as ET import re import logging import urllib -import documentViewer -#from documentViewer import getTextFromNode, serializeNode - -def intOr0(s, default=0): - """convert s to int or return default""" - try: - return int(s) - except: - return default -def getText(node): - """get the cdata content of a node""" - if node is None: - return "" - # ET: - text = node.text or "" - for e in node: - text += gettext(e) - if e.tail: - text += e.tail - - return text +from SrvTxtUtils import getInt, getText, getHttpData def serialize(node): """returns a string containing an XML snippet of node""" @@ -90,7 +69,6 @@ class MpdlXmlTextServer(SimpleItem): manage_changeMpdlXmlTextServerForm = PageTemplateFile("zpt/manage_changeMpdlXmlTextServer", globals()) def __init__(self,id,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de/mpdl/interface/", serverName=None, timeout=40): - """constructor""" self.id=id self.title=title @@ -102,12 +80,12 @@ class MpdlXmlTextServer(SimpleItem): def getHttpData(self, url, data=None): """returns result from url+data HTTP request""" - return documentViewer.getHttpData(url,data,timeout=self.timeout) + return getHttpData(url,data,timeout=self.timeout) def getServerData(self, method, data=None): """returns result from text server for method+data""" url = self.serverUrl+method - return documentViewer.getHttpData(url,data,timeout=self.timeout) + return getHttpData(url,data,timeout=self.timeout) # WTF: what does this really do? can it be integrated in getPage? def getSearch(self, pageinfo=None, docinfo=None): @@ -248,50 +226,55 @@ class MpdlXmlTextServer(SimpleItem): def processPageInfo(self, dom, docinfo, pageinfo): """processes page info divs from dom and stores in docinfo and pageinfo""" - # process all toplevel divs - alldivs = dom.findall(".//div") - pagediv = None + # assume first second level div is pageMeta + alldivs = dom.find("div") + + if alldivs is None or alldivs.get('class', '') != 'pageMeta': + logging.error("processPageInfo: pageMeta div not found!") + return + for div in alldivs: dc = div.get('class') - # page content div - if dc == 'pageContent': - pagediv = div - # pageNumberOrig - elif dc == 'pageNumberOrig': + if dc == 'pageNumberOrig': pageinfo['pageNumberOrig'] = div.text # pageNumberOrigNorm elif dc == 'pageNumberOrigNorm': pageinfo['pageNumberOrigNorm'] = div.text - # pageNumberOrigNorm + # pageHeaderTitle + elif dc == 'pageHeaderTitle': + pageinfo['pageHeaderTitle'] = div.text + + # numFigureEntries elif dc == 'countFigureEntries': - docinfo['countFigureEntries'] = intOr0(div.text) + docinfo['numFigureEntries'] = getInt(div.text) - # pageNumberOrigNorm + # numTocEntries elif dc == 'countTocEntries': # WTF: s1 = int(s)/30+1 - docinfo['countTocEntries'] = intOr0(div.text) + docinfo['numTocEntries'] = getInt(div.text) # numTextPages elif dc == 'countPages': - np = intOr0(div.text) + np = getInt(div.text) if np > 0: docinfo['numTextPages'] = np if docinfo.get('numPages', 0) == 0: - # seems to be text-only - docinfo['numTextPages'] = np + # seems to be text-only - update page count + docinfo['numPages'] = np pageinfo['end'] = min(pageinfo['end'], np) pageinfo['numgroups'] = int(np / pageinfo['groupsize']) if np % pageinfo['groupsize'] > 0: pageinfo['numgroups'] += 1 - + + #logging.debug("processPageInfo: pageinfo=%s"%repr(pageinfo)) return - def getTextPage(self, mode="text_dict", pn=1, docinfo=None, pageinfo=None): + def getTextPage(self, mode="text", pn=1, docinfo=None, pageinfo=None): """returns single page from fulltext""" logging.debug("getTextPage mode=%s, pn=%s"%(mode,pn)) # check for cached text -- but this shouldn't be called twice @@ -314,9 +297,12 @@ class MpdlXmlTextServer(SimpleItem): characterNormalization = pageinfo.get('characterNormalization', None) selfurl = docinfo['viewerUrl'] - if mode == "text_dict": - # text_dict is called textPollux in the backend + if mode == "dict" or mode == "text_dict": + # dict is called textPollux in the backend textmode = "textPollux" + elif not mode: + # default is text + textmode = "text" else: textmode = mode @@ -332,7 +318,8 @@ class MpdlXmlTextServer(SimpleItem): # page content is in
pagediv = None # ElementTree 1.2 in Python 2.6 can't do div[@class='pageContent'] - alldivs = dom.findall(".//div") + # so we look at the second level divs + alldivs = dom.findall("div") for div in alldivs: dc = div.get('class') # page content div @@ -342,7 +329,7 @@ class MpdlXmlTextServer(SimpleItem): # plain text mode if mode == "text": - if pagediv: + if pagediv is not None: links = pagediv.findall(".//a") for l in links: href = l.get('href') @@ -354,7 +341,7 @@ class MpdlXmlTextServer(SimpleItem): # text-with-links mode elif mode == "text_dict": - if pagediv: + if pagediv is not None: # check all a-tags links = pagediv.findall(".//a") for l in links: @@ -384,18 +371,18 @@ class MpdlXmlTextServer(SimpleItem): # xml mode elif mode == "xml": - if pagediv: + if pagediv is not None: return serialize(pagediv) # pureXml mode elif mode == "pureXml": - if pagediv: + if pagediv is not None: return serialize(pagediv) # gis mode elif mode == "gis": name = docinfo['name'] - if pagediv: + if pagediv is not None: # check all a-tags links = pagediv.findall(".//a") for l in links: @@ -504,17 +491,9 @@ class MpdlXmlTextServer(SimpleItem): pagediv = div elif dc == 'queryResultHits': - docinfo['tocSize_%s'%mode] = intOr0(div.text) + docinfo['tocSize_%s'%mode] = getInt(div.text) if pagediv: -# # split xml in chunks -# tocs = [] -# tocdivs = pagediv.findall('div') -# for p in zip(tocdivs[::2], tocdivs[1::2]): -# toc = serialize(p[0]) -# toc += serialize(p[1]) -# tocs.append(toc) -# logging.debug("pair: %s"%(toc)) # store XML in docinfo docinfo['tocXML_%s'%mode] = ET.tostring(pagediv, 'UTF-8')