version 1.238.2.3, 2011/07/15 19:45:52
|
version 1.238.2.10, 2011/08/05 17:04:20
|
Line 1
|
Line 1
|
|
|
from OFS.SimpleItem import SimpleItem |
from OFS.SimpleItem import SimpleItem |
from Products.PageTemplates.PageTemplateFile import PageTemplateFile |
from Products.PageTemplates.PageTemplateFile import PageTemplateFile |
|
|
Line 12 import xml.etree.ElementTree as ET
|
Line 11 import xml.etree.ElementTree as ET
|
import re |
import re |
import logging |
import logging |
import urllib |
import urllib |
import documentViewer |
|
#from documentViewer import getTextFromNode, serializeNode |
|
|
|
def intOr0(s, default=0): |
|
"""convert s to int or return default""" |
|
try: |
|
return int(s) |
|
except: |
|
return default |
|
|
|
def getText(node): |
from SrvTxtUtils import getInt, getText, getHttpData |
"""get the cdata content of a node""" |
|
if node is None: |
|
return "" |
|
# ET: |
|
text = node.text or "" |
|
for e in node: |
|
text += gettext(e) |
|
if e.tail: |
|
text += e.tail |
|
|
|
return text |
|
|
|
def serialize(node): |
def serialize(node): |
"""returns a string containing an XML snippet of node""" |
"""returns a string containing an XML snippet of node""" |
Line 90 class MpdlXmlTextServer(SimpleItem):
|
Line 69 class MpdlXmlTextServer(SimpleItem):
|
manage_changeMpdlXmlTextServerForm = PageTemplateFile("zpt/manage_changeMpdlXmlTextServer", globals()) |
manage_changeMpdlXmlTextServerForm = PageTemplateFile("zpt/manage_changeMpdlXmlTextServer", globals()) |
|
|
def __init__(self,id,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de/mpdl/interface/", serverName=None, timeout=40): |
def __init__(self,id,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de/mpdl/interface/", serverName=None, timeout=40): |
|
|
"""constructor""" |
"""constructor""" |
self.id=id |
self.id=id |
self.title=title |
self.title=title |
Line 102 class MpdlXmlTextServer(SimpleItem):
|
Line 80 class MpdlXmlTextServer(SimpleItem):
|
|
|
def getHttpData(self, url, data=None): |
def getHttpData(self, url, data=None): |
"""returns result from url+data HTTP request""" |
"""returns result from url+data HTTP request""" |
return documentViewer.getHttpData(url,data,timeout=self.timeout) |
return getHttpData(url,data,timeout=self.timeout) |
|
|
def getServerData(self, method, data=None): |
def getServerData(self, method, data=None): |
"""returns result from text server for method+data""" |
"""returns result from text server for method+data""" |
url = self.serverUrl+method |
url = self.serverUrl+method |
return documentViewer.getHttpData(url,data,timeout=self.timeout) |
return getHttpData(url,data,timeout=self.timeout) |
|
|
# WTF: what does this really do? can it be integrated in getPage? |
# WTF: what does this really do? can it be integrated in getPage? |
def getSearch(self, pageinfo=None, docinfo=None): |
def getSearch(self, pageinfo=None, docinfo=None): |
Line 248 class MpdlXmlTextServer(SimpleItem):
|
Line 226 class MpdlXmlTextServer(SimpleItem):
|
|
|
def processPageInfo(self, dom, docinfo, pageinfo): |
def processPageInfo(self, dom, docinfo, pageinfo): |
"""processes page info divs from dom and stores in docinfo and pageinfo""" |
"""processes page info divs from dom and stores in docinfo and pageinfo""" |
# process all toplevel divs |
# assume first second level div is pageMeta |
alldivs = dom.findall(".//div") |
alldivs = dom.find("div") |
pagediv = None |
|
|
if alldivs is None or alldivs.get('class', '') != 'pageMeta': |
|
logging.error("processPageInfo: pageMeta div not found!") |
|
return |
|
|
for div in alldivs: |
for div in alldivs: |
dc = div.get('class') |
dc = div.get('class') |
|
|
# page content div |
|
if dc == 'pageContent': |
|
pagediv = div |
|
|
|
# pageNumberOrig |
# pageNumberOrig |
elif dc == 'pageNumberOrig': |
if dc == 'pageNumberOrig': |
pageinfo['pageNumberOrig'] = div.text |
pageinfo['pageNumberOrig'] = div.text |
|
|
# pageNumberOrigNorm |
# pageNumberOrigNorm |
elif dc == 'pageNumberOrigNorm': |
elif dc == 'pageNumberOrigNorm': |
pageinfo['pageNumberOrigNorm'] = div.text |
pageinfo['pageNumberOrigNorm'] = div.text |
|
|
# pageNumberOrigNorm |
# pageHeaderTitle |
|
elif dc == 'pageHeaderTitle': |
|
pageinfo['pageHeaderTitle'] = div.text |
|
|
|
# numFigureEntries |
elif dc == 'countFigureEntries': |
elif dc == 'countFigureEntries': |
docinfo['countFigureEntries'] = intOr0(div.text) |
docinfo['numFigureEntries'] = getInt(div.text) |
|
|
# pageNumberOrigNorm |
# numTocEntries |
elif dc == 'countTocEntries': |
elif dc == 'countTocEntries': |
# WTF: s1 = int(s)/30+1 |
# WTF: s1 = int(s)/30+1 |
docinfo['countTocEntries'] = intOr0(div.text) |
docinfo['numTocEntries'] = getInt(div.text) |
|
|
|
# numPlaces |
|
elif dc == 'countPlaces': |
|
docinfo['numPlaces'] = getInt(div.text) |
|
|
# numTextPages |
# numTextPages |
elif dc == 'countPages': |
elif dc == 'countPages': |
np = intOr0(div.text) |
np = getInt(div.text) |
if np > 0: |
if np > 0: |
docinfo['numTextPages'] = np |
docinfo['numTextPages'] = np |
if docinfo.get('numPages', 0) == 0: |
if docinfo.get('numPages', 0) == 0: |
# seems to be text-only |
# seems to be text-only - update page count |
docinfo['numTextPages'] = np |
docinfo['numPages'] = np |
pageinfo['end'] = min(pageinfo['end'], np) |
pageinfo['end'] = min(pageinfo['end'], np) |
pageinfo['numgroups'] = int(np / pageinfo['groupsize']) |
pageinfo['numgroups'] = int(np / pageinfo['groupsize']) |
if np % pageinfo['groupsize'] > 0: |
if np % pageinfo['groupsize'] > 0: |
pageinfo['numgroups'] += 1 |
pageinfo['numgroups'] += 1 |
|
|
|
#logging.debug("processPageInfo: pageinfo=%s"%repr(pageinfo)) |
return |
return |
|
|
|
|
def getTextPage(self, mode="text_dict", pn=1, docinfo=None, pageinfo=None): |
def getTextPage(self, mode="text", pn=1, docinfo=None, pageinfo=None): |
"""returns single page from fulltext""" |
"""returns single page from fulltext""" |
logging.debug("getTextPage mode=%s, pn=%s"%(mode,pn)) |
logging.debug("getTextPage mode=%s, pn=%s"%(mode,pn)) |
# check for cached text -- but this shouldn't be called twice |
# check for cached text -- but this shouldn't be called twice |
Line 314 class MpdlXmlTextServer(SimpleItem):
|
Line 301 class MpdlXmlTextServer(SimpleItem):
|
characterNormalization = pageinfo.get('characterNormalization', None) |
characterNormalization = pageinfo.get('characterNormalization', None) |
selfurl = docinfo['viewerUrl'] |
selfurl = docinfo['viewerUrl'] |
|
|
if mode == "text_dict": |
if mode == "dict" or mode == "text_dict": |
# text_dict is called textPollux in the backend |
# dict is called textPollux in the backend |
textmode = "textPollux" |
textmode = "textPollux" |
|
elif not mode: |
|
# default is text |
|
mode = "text" |
|
textmode = "text" |
else: |
else: |
textmode = mode |
textmode = mode |
|
|
Line 332 class MpdlXmlTextServer(SimpleItem):
|
Line 323 class MpdlXmlTextServer(SimpleItem):
|
# page content is in <div class="pageContent"> |
# page content is in <div class="pageContent"> |
pagediv = None |
pagediv = None |
# ElementTree 1.2 in Python 2.6 can't do div[@class='pageContent'] |
# ElementTree 1.2 in Python 2.6 can't do div[@class='pageContent'] |
alldivs = dom.findall(".//div") |
# so we look at the second level divs |
|
alldivs = dom.findall("div") |
for div in alldivs: |
for div in alldivs: |
dc = div.get('class') |
dc = div.get('class') |
# page content div |
# page content div |
Line 342 class MpdlXmlTextServer(SimpleItem):
|
Line 334 class MpdlXmlTextServer(SimpleItem):
|
|
|
# plain text mode |
# plain text mode |
if mode == "text": |
if mode == "text": |
if pagediv: |
if pagediv is not None: |
links = pagediv.findall(".//a") |
links = pagediv.findall(".//a") |
for l in links: |
for l in links: |
href = l.get('href') |
href = l.get('href') |
Line 353 class MpdlXmlTextServer(SimpleItem):
|
Line 345 class MpdlXmlTextServer(SimpleItem):
|
return serialize(pagediv) |
return serialize(pagediv) |
|
|
# text-with-links mode |
# text-with-links mode |
elif mode == "text_dict": |
elif mode == "dict": |
if pagediv: |
if pagediv is not None: |
# check all a-tags |
# check all a-tags |
links = pagediv.findall(".//a") |
links = pagediv.findall(".//a") |
for l in links: |
for l in links: |
Line 384 class MpdlXmlTextServer(SimpleItem):
|
Line 376 class MpdlXmlTextServer(SimpleItem):
|
|
|
# xml mode |
# xml mode |
elif mode == "xml": |
elif mode == "xml": |
if pagediv: |
if pagediv is not None: |
return serialize(pagediv) |
return serialize(pagediv) |
|
|
# pureXml mode |
# pureXml mode |
elif mode == "pureXml": |
elif mode == "pureXml": |
if pagediv: |
if pagediv is not None: |
return serialize(pagediv) |
return serialize(pagediv) |
|
|
# gis mode |
# gis mode |
elif mode == "gis": |
elif mode == "gis": |
name = docinfo['name'] |
name = docinfo['name'] |
if pagediv: |
if pagediv is not None: |
# check all a-tags |
# check all a-tags |
links = pagediv.findall(".//a") |
links = pagediv.findall(".//a") |
for l in links: |
for l in links: |
Line 504 class MpdlXmlTextServer(SimpleItem):
|
Line 496 class MpdlXmlTextServer(SimpleItem):
|
pagediv = div |
pagediv = div |
|
|
elif dc == 'queryResultHits': |
elif dc == 'queryResultHits': |
docinfo['tocSize_%s'%mode] = intOr0(div.text) |
docinfo['tocSize_%s'%mode] = getInt(div.text) |
|
|
if pagediv: |
if pagediv: |
# # split xml in chunks |
|
# tocs = [] |
|
# tocdivs = pagediv.findall('div') |
|
# for p in zip(tocdivs[::2], tocdivs[1::2]): |
|
# toc = serialize(p[0]) |
|
# toc += serialize(p[1]) |
|
# tocs.append(toc) |
|
# logging.debug("pair: %s"%(toc)) |
|
# store XML in docinfo |
# store XML in docinfo |
docinfo['tocXML_%s'%mode] = ET.tostring(pagediv, 'UTF-8') |
docinfo['tocXML_%s'%mode] = ET.tostring(pagediv, 'UTF-8') |
|
|