version 1.238.2.7, 2011/08/04 14:57:02
|
version 1.238.2.8, 2011/08/05 09:24:42
|
Line 228 class MpdlXmlTextServer(SimpleItem):
|
Line 228 class MpdlXmlTextServer(SimpleItem):
|
"""processes page info divs from dom and stores in docinfo and pageinfo""" |
"""processes page info divs from dom and stores in docinfo and pageinfo""" |
# assume first second level div is pageMeta |
# assume first second level div is pageMeta |
alldivs = dom.find("div") |
alldivs = dom.find("div") |
|
|
|
if alldivs is None or alldivs.get('class', '') != 'pageMeta': |
|
logging.error("processPageInfo: pageMeta div not found!") |
|
return |
|
|
for div in alldivs: |
for div in alldivs: |
dc = div.get('class') |
dc = div.get('class') |
|
|
Line 265 class MpdlXmlTextServer(SimpleItem):
|
Line 270 class MpdlXmlTextServer(SimpleItem):
|
if np % pageinfo['groupsize'] > 0: |
if np % pageinfo['groupsize'] > 0: |
pageinfo['numgroups'] += 1 |
pageinfo['numgroups'] += 1 |
|
|
|
#logging.debug("processPageInfo: pageinfo=%s"%repr(pageinfo)) |
return |
return |
|
|
|
|
Line 291 class MpdlXmlTextServer(SimpleItem):
|
Line 297 class MpdlXmlTextServer(SimpleItem):
|
characterNormalization = pageinfo.get('characterNormalization', None) |
characterNormalization = pageinfo.get('characterNormalization', None) |
selfurl = docinfo['viewerUrl'] |
selfurl = docinfo['viewerUrl'] |
|
|
if mode == "dict": |
if mode == "dict" or mode == "text_dict": |
# text_dict is called textPollux in the backend |
# dict is called textPollux in the backend |
textmode = "textPollux" |
textmode = "textPollux" |
elif not mode: |
elif not mode: |
# default is text |
# default is text |
Line 323 class MpdlXmlTextServer(SimpleItem):
|
Line 329 class MpdlXmlTextServer(SimpleItem):
|
|
|
# plain text mode |
# plain text mode |
if mode == "text": |
if mode == "text": |
if pagediv: |
if pagediv is not None: |
links = pagediv.findall(".//a") |
links = pagediv.findall(".//a") |
for l in links: |
for l in links: |
href = l.get('href') |
href = l.get('href') |
Line 335 class MpdlXmlTextServer(SimpleItem):
|
Line 341 class MpdlXmlTextServer(SimpleItem):
|
|
|
# text-with-links mode |
# text-with-links mode |
elif mode == "text_dict": |
elif mode == "text_dict": |
if pagediv: |
if pagediv is not None: |
# check all a-tags |
# check all a-tags |
links = pagediv.findall(".//a") |
links = pagediv.findall(".//a") |
for l in links: |
for l in links: |
Line 365 class MpdlXmlTextServer(SimpleItem):
|
Line 371 class MpdlXmlTextServer(SimpleItem):
|
|
|
# xml mode |
# xml mode |
elif mode == "xml": |
elif mode == "xml": |
if pagediv: |
if pagediv is not None: |
return serialize(pagediv) |
return serialize(pagediv) |
|
|
# pureXml mode |
# pureXml mode |
elif mode == "pureXml": |
elif mode == "pureXml": |
if pagediv: |
if pagediv is not None: |
return serialize(pagediv) |
return serialize(pagediv) |
|
|
# gis mode |
# gis mode |
elif mode == "gis": |
elif mode == "gis": |
name = docinfo['name'] |
name = docinfo['name'] |
if pagediv: |
if pagediv is not None: |
# check all a-tags |
# check all a-tags |
links = pagediv.findall(".//a") |
links = pagediv.findall(".//a") |
for l in links: |
for l in links: |