1 | from OFS.SimpleItem import SimpleItem |
---|
2 | from Products.PageTemplates.PageTemplateFile import PageTemplateFile |
---|
3 | |
---|
4 | import xml.etree.ElementTree as ET |
---|
5 | |
---|
6 | import re |
---|
7 | import logging |
---|
8 | import urllib |
---|
9 | import urlparse |
---|
10 | import base64 |
---|
11 | |
---|
12 | from datetime import datetime |
---|
13 | |
---|
14 | from SrvTxtUtils import getInt, getText, getHttpData |
---|
15 | |
---|
16 | def serialize(node): |
---|
17 | """returns a string containing an XML snippet of node""" |
---|
18 | s = ET.tostring(node, 'UTF-8') |
---|
19 | # snip off XML declaration |
---|
20 | if s.startswith('<?xml'): |
---|
21 | i = s.find('?>') |
---|
22 | return s[i+3:] |
---|
23 | |
---|
24 | return s |
---|
25 | |
---|
26 | |
---|
27 | class MpiwgXmlTextServer(SimpleItem): |
---|
28 | """TextServer implementation for MPIWG-XML server""" |
---|
29 | meta_type="MPIWG-XML TextServer" |
---|
30 | |
---|
31 | manage_options=( |
---|
32 | {'label':'Config','action':'manage_changeMpiwgXmlTextServerForm'}, |
---|
33 | )+SimpleItem.manage_options |
---|
34 | |
---|
35 | manage_changeMpiwgXmlTextServerForm = PageTemplateFile("zpt/manage_changeMpiwgXmlTextServer", globals()) |
---|
36 | |
---|
37 | def __init__(self,id,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de/mpiwg-mpdl-cms-web/", timeout=40, serverName=None, repositoryType='production'): |
---|
38 | """constructor""" |
---|
39 | self.id=id |
---|
40 | self.title=title |
---|
41 | self.timeout = timeout |
---|
42 | self.repositoryType = repositoryType |
---|
43 | if serverName is None: |
---|
44 | self.serverUrl = serverUrl |
---|
45 | else: |
---|
46 | self.serverUrl = "http://%s/mpiwg-mpdl-cms-web/"%serverName |
---|
47 | |
---|
48 | def getHttpData(self, url, data=None): |
---|
49 | """returns result from url+data HTTP request""" |
---|
50 | return getHttpData(url,data,timeout=self.timeout) |
---|
51 | |
---|
52 | def getServerData(self, method, data=None): |
---|
53 | """returns result from text server for method+data""" |
---|
54 | url = self.serverUrl+method |
---|
55 | return getHttpData(url,data,timeout=self.timeout) |
---|
56 | |
---|
57 | |
---|
58 | def getRepositoryType(self): |
---|
59 | """returns the repository type, e.g. 'production'""" |
---|
60 | return getattr(self, 'repositoryType', None) |
---|
61 | |
---|
62 | def getTextDownloadUrl(self, type='xml', docinfo=None): |
---|
63 | """returns a URL to download the current text""" |
---|
64 | docpath = docinfo.get('textURLPath', None) |
---|
65 | if not docpath: |
---|
66 | return None |
---|
67 | |
---|
68 | docpath = docpath.replace('.xml','.'+type) |
---|
69 | url = '%sdoc/GetDocument?id=%s'%(self.serverUrl.replace('interface/',''), docpath) |
---|
70 | return url |
---|
71 | |
---|
72 | |
---|
73 | def getPlacesOnPage(self, docinfo=None, pn=None): |
---|
74 | """Returns list of GIS places of page pn""" |
---|
75 | #FIXME! |
---|
76 | docpath = docinfo.get('textURLPath',None) |
---|
77 | if not docpath: |
---|
78 | return None |
---|
79 | |
---|
80 | places=[] |
---|
81 | text=self.getServerData("xpath.xql", "document=%s&xpath=//place&pn=%s"%(docpath,pn)) |
---|
82 | dom = ET.fromstring(text) |
---|
83 | result = dom.findall(".//resultPage/place") |
---|
84 | for l in result: |
---|
85 | id = l.get("id") |
---|
86 | name = l.text |
---|
87 | place = {'id': id, 'name': name} |
---|
88 | places.append(place) |
---|
89 | |
---|
90 | return places |
---|
91 | |
---|
92 | |
---|
93 | def getTextInfo(self, mode=None, docinfo=None): |
---|
94 | """reads document info, including page concordance, from text server""" |
---|
95 | logging.debug("getTextInfo mode=%s"%mode) |
---|
96 | |
---|
97 | field = '' |
---|
98 | if mode in ['pages', 'toc', 'figures', 'handwritten']: |
---|
99 | # translate mode to field param |
---|
100 | field = '&field=%s'%mode |
---|
101 | else: |
---|
102 | mode = None |
---|
103 | |
---|
104 | # check cached info |
---|
105 | if mode: |
---|
106 | # cached toc-request? |
---|
107 | if 'full_%s'%mode in docinfo: |
---|
108 | return docinfo |
---|
109 | |
---|
110 | else: |
---|
111 | # cached but no toc-request? |
---|
112 | if 'numTextPages' in docinfo: |
---|
113 | return docinfo |
---|
114 | |
---|
115 | docpath = docinfo.get('textURLPath', None) |
---|
116 | if docpath is None: |
---|
117 | logging.error("getTextInfo: no textURLPath!") |
---|
118 | return docinfo |
---|
119 | |
---|
120 | # fetch docinfo |
---|
121 | pagexml = self.getServerData("query/GetDocInfo","docId=%s%s"%(docpath,field)) |
---|
122 | dom = ET.fromstring(pagexml) |
---|
123 | # all info in tag <doc> |
---|
124 | doc = dom |
---|
125 | if doc is None: |
---|
126 | logging.error("getTextInfo: unable to find document-tag!") |
---|
127 | else: |
---|
128 | if mode is None: |
---|
129 | # get general info from system-tag |
---|
130 | sys = doc.find('system') |
---|
131 | if sys is not None: |
---|
132 | docinfo['numTextPages'] = getInt(getText(sys.find('countPages'))) |
---|
133 | docinfo['numFigureEntries'] = getInt(getText(sys.find('countFigures'))) |
---|
134 | docinfo['numHandwritten'] = getInt(getText(sys.find('countHandwritten'))) |
---|
135 | docinfo['numTocEntries'] = getInt(getText(sys.find('countTocEntries'))) |
---|
136 | |
---|
137 | else: |
---|
138 | # result is in list-tag |
---|
139 | l = doc.find('list') |
---|
140 | if l is not None: |
---|
141 | lt = l.get('type') |
---|
142 | # pageNumbers |
---|
143 | if lt == 'pages': |
---|
144 | # contains tags with page numbers |
---|
145 | # <item n="14" o="2" o-norm="2" file="0014"/> |
---|
146 | # n=scan number, o=original page no, on=normalized original page no |
---|
147 | # pageNumbers is a dict indexed by scan number |
---|
148 | pages = {} |
---|
149 | for i in l: |
---|
150 | page = {} |
---|
151 | pn = getInt(i.get('n')) |
---|
152 | page['pn'] = pn |
---|
153 | no = i.get('o') |
---|
154 | page['no'] = no |
---|
155 | non = i.get('o-norm') |
---|
156 | page['non'] = non |
---|
157 | |
---|
158 | if pn > 0: |
---|
159 | pages[pn] = page |
---|
160 | |
---|
161 | docinfo['pageNumbers'] = pages |
---|
162 | |
---|
163 | # toc |
---|
164 | elif lt == 'toc' or lt == 'figures' or lt == 'handwritten': |
---|
165 | # contains tags with table of contents/figures |
---|
166 | # <item n="2.1." lv="2">CAP.I. <ref o="119">132</ref></item> |
---|
167 | tocs = [] |
---|
168 | for te in l: |
---|
169 | if te.tag == 'item': |
---|
170 | toc = {} |
---|
171 | toc['level-string'] = te.get('n') |
---|
172 | toc['level'] = te.get('lv') |
---|
173 | toc['content'] = te.text.strip() |
---|
174 | ref = te.find('ref') |
---|
175 | toc['pn'] = getInt(ref.text) |
---|
176 | toc['no'] = ref.get('o') |
---|
177 | toc['non'] = ref.get('o-norm') |
---|
178 | tocs.append(toc) |
---|
179 | |
---|
180 | # save as full_toc/full_figures |
---|
181 | docinfo['full_%s'%mode] = tocs |
---|
182 | |
---|
183 | return docinfo |
---|
184 | |
---|
185 | |
---|
186 | def getTextPage(self, mode="text", pn=1, docinfo=None, pageinfo=None): |
---|
187 | """returns single page from fulltext""" |
---|
188 | |
---|
189 | logging.debug("getTextPage mode=%s, pn=%s"%(mode,pn)) |
---|
190 | startTime = datetime.now() |
---|
191 | # check for cached text -- but ideally this shouldn't be called twice |
---|
192 | if pageinfo.has_key('textPage'): |
---|
193 | logging.debug("getTextPage: using cached text") |
---|
194 | return pageinfo['textPage'] |
---|
195 | |
---|
196 | docpath = docinfo.get('textURLPath', None) |
---|
197 | if not docpath: |
---|
198 | return None |
---|
199 | |
---|
200 | # stuff for constructing full urls |
---|
201 | selfurl = docinfo['viewerUrl'] |
---|
202 | textParams = {'docId': docpath, |
---|
203 | 'page': pn} |
---|
204 | |
---|
205 | normMode = pageinfo.get('characterNormalization', 'reg') |
---|
206 | # TODO: change values in form |
---|
207 | if normMode == 'regPlusNorm': |
---|
208 | normMode = 'norm' |
---|
209 | |
---|
210 | # TODO: this should not be necessary when the backend is fixed |
---|
211 | #textParams['normalization'] = normMode |
---|
212 | |
---|
213 | if not mode: |
---|
214 | # default is dict |
---|
215 | mode = 'text' |
---|
216 | |
---|
217 | modes = mode.split(',') |
---|
218 | # check for multiple layers |
---|
219 | if len(modes) > 1: |
---|
220 | logging.debug("getTextPage: more than one mode=%s"%mode) |
---|
221 | |
---|
222 | # search mode |
---|
223 | if 'search' in modes: |
---|
224 | # add highlighting |
---|
225 | highlightQuery = pageinfo.get('highlightQuery', None) |
---|
226 | if highlightQuery: |
---|
227 | textParams['highlightQuery'] = highlightQuery |
---|
228 | textParams['highlightElem'] = pageinfo.get('highlightElement', '') |
---|
229 | textParams['highlightElemPos'] = pageinfo.get('highlightElementPos', '') |
---|
230 | |
---|
231 | # ignore mode in the following |
---|
232 | modes.remove('search') |
---|
233 | |
---|
234 | # pundit mode |
---|
235 | punditMode = False |
---|
236 | if 'pundit' in modes: |
---|
237 | punditMode = True |
---|
238 | # ignore mode in the following |
---|
239 | modes.remove('pundit') |
---|
240 | |
---|
241 | # other modes don't combine |
---|
242 | if 'dict' in modes: |
---|
243 | textmode = 'dict' |
---|
244 | textParams['outputFormat'] = 'html' |
---|
245 | elif 'xml' in modes: |
---|
246 | textmode = 'xml' |
---|
247 | textParams['outputFormat'] = 'xmlDisplay' |
---|
248 | normMode = 'orig' |
---|
249 | elif 'gis' in modes: |
---|
250 | #FIXME! |
---|
251 | textmode = 'gis' |
---|
252 | else: |
---|
253 | # text is default mode |
---|
254 | textmode = 'plain' |
---|
255 | textParams['outputFormat'] = 'html' |
---|
256 | |
---|
257 | try: |
---|
258 | # fetch the page |
---|
259 | pagexml = self.getServerData("query/GetPage",urllib.urlencode(textParams)) |
---|
260 | dom = ET.fromstring(pagexml) |
---|
261 | except Exception, e: |
---|
262 | logging.error("Error reading page: %s"%e) |
---|
263 | return None |
---|
264 | |
---|
265 | # plain text or text-with-links mode |
---|
266 | if textmode == "plain" or textmode == "dict": |
---|
267 | # the text is in div@class=text |
---|
268 | pagediv = dom.find(".//div[@class='text']") |
---|
269 | logging.debug("pagediv: %s"%repr(pagediv)) |
---|
270 | if pagediv is not None: |
---|
271 | # add textmode and normMode classes |
---|
272 | #pagediv.set('class', 'text %s %s'%(textmode, normMode)) |
---|
273 | self._processWTags(textmode, normMode, pagediv) |
---|
274 | #self._processPbTag(pagediv, pageinfo) |
---|
275 | self._processFigures(pagediv, docinfo) |
---|
276 | #self._fixEmptyDivs(pagediv) |
---|
277 | # get full url assuming documentViewer is parent |
---|
278 | selfurl = self.getLink() |
---|
279 | # check all a-tags |
---|
280 | links = pagediv.findall('.//a') |
---|
281 | for l in links: |
---|
282 | href = l.get('href') |
---|
283 | if href: |
---|
284 | # is link with href |
---|
285 | linkurl = urlparse.urlparse(href) |
---|
286 | if linkurl.path.endswith('GetDictionaryEntries'): |
---|
287 | #TODO: replace wordInfo page |
---|
288 | # add target to open new page |
---|
289 | l.set('target', '_blank') |
---|
290 | |
---|
291 | if punditMode: |
---|
292 | self._addPunditAttributes(pagediv, pageinfo, docinfo) |
---|
293 | |
---|
294 | s = serialize(pagediv) |
---|
295 | logging.debug("getTextPage done in %s"%(datetime.now()-startTime)) |
---|
296 | return s |
---|
297 | |
---|
298 | # xml mode |
---|
299 | elif textmode == "xml": |
---|
300 | # the text is in body |
---|
301 | pagediv = dom.find(".//body") |
---|
302 | logging.debug("pagediv: %s"%repr(pagediv)) |
---|
303 | if pagediv is not None: |
---|
304 | return serialize(pagediv) |
---|
305 | |
---|
306 | # pureXml mode WTF? |
---|
307 | elif textmode == "pureXml": |
---|
308 | # the text is in body |
---|
309 | pagediv = dom.find(".//body") |
---|
310 | logging.debug("pagediv: %s"%repr(pagediv)) |
---|
311 | if pagediv is not None: |
---|
312 | return serialize(pagediv) |
---|
313 | |
---|
314 | # gis mode FIXME! |
---|
315 | elif textmode == "gis": |
---|
316 | # the text is in div@class=text |
---|
317 | pagediv = dom.find(".//div[@class='text']") |
---|
318 | logging.debug("pagediv: %s"%repr(pagediv)) |
---|
319 | if pagediv is not None: |
---|
320 | # fix empty div tags |
---|
321 | self._fixEmptyDivs(pagediv) |
---|
322 | # check all a-tags |
---|
323 | links = pagediv.findall(".//a") |
---|
324 | # add our URL as backlink |
---|
325 | selfurl = self.getLink() |
---|
326 | doc = base64.b64encode(selfurl) |
---|
327 | for l in links: |
---|
328 | href = l.get('href') |
---|
329 | if href: |
---|
330 | if href.startswith('http://mappit.mpiwg-berlin.mpg.de'): |
---|
331 | l.set('href', re.sub(r'doc=[\w+/=]+', 'doc=%s'%doc, href)) |
---|
332 | l.set('target', '_blank') |
---|
333 | |
---|
334 | return serialize(pagediv) |
---|
335 | |
---|
336 | logging.error("getTextPage: error in text mode %s or in text!"%(textmode)) |
---|
337 | return None |
---|
338 | |
---|
339 | def _processWTags(self, textMode, normMode, pagediv): |
---|
340 | """selects the necessary information from w-spans and removes the rest from pagediv""" |
---|
341 | logging.debug("processWTags(textMode=%s,norm=%s,pagediv"%(repr(textMode),repr(normMode))) |
---|
342 | startTime = datetime.now() |
---|
343 | wtags = pagediv.findall(".//span[@class='w']") |
---|
344 | for wtag in wtags: |
---|
345 | if textMode == 'dict': |
---|
346 | # delete non-a-tags |
---|
347 | wtag.remove(wtag.find("span[@class='nodictionary orig']")) |
---|
348 | wtag.remove(wtag.find("span[@class='nodictionary reg']")) |
---|
349 | wtag.remove(wtag.find("span[@class='nodictionary norm']")) |
---|
350 | # delete non-matching children of a-tag and suppress remaining tag name |
---|
351 | atag = wtag.find("*[@class='dictionary']") |
---|
352 | if normMode == 'orig': |
---|
353 | atag.remove(atag.find("span[@class='reg']")) |
---|
354 | atag.remove(atag.find("span[@class='norm']")) |
---|
355 | atag.find("span[@class='orig']").tag = None |
---|
356 | elif normMode == 'reg': |
---|
357 | atag.remove(atag.find("span[@class='orig']")) |
---|
358 | atag.remove(atag.find("span[@class='norm']")) |
---|
359 | atag.find("span[@class='reg']").tag = None |
---|
360 | elif normMode == 'norm': |
---|
361 | atag.remove(atag.find("span[@class='orig']")) |
---|
362 | atag.remove(atag.find("span[@class='reg']")) |
---|
363 | atag.find("span[@class='norm']").tag = None |
---|
364 | |
---|
365 | else: |
---|
366 | # delete a-tag |
---|
367 | wtag.remove(wtag.find("*[@class='dictionary']")) |
---|
368 | # delete non-matching children and suppress remaining tag name |
---|
369 | if normMode == 'orig': |
---|
370 | wtag.remove(wtag.find("span[@class='nodictionary reg']")) |
---|
371 | wtag.remove(wtag.find("span[@class='nodictionary norm']")) |
---|
372 | wtag.find("span[@class='nodictionary orig']").tag = None |
---|
373 | elif normMode == 'reg': |
---|
374 | wtag.remove(wtag.find("span[@class='nodictionary orig']")) |
---|
375 | wtag.remove(wtag.find("span[@class='nodictionary norm']")) |
---|
376 | wtag.find("span[@class='nodictionary reg']").tag = None |
---|
377 | elif normMode == 'norm': |
---|
378 | wtag.remove(wtag.find("span[@class='nodictionary orig']")) |
---|
379 | wtag.remove(wtag.find("span[@class='nodictionary reg']")) |
---|
380 | wtag.find("span[@class='nodictionary norm']").tag = None |
---|
381 | |
---|
382 | # suppress w-tag name |
---|
383 | wtag.tag = None |
---|
384 | |
---|
385 | logging.debug("processWTags in %s"%(datetime.now()-startTime)) |
---|
386 | return pagediv |
---|
387 | |
---|
388 | def _processPbTag(self, pagediv, pageinfo): |
---|
389 | """extracts information from pb-tag and removes it from pagediv""" |
---|
390 | pbdiv = pagediv.find(".//span[@class='pb']") |
---|
391 | if pbdiv is None: |
---|
392 | logging.warning("getTextPage: no pb-span!") |
---|
393 | return pagediv |
---|
394 | |
---|
395 | # extract running head |
---|
396 | rh = pbdiv.find(".//span[@class='rhead']") |
---|
397 | if rh is not None: |
---|
398 | pageinfo['pageHeaderTitle'] = getText(rh) |
---|
399 | |
---|
400 | # remove pb-div from parent |
---|
401 | ppdiv = pagediv.find(".//span[@class='pb']/..") |
---|
402 | ppdiv.remove(pbdiv) |
---|
403 | return pagediv |
---|
404 | |
---|
405 | def _addPunditAttributes(self, pagediv, pageinfo, docinfo): |
---|
406 | """add about attributes for pundit annotation tool""" |
---|
407 | textid = docinfo.get('DRI', "fn=%s"%docinfo.get('documentPath', '???')) |
---|
408 | pn = pageinfo.get('pn', '1') |
---|
409 | # TODO: use pn as well? |
---|
410 | # check all div-tags |
---|
411 | divs = pagediv.findall(".//div") |
---|
412 | for d in divs: |
---|
413 | id = d.get('id') |
---|
414 | if id: |
---|
415 | # TODO: check path (cf RFC2396) |
---|
416 | d.set('about', "http://echo.mpiwg-berlin.mpg.de/%s/pn=%s/#%s"%(textid,pn,id)) |
---|
417 | cls = d.get('class','') |
---|
418 | cls += ' pundit-content' |
---|
419 | d.set('class', cls.strip()) |
---|
420 | |
---|
421 | return pagediv |
---|
422 | |
---|
423 | def _processFigures(self, pagediv, docinfo): |
---|
424 | """processes figure-tags""" |
---|
425 | # unfortunately etree can not select class.startswith('figure') |
---|
426 | divs = pagediv.findall(".//span[@class]") |
---|
427 | scalerUrl = docinfo['digilibScalerUrl'] |
---|
428 | viewerUrl = docinfo['digilibViewerUrl'] |
---|
429 | for d in divs: |
---|
430 | if not d.get('class').startswith('figure'): |
---|
431 | continue |
---|
432 | |
---|
433 | try: |
---|
434 | a = d.find('a') |
---|
435 | img = a.find('img') |
---|
436 | imgsrc = img.get('src') |
---|
437 | imgurl = urlparse.urlparse(imgsrc) |
---|
438 | imgq = imgurl.query |
---|
439 | imgparams = urlparse.parse_qs(imgq) |
---|
440 | fn = imgparams.get('fn', None) |
---|
441 | if fn is not None: |
---|
442 | # parse_qs puts parameters in lists |
---|
443 | fn = fn[0] |
---|
444 | # TODO: check valid path |
---|
445 | # fix img@src |
---|
446 | newsrc = '%s?fn=%s&dw=200&dh=200'%(scalerUrl,fn) |
---|
447 | img.set('src', newsrc) |
---|
448 | # fix a@href |
---|
449 | newlink = '%s?fn=%s'%(viewerUrl,fn) |
---|
450 | a.set('href', newlink) |
---|
451 | a.set('target', '_blank') |
---|
452 | |
---|
453 | except: |
---|
454 | logging.warn("processFigures: strange figure!") |
---|
455 | |
---|
456 | |
---|
457 | def _cleanSearchResult(self, pagediv): |
---|
458 | """fixes search result html (change pbs and figures)""" |
---|
459 | # replace figure-tag with figureNumText |
---|
460 | for fig in pagediv.findall(".//span[@class='figure']"): |
---|
461 | txt = fig.findtext(".//span[@class='figureNumText']") |
---|
462 | tail = fig.tail |
---|
463 | fig.clear() |
---|
464 | fig.set('class', 'figure') |
---|
465 | fig.text = txt |
---|
466 | fig.tail = tail |
---|
467 | |
---|
468 | # replace lb-tag with "//" |
---|
469 | for lb in pagediv.findall(".//br[@class='lb']"): |
---|
470 | lb.tag = 'span' |
---|
471 | lb.text = '//' |
---|
472 | |
---|
473 | # replace pb-tag with "///" |
---|
474 | for pb in pagediv.findall(".//span[@class='pb']"): |
---|
475 | tail = pb.tail |
---|
476 | pb.clear() |
---|
477 | pb.set('class', 'pb') |
---|
478 | pb.text = '///' |
---|
479 | pb.tail = tail |
---|
480 | |
---|
481 | return pagediv |
---|
482 | |
---|
483 | def _cleanSearchResult2(self, pagediv): |
---|
484 | """fixes search result html (change pbs and figures)""" |
---|
485 | # unfortunately etree can not select class.startswith('figure') |
---|
486 | divs = pagediv.findall(".//span[@class]") |
---|
487 | for d in divs: |
---|
488 | cls = d.get('class') |
---|
489 | if cls.startswith('figure'): |
---|
490 | # replace figure-tag with figureNumText |
---|
491 | txt = d.findtext(".//span[@class='figureNumText']") |
---|
492 | d.clear() |
---|
493 | d.set('class', 'figure') |
---|
494 | d.text = txt |
---|
495 | |
---|
496 | elif cls.startswith('pb'): |
---|
497 | # replace pb-tag with "//" |
---|
498 | d.clear() |
---|
499 | d.set('class', 'pb') |
---|
500 | d.text = '//' |
---|
501 | |
---|
502 | return pagediv |
---|
503 | |
---|
504 | |
---|
505 | |
---|
506 | def _fixEmptyDivs(self, pagediv): |
---|
507 | """fixes empty div-tags by inserting a space""" |
---|
508 | divs = pagediv.findall('.//div') |
---|
509 | for d in divs: |
---|
510 | if len(d) == 0 and not d.text: |
---|
511 | # make empty divs non-empty |
---|
512 | d.text = ' ' |
---|
513 | |
---|
514 | return pagediv |
---|
515 | |
---|
516 | |
---|
517 | def getSearchResults(self, mode, query=None, pageinfo=None, docinfo=None): |
---|
518 | """loads list of search results and stores XML in docinfo""" |
---|
519 | normMode = pageinfo.get('characterNormalization', 'reg') |
---|
520 | logging.debug("getSearchResults mode=%s query=%s norm=%s"%(mode, query, normMode)) |
---|
521 | if mode == "none": |
---|
522 | return docinfo |
---|
523 | |
---|
524 | #TODO: put mode into query |
---|
525 | |
---|
526 | cachedQuery = docinfo.get('cachedQuery', None) |
---|
527 | if cachedQuery is not None: |
---|
528 | # cached search result |
---|
529 | if cachedQuery == '%s_%s_%s'%(mode,query,normMode): |
---|
530 | # same query |
---|
531 | return docinfo |
---|
532 | |
---|
533 | else: |
---|
534 | # different query |
---|
535 | del docinfo['resultSize'] |
---|
536 | del docinfo['results'] |
---|
537 | |
---|
538 | # cache query |
---|
539 | docinfo['cachedQuery'] = '%s_%s_%s'%(mode,query,normMode) |
---|
540 | |
---|
541 | # fetch full results |
---|
542 | docpath = docinfo['textURLPath'] |
---|
543 | params = {'docId': docpath, |
---|
544 | 'query': query, |
---|
545 | 'pageSize': 1000, |
---|
546 | 'page': 1, |
---|
547 | 'outputFormat': 'html'} |
---|
548 | pagexml = self.getServerData("query/QueryDocument",urllib.urlencode(params)) |
---|
549 | results = [] |
---|
550 | try: |
---|
551 | dom = ET.fromstring(pagexml) |
---|
552 | # clean html output |
---|
553 | self._processWTags('plain', normMode, dom) |
---|
554 | self._cleanSearchResult(dom) |
---|
555 | # page content is currently in multiple <td align=left> |
---|
556 | alldivs = dom.findall(".//tr[@class='hit']") |
---|
557 | for div in alldivs: |
---|
558 | # change tr to div |
---|
559 | div.tag = 'div' |
---|
560 | # change td to span |
---|
561 | for d in div.findall('td'): |
---|
562 | d.tag = 'span' |
---|
563 | |
---|
564 | # TODO: can we put etree in the session? |
---|
565 | results.append(div) |
---|
566 | |
---|
567 | except Exception, e: |
---|
568 | logging.error("GetSearchResults: Error parsing search result: %s"%e) |
---|
569 | |
---|
570 | # store results in docinfo |
---|
571 | docinfo['resultSize'] = len(results) |
---|
572 | docinfo['results'] = results |
---|
573 | |
---|
574 | return docinfo |
---|
575 | |
---|
576 | |
---|
577 | def getResultsPage(self, mode="text", query=None, pn=None, start=None, size=None, pageinfo=None, docinfo=None): |
---|
578 | """returns single page from the list of search results""" |
---|
579 | logging.debug("getResultsPage mode=%s, pn=%s"%(mode,pn)) |
---|
580 | # get (cached) result |
---|
581 | self.getSearchResults(mode=mode, query=query, pageinfo=pageinfo, docinfo=docinfo) |
---|
582 | |
---|
583 | resultxml = docinfo.get('results', None) |
---|
584 | if not resultxml: |
---|
585 | logging.error("getResultPage: unable to find results") |
---|
586 | return "Error: no result!" |
---|
587 | |
---|
588 | if size is None: |
---|
589 | size = pageinfo.get('resultPageSize', 10) |
---|
590 | |
---|
591 | if start is None: |
---|
592 | start = (pn - 1) * size |
---|
593 | |
---|
594 | if resultxml is not None: |
---|
595 | # paginate |
---|
596 | first = start-1 |
---|
597 | last = first+size |
---|
598 | tocdivs = resultxml[first:last] |
---|
599 | |
---|
600 | toc = ET.Element('div', attrib={'class':'queryResultPage'}) |
---|
601 | for div in tocdivs: |
---|
602 | # check all a-tags |
---|
603 | links = div.findall(".//a") |
---|
604 | for l in links: |
---|
605 | href = l.get('href') |
---|
606 | if href: |
---|
607 | # assume all links go to pages |
---|
608 | linkUrl = urlparse.urlparse(href) |
---|
609 | linkParams = urlparse.parse_qs(linkUrl.query) |
---|
610 | # take some parameters (make sure it works even if the link was already parsed) |
---|
611 | params = {'pn': linkParams.get('page',linkParams.get('pn', None)), |
---|
612 | 'highlightQuery': linkParams.get('highlightQuery',None), |
---|
613 | 'highlightElement': linkParams.get('highlightElem',linkParams.get('highlightElement',None)), |
---|
614 | 'highlightElementPos': linkParams.get('highlightElemPos',linkParams.get('highlightElementPos',None)) |
---|
615 | } |
---|
616 | if not params['pn']: |
---|
617 | logging.warn("getResultsPage: link has no page: %s"%href) |
---|
618 | |
---|
619 | url = self.getLink(params=params) |
---|
620 | l.set('href', url) |
---|
621 | |
---|
622 | toc.append(div) |
---|
623 | |
---|
624 | return serialize(toc) |
---|
625 | |
---|
626 | return "ERROR: no results!" |
---|
627 | |
---|
628 | |
---|
629 | def getToc(self, mode='text', docinfo=None): |
---|
630 | """returns list of table of contents from docinfo""" |
---|
631 | logging.debug("getToc mode=%s"%mode) |
---|
632 | if mode == 'text': |
---|
633 | queryType = 'toc' |
---|
634 | else: |
---|
635 | queryType = mode |
---|
636 | |
---|
637 | if not 'full_%s'%queryType in docinfo: |
---|
638 | # get new toc |
---|
639 | docinfo = self.getTextInfo(queryType, docinfo) |
---|
640 | |
---|
641 | return docinfo.get('full_%s'%queryType, []) |
---|
642 | |
---|
643 | |
---|
644 | def getTocPage(self, mode='text', pn=None, start=None, size=None, pageinfo=None, docinfo=None): |
---|
645 | """returns single page from the table of contents""" |
---|
646 | logging.debug("getTocPage mode=%s, pn=%s start=%s size=%s"%(mode,repr(pn),repr(start),repr(size))) |
---|
647 | fulltoc = self.getToc(mode=mode, docinfo=docinfo) |
---|
648 | if len(fulltoc) < 1: |
---|
649 | logging.error("getTocPage: unable to find toc!") |
---|
650 | return "Error: no table of contents!" |
---|
651 | |
---|
652 | if size is None: |
---|
653 | size = pageinfo.get('tocPageSize', 30) |
---|
654 | |
---|
655 | if start is None: |
---|
656 | start = (pn - 1) * size |
---|
657 | |
---|
658 | # paginate |
---|
659 | first = (start - 1) |
---|
660 | last = first + size |
---|
661 | tocs = fulltoc[first:last] |
---|
662 | tp = '<div>' |
---|
663 | label = {'figures': 'Figure', 'handwritten': 'Handwritten note'}.get(mode, 'Item') |
---|
664 | for toc in tocs: |
---|
665 | pageurl = self.getLink('pn', toc['pn']) |
---|
666 | tp += '<div class="tocline">' |
---|
667 | content = toc['content'] |
---|
668 | if content: |
---|
669 | tp += '<div class="toc name">[%s] %s</div>'%(toc['level-string'], toc['content']) |
---|
670 | else: |
---|
671 | tp += '<div class="toc name">[%s %s]</div>'%(label, toc['level-string']) |
---|
672 | |
---|
673 | if toc.get('no', None): |
---|
674 | tp += '<div class="toc page"><a href="%s">Page: %s (%s)</a></div>'%(pageurl, toc['pn'], toc['no']) |
---|
675 | else: |
---|
676 | tp += '<div class="toc page"><a href="%s">Page: %s</a></div>'%(pageurl, toc['pn']) |
---|
677 | |
---|
678 | tp += '</div>\n' |
---|
679 | |
---|
680 | tp += '</div>\n' |
---|
681 | |
---|
682 | return tp |
---|
683 | |
---|
684 | |
---|
685 | def manage_changeMpiwgXmlTextServer(self,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de/mpdl/interface/",timeout=40,repositoryType=None,RESPONSE=None): |
---|
686 | """change settings""" |
---|
687 | self.title=title |
---|
688 | self.timeout = timeout |
---|
689 | self.serverUrl = serverUrl |
---|
690 | if repositoryType: |
---|
691 | self.repositoryType = repositoryType |
---|
692 | if RESPONSE is not None: |
---|
693 | RESPONSE.redirect('manage_main') |
---|
694 | |
---|
695 | # management methods |
---|
696 | def manage_addMpiwgXmlTextServerForm(self): |
---|
697 | """Form for adding""" |
---|
698 | pt = PageTemplateFile("zpt/manage_addMpiwgXmlTextServer", globals()).__of__(self) |
---|
699 | return pt() |
---|
700 | |
---|
701 | def manage_addMpiwgXmlTextServer(self,id,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de/mpdl/interface/",timeout=40,RESPONSE=None): |
---|
702 | """add MpiwgXmlTextServer""" |
---|
703 | newObj = MpiwgXmlTextServer(id=id,title=title,serverUrl=serverUrl,timeout=timeout) |
---|
704 | self.Destination()._setObject(id, newObj) |
---|
705 | if RESPONSE is not None: |
---|
706 | RESPONSE.redirect('manage_main') |
---|
707 | |
---|
708 | |
---|