1 | from OFS.SimpleItem import SimpleItem |
---|
2 | from Products.PageTemplates.PageTemplateFile import PageTemplateFile |
---|
3 | |
---|
4 | import xml.etree.ElementTree as ET |
---|
5 | |
---|
6 | import re |
---|
7 | import logging |
---|
8 | import urllib |
---|
9 | import urlparse |
---|
10 | import base64 |
---|
11 | |
---|
12 | from HocrTxtUtils import getInt, getText, getHttpData |
---|
13 | |
---|
14 | def serialize(node): |
---|
15 | """returns a string containing an XML snippet of node""" |
---|
16 | s = ET.tostring(node, 'UTF-8') |
---|
17 | # snip off XML declaration |
---|
18 | if s.startswith('<?xml'): |
---|
19 | i = s.find('?>') |
---|
20 | return s[i+3:] |
---|
21 | |
---|
22 | return s |
---|
23 | |
---|
24 | |
---|
25 | class HocrTextServer(SimpleItem): |
---|
26 | """TextServer implementation for MPDL-XML eXist server""" |
---|
27 | meta_type="Hocr TextServer" |
---|
28 | |
---|
29 | manage_options=( |
---|
30 | {'label':'Config','action':'manage_changeHocrTextServerForm'}, |
---|
31 | )+SimpleItem.manage_options |
---|
32 | |
---|
33 | manage_changeHocrTextServerForm = PageTemplateFile("zpt/manage_changeHocrTextServer", globals()) |
---|
34 | |
---|
35 | def __init__(self,id,title="",serverUrl="http://localhost:8080/hocr", timeout=40, repositoryType='production'): |
---|
36 | """constructor""" |
---|
37 | self.id=id |
---|
38 | self.title=title |
---|
39 | self.timeout = timeout |
---|
40 | self.repositoryType = repositoryType |
---|
41 | |
---|
42 | self.serverUrl = serverUrl |
---|
43 | |
---|
44 | def getHttpData(self, url, data=None): |
---|
45 | """returns result from url+data HTTP request""" |
---|
46 | return getHttpData(url,data,timeout=self.timeout) |
---|
47 | |
---|
48 | def getServerData(self, pn, data=None): |
---|
49 | """returns result from text server for method+data""" |
---|
50 | url = self.serverUrl |
---|
51 | return getHttpData(url,pn,data=data,timeout=self.timeout) |
---|
52 | |
---|
53 | |
---|
54 | def getRepositoryType(self): |
---|
55 | """returns the repository type, e.g. 'production'""" |
---|
56 | return getattr(self, 'repositoryType', None) |
---|
57 | |
---|
58 | def getTextDownloadUrl(self, type='xml', docinfo=None): |
---|
59 | """returns a URL to download the current text""" |
---|
60 | docpath = docinfo.get('textURLPath', None) |
---|
61 | if not docpath: |
---|
62 | return None |
---|
63 | |
---|
64 | docpath = docpath.replace('.xml','.'+type) |
---|
65 | url = '%sgetDoc?doc=%s'%(self.serverUrl.replace('interface/',''), docpath) |
---|
66 | return url |
---|
67 | |
---|
68 | |
---|
69 | def getPlacesOnPage(self, docinfo=None, pn=None): |
---|
70 | """Returns list of GIS places of page pn""" |
---|
71 | docpath = docinfo.get('textURLPath',None) |
---|
72 | if not docpath: |
---|
73 | return None |
---|
74 | |
---|
75 | places=[] |
---|
76 | text=self.getServerData("xpath.xql", "document=%s&xpath=//place&pn=%s"%(docpath,pn)) |
---|
77 | dom = ET.fromstring(text) |
---|
78 | result = dom.findall(".//resultPage/place") |
---|
79 | for l in result: |
---|
80 | id = l.get("id") |
---|
81 | name = l.text |
---|
82 | place = {'id': id, 'name': name} |
---|
83 | places.append(place) |
---|
84 | |
---|
85 | return places |
---|
86 | |
---|
87 | |
---|
88 | def getTextInfo(self, mode='', docinfo=None): |
---|
89 | """reads document info, including page concordance, from text server""" |
---|
90 | logging.debug("getTextInfo mode=%s"%mode) |
---|
91 | if mode not in ['toc', 'figures', '']: |
---|
92 | mode = '' |
---|
93 | # check cached info |
---|
94 | if mode: |
---|
95 | # cached toc-request? |
---|
96 | if 'full_%s'%mode in docinfo: |
---|
97 | return docinfo |
---|
98 | |
---|
99 | else: |
---|
100 | # no toc-request |
---|
101 | if 'numTextPages' in docinfo: |
---|
102 | return docinfo |
---|
103 | |
---|
104 | docpath = docinfo.get('textURLPath', None) |
---|
105 | if docpath is None: |
---|
106 | logging.error("getTextInfo: no textURLPath!") |
---|
107 | return docinfo |
---|
108 | |
---|
109 | try: |
---|
110 | # we need to set a result set size |
---|
111 | pagesize = 10000 |
---|
112 | pn = 1 |
---|
113 | # fetch docinfo |
---|
114 | pagexml = self.getServerData("doc-info.xql","document=%s&info=%s&pageSize=%s&pn=%s"%(docpath,mode,pagesize,pn)) |
---|
115 | dom = ET.fromstring(pagexml) |
---|
116 | # all info in tag <document> |
---|
117 | doc = dom.find("document") |
---|
118 | except Exception, e: |
---|
119 | logging.error("getTextInfo: Error reading doc info: %s"%e) |
---|
120 | return docinfo |
---|
121 | |
---|
122 | if doc is None: |
---|
123 | logging.error("getTextInfo: unable to find document-tag!") |
---|
124 | else: |
---|
125 | # go through all child elements |
---|
126 | for tag in doc: |
---|
127 | name = tag.tag |
---|
128 | # numTextPages |
---|
129 | if name == 'countPages': |
---|
130 | np = getInt(tag.text) |
---|
131 | if np > 0: |
---|
132 | docinfo['numTextPages'] = np |
---|
133 | |
---|
134 | # numFigureEntries |
---|
135 | elif name == 'countFigureEntries': |
---|
136 | docinfo['numFigureEntries'] = getInt(tag.text) |
---|
137 | |
---|
138 | # numTocEntries |
---|
139 | elif name == 'countTocEntries': |
---|
140 | # WTF: s1 = int(s)/30+1 |
---|
141 | docinfo['numTocEntries'] = getInt(tag.text) |
---|
142 | |
---|
143 | # numPlaces |
---|
144 | elif name == 'countPlaces': |
---|
145 | docinfo['numPlaces'] = getInt(tag.text) |
---|
146 | |
---|
147 | # pageNumbers |
---|
148 | elif name == 'pageNumbers': |
---|
149 | # contains tags with page numbers |
---|
150 | # <pn><n>4</n><no>4</no><non/></pn> |
---|
151 | # n=scan number, no=original page no, non=normalized original page no |
---|
152 | # pageNumbers is a dict indexed by scan number |
---|
153 | pages = {} |
---|
154 | for pn in tag: |
---|
155 | page = {} |
---|
156 | n = 0 |
---|
157 | for p in pn: |
---|
158 | if p.tag == 'n': |
---|
159 | n = getInt(p.text) |
---|
160 | page['pn'] = n |
---|
161 | elif p.tag == 'no': |
---|
162 | page['no'] = p.text |
---|
163 | elif p.tag == 'non': |
---|
164 | page['non'] = p.text |
---|
165 | |
---|
166 | if n > 0: |
---|
167 | pages[n] = page |
---|
168 | |
---|
169 | docinfo['pageNumbers'] = pages |
---|
170 | #logging.debug("got pageNumbers=%s"%repr(pages)) |
---|
171 | |
---|
172 | # toc |
---|
173 | elif name == 'toc': |
---|
174 | # contains tags with table of contents/figures |
---|
175 | # <toc-entry><page>13</page><level>3</level><content>Chapter I</content><level-string>1.</level-string><real-level>1</real-level></toc-entry> |
---|
176 | tocs = [] |
---|
177 | for te in tag: |
---|
178 | toc = {} |
---|
179 | for t in te: |
---|
180 | if t.tag == 'page': |
---|
181 | toc['pn'] = getInt(t.text) |
---|
182 | elif t.tag == 'level': |
---|
183 | toc['level'] = t.text |
---|
184 | elif t.tag == 'content': |
---|
185 | toc['content'] = t.text |
---|
186 | elif t.tag == 'level-string': |
---|
187 | toc['level-string'] = t.text |
---|
188 | elif t.tag == 'real-level': |
---|
189 | toc['real-level'] = t.text |
---|
190 | |
---|
191 | tocs.append(toc) |
---|
192 | |
---|
193 | # save as full_toc/full_figures |
---|
194 | docinfo['full_%s'%mode] = tocs |
---|
195 | |
---|
196 | return docinfo |
---|
197 | |
---|
198 | |
---|
199 | def processPageInfo(self, dom, docinfo, pageinfo): |
---|
200 | """processes page info divs from dom and stores in docinfo and pageinfo""" |
---|
201 | # assume first second level div is pageMeta |
---|
202 | alldivs = dom.find("div") |
---|
203 | |
---|
204 | if alldivs is None or alldivs.get('class', '') != 'pageMeta': |
---|
205 | logging.error("processPageInfo: pageMeta div not found!") |
---|
206 | return |
---|
207 | |
---|
208 | for div in alldivs: |
---|
209 | dc = div.get('class') |
---|
210 | |
---|
211 | # pageNumberOrig |
---|
212 | if dc == 'pageNumberOrig': |
---|
213 | pageinfo['pageNumberOrig'] = div.text |
---|
214 | |
---|
215 | # pageNumberOrigNorm |
---|
216 | elif dc == 'pageNumberOrigNorm': |
---|
217 | pageinfo['pageNumberOrigNorm'] = div.text |
---|
218 | |
---|
219 | # pageHeaderTitle |
---|
220 | elif dc == 'pageHeaderTitle': |
---|
221 | pageinfo['pageHeaderTitle'] = div.text |
---|
222 | |
---|
223 | #logging.debug("processPageInfo: pageinfo=%s"%repr(pageinfo)) |
---|
224 | return |
---|
225 | |
---|
226 | |
---|
227 | def getTextPage(self, mode="text", pn=1, docinfo=None, pageinfo=None): |
---|
228 | """returns single page from fulltext""" |
---|
229 | |
---|
230 | |
---|
231 | logging.debug("getTextPage Hocr mode=%s, pn=%s"%(mode,pn)) |
---|
232 | # check for cached text -- but ideally this shouldn't be called twice |
---|
233 | if pageinfo.has_key('textPage'): |
---|
234 | logging.debug("getTextPage: using cached text") |
---|
235 | return pageinfo['textPage'] |
---|
236 | |
---|
237 | docpath = docinfo.get('textURLPath', None) |
---|
238 | |
---|
239 | docpath=docpath.replace("pages","hocr") |
---|
240 | |
---|
241 | logging.debug("getTextPage docpath= %s"%docpath) |
---|
242 | if not docpath: |
---|
243 | return None |
---|
244 | |
---|
245 | # stuff for constructing full urls |
---|
246 | selfurl = docinfo['viewerUrl'] |
---|
247 | textParams = {'document': docpath, |
---|
248 | 'pn': pn} |
---|
249 | if 'characterNormalization' in pageinfo: |
---|
250 | textParams['characterNormalization'] = pageinfo['characterNormalization'] |
---|
251 | |
---|
252 | if not mode: |
---|
253 | # default is dict |
---|
254 | mode = 'text' |
---|
255 | |
---|
256 | logging.debug("getTextPage mode=%s, pn=%s"%(mode,pn)) |
---|
257 | modes = mode.split(',') |
---|
258 | # check for multiple layers |
---|
259 | if len(modes) > 1: |
---|
260 | logging.debug("getTextPage: more than one mode=%s"%mode) |
---|
261 | |
---|
262 | # search mode |
---|
263 | if 'search' in modes: |
---|
264 | # add highlighting |
---|
265 | highlightQuery = pageinfo.get('highlightQuery', None) |
---|
266 | if highlightQuery: |
---|
267 | textParams['highlightQuery'] = highlightQuery |
---|
268 | textParams['highlightElement'] = pageinfo.get('highlightElement', '') |
---|
269 | textParams['highlightElementPos'] = pageinfo.get('highlightElementPos', '') |
---|
270 | |
---|
271 | # ignore mode in the following |
---|
272 | modes.remove('search') |
---|
273 | |
---|
274 | # pundit mode |
---|
275 | punditMode = False |
---|
276 | if 'pundit' in modes: |
---|
277 | punditMode = True |
---|
278 | # ignore mode in the following |
---|
279 | modes.remove('pundit') |
---|
280 | |
---|
281 | # other modes don't combine |
---|
282 | if 'dict' in modes: |
---|
283 | # dict is called textPollux in the backend |
---|
284 | textmode = 'textPollux' |
---|
285 | elif 'xml' in modes: |
---|
286 | # xml mode |
---|
287 | textmode = 'xml' |
---|
288 | textParams['characterNormalization'] = 'orig' |
---|
289 | elif 'gis' in modes: |
---|
290 | textmode = 'gis' |
---|
291 | else: |
---|
292 | # text is default mode |
---|
293 | textmode = 'text' |
---|
294 | |
---|
295 | textParams['mode'] = textmode |
---|
296 | |
---|
297 | logging.debug("getTextPage (textparams: %s"%textParams) |
---|
298 | |
---|
299 | try: |
---|
300 | # fetch the page |
---|
301 | pagexml = self.getServerData(pn,urllib.urlencode(textParams)) |
---|
302 | return pagexml |
---|
303 | except Exception, e: |
---|
304 | logging.error("getTextPage: Error reading page: %s"%e) |
---|
305 | return None |
---|
306 | |
---|
307 | |
---|
308 | |
---|
309 | return None |
---|
310 | |
---|
311 | def addPunditAttributes(self, pagediv, pageinfo, docinfo): |
---|
312 | """add about attributes for pundit annotation tool""" |
---|
313 | textid = docinfo.get('DRI', "fn=%s"%docinfo.get('documentPath', '???')) |
---|
314 | pn = pageinfo.get('pn', '1') |
---|
315 | # TODO: use pn as well? |
---|
316 | # check all div-tags |
---|
317 | divs = pagediv.findall(".//div") |
---|
318 | for d in divs: |
---|
319 | id = d.get('id') |
---|
320 | if id: |
---|
321 | d.set('about', "http://echo.mpiwg-berlin.mpg.de/%s/pn=%s/#%s"%(textid,pn,id)) |
---|
322 | cls = d.get('class','') |
---|
323 | cls += ' pundit-content' |
---|
324 | d.set('class', cls.strip()) |
---|
325 | |
---|
326 | return pagediv |
---|
327 | |
---|
328 | def getSearchResults(self, mode, query=None, pageinfo=None, docinfo=None): |
---|
329 | """loads list of search results and stores XML in docinfo""" |
---|
330 | |
---|
331 | logging.debug("getSearchResults mode=%s query=%s"%(mode, query)) |
---|
332 | if mode == "none": |
---|
333 | return docinfo |
---|
334 | |
---|
335 | cachedQuery = docinfo.get('cachedQuery', None) |
---|
336 | if cachedQuery is not None: |
---|
337 | # cached search result |
---|
338 | if cachedQuery == '%s_%s'%(mode,query): |
---|
339 | # same query |
---|
340 | return docinfo |
---|
341 | |
---|
342 | else: |
---|
343 | # different query |
---|
344 | del docinfo['resultSize'] |
---|
345 | del docinfo['resultXML'] |
---|
346 | |
---|
347 | # cache query |
---|
348 | docinfo['cachedQuery'] = '%s_%s'%(mode,query) |
---|
349 | |
---|
350 | # fetch full results |
---|
351 | docpath = docinfo['textURLPath'] |
---|
352 | params = {'document': docpath, |
---|
353 | 'mode': 'text', |
---|
354 | 'queryType': mode, |
---|
355 | 'query': query, |
---|
356 | 'queryResultPageSize': 1000, |
---|
357 | 'queryResultPN': 1, |
---|
358 | 'characterNormalization': pageinfo.get('characterNormalization', 'reg')} |
---|
359 | pagexml = self.getServerData("doc-query.xql",urllib.urlencode(params)) |
---|
360 | #pagexml = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&s=%s&viewMode=%s&characterNormalization=%s&highlightElementPos=%s&highlightElement=%s&highlightQuery=%s"%(docpath, 'text', queryType, urllib.quote(query), pagesize, pn, s, viewMode,characterNormalization, highlightElementPos, highlightElement, urllib.quote(highlightQuery))) |
---|
361 | dom = ET.fromstring(pagexml) |
---|
362 | # page content is in <div class="queryResultPage"> |
---|
363 | pagediv = None |
---|
364 | # ElementTree 1.2 in Python 2.6 can't do div[@class='queryResultPage'] |
---|
365 | alldivs = dom.findall("div") |
---|
366 | for div in alldivs: |
---|
367 | dc = div.get('class') |
---|
368 | # page content div |
---|
369 | if dc == 'queryResultPage': |
---|
370 | pagediv = div |
---|
371 | |
---|
372 | elif dc == 'queryResultHits': |
---|
373 | docinfo['resultSize'] = getInt(div.text) |
---|
374 | |
---|
375 | if pagediv is not None: |
---|
376 | # store XML in docinfo |
---|
377 | docinfo['resultXML'] = ET.tostring(pagediv, 'UTF-8') |
---|
378 | |
---|
379 | return docinfo |
---|
380 | |
---|
381 | |
---|
382 | def getResultsPage(self, mode="text", query=None, pn=None, start=None, size=None, pageinfo=None, docinfo=None): |
---|
383 | """returns single page from the table of contents""" |
---|
384 | logging.debug("getResultsPage mode=%s, pn=%s"%(mode,pn)) |
---|
385 | # get (cached) result |
---|
386 | self.getSearchResults(mode=mode, query=query, pageinfo=pageinfo, docinfo=docinfo) |
---|
387 | |
---|
388 | resultxml = docinfo.get('resultXML', None) |
---|
389 | if not resultxml: |
---|
390 | logging.error("getResultPage: unable to find resultXML") |
---|
391 | return "Error: no result!" |
---|
392 | |
---|
393 | if size is None: |
---|
394 | size = pageinfo.get('resultPageSize', 10) |
---|
395 | |
---|
396 | if start is None: |
---|
397 | start = (pn - 1) * size |
---|
398 | |
---|
399 | fullresult = ET.fromstring(resultxml) |
---|
400 | |
---|
401 | if fullresult is not None: |
---|
402 | # paginate |
---|
403 | first = start-1 |
---|
404 | len = size |
---|
405 | del fullresult[:first] |
---|
406 | del fullresult[len:] |
---|
407 | tocdivs = fullresult |
---|
408 | |
---|
409 | # check all a-tags |
---|
410 | links = tocdivs.findall(".//a") |
---|
411 | for l in links: |
---|
412 | href = l.get('href') |
---|
413 | if href: |
---|
414 | # assume all links go to pages |
---|
415 | linkUrl = urlparse.urlparse(href) |
---|
416 | linkParams = urlparse.parse_qs(linkUrl.query) |
---|
417 | # take some parameters |
---|
418 | params = {'pn': linkParams['pn'], |
---|
419 | 'highlightQuery': linkParams.get('highlightQuery',''), |
---|
420 | 'highlightElement': linkParams.get('highlightElement',''), |
---|
421 | 'highlightElementPos': linkParams.get('highlightElementPos','') |
---|
422 | } |
---|
423 | url = self.getLink(params=params) |
---|
424 | l.set('href', url) |
---|
425 | |
---|
426 | return serialize(tocdivs) |
---|
427 | |
---|
428 | return "ERROR: no results!" |
---|
429 | |
---|
430 | |
---|
431 | def getToc(self, mode='text', docinfo=None): |
---|
432 | """returns list of table of contents from docinfo""" |
---|
433 | logging.debug("getToc mode=%s"%mode) |
---|
434 | if mode == 'text': |
---|
435 | queryType = 'toc' |
---|
436 | else: |
---|
437 | queryType = mode |
---|
438 | |
---|
439 | if not 'full_%s'%queryType in docinfo: |
---|
440 | # get new toc |
---|
441 | docinfo = self.getTextInfo(queryType, docinfo) |
---|
442 | |
---|
443 | return docinfo.get('full_%s'%queryType, []) |
---|
444 | |
---|
445 | def getTocPage(self, mode='text', pn=None, start=None, size=None, pageinfo=None, docinfo=None): |
---|
446 | """returns single page from the table of contents""" |
---|
447 | logging.debug("getTocPage mode=%s, pn=%s start=%s size=%s"%(mode,repr(pn),repr(start),repr(size))) |
---|
448 | fulltoc = self.getToc(mode=mode, docinfo=docinfo) |
---|
449 | if len(fulltoc) < 1: |
---|
450 | logging.error("getTocPage: unable to find toc!") |
---|
451 | return "Error: no table of contents!" |
---|
452 | |
---|
453 | if size is None: |
---|
454 | size = pageinfo.get('tocPageSize', 30) |
---|
455 | |
---|
456 | if start is None: |
---|
457 | start = (pn - 1) * size |
---|
458 | |
---|
459 | # paginate |
---|
460 | first = (start - 1) |
---|
461 | last = first + size |
---|
462 | tocs = fulltoc[first:last] |
---|
463 | tp = '<div>' |
---|
464 | for toc in tocs: |
---|
465 | pageurl = self.getLink('pn', toc['pn']) |
---|
466 | tp += '<div class="tocline">' |
---|
467 | tp += '<div class="toc name">[%s %s]</div>'%(toc['level-string'], toc['content']) |
---|
468 | tp += '<div class="toc float right page"><a href="%s">Page: %s</a></div>'%(pageurl, toc['pn']) |
---|
469 | tp += '</div>\n' |
---|
470 | |
---|
471 | tp += '</div>\n' |
---|
472 | |
---|
473 | return tp |
---|
474 | |
---|
475 | |
---|
476 | def manage_changeHocrTextServer(self,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de/mpdl/interface/",timeout=40,repositoryType=None,RESPONSE=None): |
---|
477 | """change settings""" |
---|
478 | self.title=title |
---|
479 | self.timeout = timeout |
---|
480 | self.serverUrl = serverUrl |
---|
481 | if repositoryType: |
---|
482 | self.repositoryType = repositoryType |
---|
483 | if RESPONSE is not None: |
---|
484 | RESPONSE.redirect('manage_main') |
---|
485 | |
---|
486 | # management methods |
---|
487 | def manage_addHocrTextServerForm(self): |
---|
488 | """Form for adding""" |
---|
489 | pt = PageTemplateFile("zpt/manage_addHocrTextServer", globals()).__of__(self) |
---|
490 | return pt() |
---|
491 | |
---|
492 | def manage_addHocrTextServer(self,id,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de/mpdl/interface/",timeout=40,RESPONSE=None): |
---|
493 | #def manage_addHocrTextServer(self,id,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de:30030/mpdl/interface/",timeout=40,RESPONSE=None): |
---|
494 | """add zogiimage""" |
---|
495 | newObj = HocrTextServer(id,title,serverUrl,timeout) |
---|
496 | self.Destination()._setObject(id, newObj) |
---|
497 | if RESPONSE is not None: |
---|
498 | RESPONSE.redirect('manage_main') |
---|
499 | |
---|
500 | |
---|