Mercurial > hg > documentViewer

comparison HocrTextServer.py @ 617:7aefbddddaf9

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
alpaha of hocr server support
author dwinter
date Wed, 23 Jul 2014 17:36:04 +0200
parents
children
comparison
equal deleted inserted replaced
616:3f9b42840901 617:7aefbddddaf9
1 from OFS.SimpleItem import SimpleItem
2 from Products.PageTemplates.PageTemplateFile import PageTemplateFile
3
4 import xml.etree.ElementTree as ET
5
6 import re
7 import logging
8 import urllib
9 import urlparse
10 import base64
11
12 from HocrTxtUtils import getInt, getText, getHttpData
13
14 def serialize(node):
15 """returns a string containing an XML snippet of node"""
16 s = ET.tostring(node, 'UTF-8')
17 # snip off XML declaration
18 if s.startswith('<?xml'):
19 i = s.find('?>')
20 return s[i+3:]
21
22 return s
23
24
25 class HocrTextServer(SimpleItem):
26 """TextServer implementation for MPDL-XML eXist server"""
27 meta_type="Hocr TextServer"
28
29 manage_options=(
30 {'label':'Config','action':'manage_changeHocrTextServerForm'},
31 )+SimpleItem.manage_options
32
33 manage_changeHocrTextServerForm = PageTemplateFile("zpt/manage_changeHocrTextServer", globals())
34
35 def __init__(self,id,title="",serverUrl="http://localhost:8080/hocr", timeout=40, repositoryType='production'):
36 """constructor"""
37 self.id=id
38 self.title=title
39 self.timeout = timeout
40 self.repositoryType = repositoryType
41
42 self.serverUrl = serverUrl
43
44 def getHttpData(self, url, data=None):
45 """returns result from url+data HTTP request"""
46 return getHttpData(url,data,timeout=self.timeout)
47
48 def getServerData(self, pn, data=None):
49 """returns result from text server for method+data"""
50 url = self.serverUrl
51 return getHttpData(url,pn,data=data,timeout=self.timeout)
52
53
54 def getRepositoryType(self):
55 """returns the repository type, e.g. 'production'"""
56 return getattr(self, 'repositoryType', None)
57
58 def getTextDownloadUrl(self, type='xml', docinfo=None):
59 """returns a URL to download the current text"""
60 docpath = docinfo.get('textURLPath', None)
61 if not docpath:
62 return None
63
64 docpath = docpath.replace('.xml','.'+type)
65 url = '%sgetDoc?doc=%s'%(self.serverUrl.replace('interface/',''), docpath)
66 return url
67
68
69 def getPlacesOnPage(self, docinfo=None, pn=None):
70 """Returns list of GIS places of page pn"""
71 docpath = docinfo.get('textURLPath',None)
72 if not docpath:
73 return None
74
75 places=[]
76 text=self.getServerData("xpath.xql", "document=%s&xpath=//place&pn=%s"%(docpath,pn))
77 dom = ET.fromstring(text)
78 result = dom.findall(".//resultPage/place")
79 for l in result:
80 id = l.get("id")
81 name = l.text
82 place = {'id': id, 'name': name}
83 places.append(place)
84
85 return places
86
87
88 def getTextInfo(self, mode='', docinfo=None):
89 """reads document info, including page concordance, from text server"""
90 logging.debug("getTextInfo mode=%s"%mode)
91 if mode not in ['toc', 'figures', '']:
92 mode = ''
93 # check cached info
94 if mode:
95 # cached toc-request?
96 if 'full_%s'%mode in docinfo:
97 return docinfo
98
99 else:
100 # no toc-request
101 if 'numTextPages' in docinfo:
102 return docinfo
103
104 docpath = docinfo.get('textURLPath', None)
105 if docpath is None:
106 logging.error("getTextInfo: no textURLPath!")
107 return docinfo
108
109 try:
110 # we need to set a result set size
111 pagesize = 10000
112 pn = 1
113 # fetch docinfo
114 pagexml = self.getServerData("doc-info.xql","document=%s&info=%s&pageSize=%s&pn=%s"%(docpath,mode,pagesize,pn))
115 dom = ET.fromstring(pagexml)
116 # all info in tag <document>
117 doc = dom.find("document")
118 except Exception, e:
119 logging.error("getTextInfo: Error reading doc info: %s"%e)
120 return docinfo
121
122 if doc is None:
123 logging.error("getTextInfo: unable to find document-tag!")
124 else:
125 # go through all child elements
126 for tag in doc:
127 name = tag.tag
128 # numTextPages
129 if name == 'countPages':
130 np = getInt(tag.text)
131 if np > 0:
132 docinfo['numTextPages'] = np
133
134 # numFigureEntries
135 elif name == 'countFigureEntries':
136 docinfo['numFigureEntries'] = getInt(tag.text)
137
138 # numTocEntries
139 elif name == 'countTocEntries':
140 # WTF: s1 = int(s)/30+1
141 docinfo['numTocEntries'] = getInt(tag.text)
142
143 # numPlaces
144 elif name == 'countPlaces':
145 docinfo['numPlaces'] = getInt(tag.text)
146
147 # pageNumbers
148 elif name == 'pageNumbers':
149 # contains tags with page numbers
150 # <pn><n>4</n><no>4</no><non/></pn>
151 # n=scan number, no=original page no, non=normalized original page no
152 # pageNumbers is a dict indexed by scan number
153 pages = {}
154 for pn in tag:
155 page = {}
156 n = 0
157 for p in pn:
158 if p.tag == 'n':
159 n = getInt(p.text)
160 page['pn'] = n
161 elif p.tag == 'no':
162 page['no'] = p.text
163 elif p.tag == 'non':
164 page['non'] = p.text
165
166 if n > 0:
167 pages[n] = page
168
169 docinfo['pageNumbers'] = pages
170 #logging.debug("got pageNumbers=%s"%repr(pages))
171
172 # toc
173 elif name == 'toc':
174 # contains tags with table of contents/figures
175 # <toc-entry><page>13</page><level>3</level><content>Chapter I</content><level-string>1.</level-string><real-level>1</real-level></toc-entry>
176 tocs = []
177 for te in tag:
178 toc = {}
179 for t in te:
180 if t.tag == 'page':
181 toc['pn'] = getInt(t.text)
182 elif t.tag == 'level':
183 toc['level'] = t.text
184 elif t.tag == 'content':
185 toc['content'] = t.text
186 elif t.tag == 'level-string':
187 toc['level-string'] = t.text
188 elif t.tag == 'real-level':
189 toc['real-level'] = t.text
190
191 tocs.append(toc)
192
193 # save as full_toc/full_figures
194 docinfo['full_%s'%mode] = tocs
195
196 return docinfo
197
198
199 def processPageInfo(self, dom, docinfo, pageinfo):
200 """processes page info divs from dom and stores in docinfo and pageinfo"""
201 # assume first second level div is pageMeta
202 alldivs = dom.find("div")
203
204 if alldivs is None or alldivs.get('class', '') != 'pageMeta':
205 logging.error("processPageInfo: pageMeta div not found!")
206 return
207
208 for div in alldivs:
209 dc = div.get('class')
210
211 # pageNumberOrig
212 if dc == 'pageNumberOrig':
213 pageinfo['pageNumberOrig'] = div.text
214
215 # pageNumberOrigNorm
216 elif dc == 'pageNumberOrigNorm':
217 pageinfo['pageNumberOrigNorm'] = div.text
218
219 # pageHeaderTitle
220 elif dc == 'pageHeaderTitle':
221 pageinfo['pageHeaderTitle'] = div.text
222
223 #logging.debug("processPageInfo: pageinfo=%s"%repr(pageinfo))
224 return
225
226
227 def getTextPage(self, mode="text", pn=1, docinfo=None, pageinfo=None):
228 """returns single page from fulltext"""
229
230
231 logging.debug("getTextPage Hocr mode=%s, pn=%s"%(mode,pn))
232 # check for cached text -- but ideally this shouldn't be called twice
233 if pageinfo.has_key('textPage'):
234 logging.debug("getTextPage: using cached text")
235 return pageinfo['textPage']
236
237 docpath = docinfo.get('textURLPath', None)
238
239 docpath=docpath.replace("pages","hocr")
240
241 logging.debug("getTextPage docpath= %s"%docpath)
242 if not docpath:
243 return None
244
245 # stuff for constructing full urls
246 selfurl = docinfo['viewerUrl']
247 textParams = {'document': docpath,
248 'pn': pn}
249 if 'characterNormalization' in pageinfo:
250 textParams['characterNormalization'] = pageinfo['characterNormalization']
251
252 if not mode:
253 # default is dict
254 mode = 'text'
255
256 logging.debug("getTextPage mode=%s, pn=%s"%(mode,pn))
257 modes = mode.split(',')
258 # check for multiple layers
259 if len(modes) > 1:
260 logging.debug("getTextPage: more than one mode=%s"%mode)
261
262 # search mode
263 if 'search' in modes:
264 # add highlighting
265 highlightQuery = pageinfo.get('highlightQuery', None)
266 if highlightQuery:
267 textParams['highlightQuery'] = highlightQuery
268 textParams['highlightElement'] = pageinfo.get('highlightElement', '')
269 textParams['highlightElementPos'] = pageinfo.get('highlightElementPos', '')
270
271 # ignore mode in the following
272 modes.remove('search')
273
274 # pundit mode
275 punditMode = False
276 if 'pundit' in modes:
277 punditMode = True
278 # ignore mode in the following
279 modes.remove('pundit')
280
281 # other modes don't combine
282 if 'dict' in modes:
283 # dict is called textPollux in the backend
284 textmode = 'textPollux'
285 elif 'xml' in modes:
286 # xml mode
287 textmode = 'xml'
288 textParams['characterNormalization'] = 'orig'
289 elif 'gis' in modes:
290 textmode = 'gis'
291 else:
292 # text is default mode
293 textmode = 'text'
294
295 textParams['mode'] = textmode
296
297 logging.debug("getTextPage (textparams: %s"%textParams)
298
299 try:
300 # fetch the page
301 pagexml = self.getServerData(pn,urllib.urlencode(textParams))
302 return pagexml
303 except Exception, e:
304 logging.error("getTextPage: Error reading page: %s"%e)
305 return None
306
307
308
309 return None
310
311 def addPunditAttributes(self, pagediv, pageinfo, docinfo):
312 """add about attributes for pundit annotation tool"""
313 textid = docinfo.get('DRI', "fn=%s"%docinfo.get('documentPath', '???'))
314 pn = pageinfo.get('pn', '1')
315 # TODO: use pn as well?
316 # check all div-tags
317 divs = pagediv.findall(".//div")
318 for d in divs:
319 id = d.get('id')
320 if id:
321 d.set('about', "http://echo.mpiwg-berlin.mpg.de/%s/pn=%s/#%s"%(textid,pn,id))
322 cls = d.get('class','')
323 cls += ' pundit-content'
324 d.set('class', cls.strip())
325
326 return pagediv
327
328 def getSearchResults(self, mode, query=None, pageinfo=None, docinfo=None):
329 """loads list of search results and stores XML in docinfo"""
330
331 logging.debug("getSearchResults mode=%s query=%s"%(mode, query))
332 if mode == "none":
333 return docinfo
334
335 cachedQuery = docinfo.get('cachedQuery', None)
336 if cachedQuery is not None:
337 # cached search result
338 if cachedQuery == '%s_%s'%(mode,query):
339 # same query
340 return docinfo
341
342 else:
343 # different query
344 del docinfo['resultSize']
345 del docinfo['resultXML']
346
347 # cache query
348 docinfo['cachedQuery'] = '%s_%s'%(mode,query)
349
350 # fetch full results
351 docpath = docinfo['textURLPath']
352 params = {'document': docpath,
353 'mode': 'text',
354 'queryType': mode,
355 'query': query,
356 'queryResultPageSize': 1000,
357 'queryResultPN': 1,
358 'characterNormalization': pageinfo.get('characterNormalization', 'reg')}
359 pagexml = self.getServerData("doc-query.xql",urllib.urlencode(params))
360 #pagexml = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&s=%s&viewMode=%s&characterNormalization=%s&highlightElementPos=%s&highlightElement=%s&highlightQuery=%s"%(docpath, 'text', queryType, urllib.quote(query), pagesize, pn, s, viewMode,characterNormalization, highlightElementPos, highlightElement, urllib.quote(highlightQuery)))
361 dom = ET.fromstring(pagexml)
362 # page content is in <div class="queryResultPage">
363 pagediv = None
364 # ElementTree 1.2 in Python 2.6 can't do div[@class='queryResultPage']
365 alldivs = dom.findall("div")
366 for div in alldivs:
367 dc = div.get('class')
368 # page content div
369 if dc == 'queryResultPage':
370 pagediv = div
371
372 elif dc == 'queryResultHits':
373 docinfo['resultSize'] = getInt(div.text)
374
375 if pagediv is not None:
376 # store XML in docinfo
377 docinfo['resultXML'] = ET.tostring(pagediv, 'UTF-8')
378
379 return docinfo
380
381
382 def getResultsPage(self, mode="text", query=None, pn=None, start=None, size=None, pageinfo=None, docinfo=None):
383 """returns single page from the table of contents"""
384 logging.debug("getResultsPage mode=%s, pn=%s"%(mode,pn))
385 # get (cached) result
386 self.getSearchResults(mode=mode, query=query, pageinfo=pageinfo, docinfo=docinfo)
387
388 resultxml = docinfo.get('resultXML', None)
389 if not resultxml:
390 logging.error("getResultPage: unable to find resultXML")
391 return "Error: no result!"
392
393 if size is None:
394 size = pageinfo.get('resultPageSize', 10)
395
396 if start is None:
397 start = (pn - 1) * size
398
399 fullresult = ET.fromstring(resultxml)
400
401 if fullresult is not None:
402 # paginate
403 first = start-1
404 len = size
405 del fullresult[:first]
406 del fullresult[len:]
407 tocdivs = fullresult
408
409 # check all a-tags
410 links = tocdivs.findall(".//a")
411 for l in links:
412 href = l.get('href')
413 if href:
414 # assume all links go to pages
415 linkUrl = urlparse.urlparse(href)
416 linkParams = urlparse.parse_qs(linkUrl.query)
417 # take some parameters
418 params = {'pn': linkParams['pn'],
419 'highlightQuery': linkParams.get('highlightQuery',''),
420 'highlightElement': linkParams.get('highlightElement',''),
421 'highlightElementPos': linkParams.get('highlightElementPos','')
422 }
423 url = self.getLink(params=params)
424 l.set('href', url)
425
426 return serialize(tocdivs)
427
428 return "ERROR: no results!"
429
430
431 def getToc(self, mode='text', docinfo=None):
432 """returns list of table of contents from docinfo"""
433 logging.debug("getToc mode=%s"%mode)
434 if mode == 'text':
435 queryType = 'toc'
436 else:
437 queryType = mode
438
439 if not 'full_%s'%queryType in docinfo:
440 # get new toc
441 docinfo = self.getTextInfo(queryType, docinfo)
442
443 return docinfo.get('full_%s'%queryType, [])
444
445 def getTocPage(self, mode='text', pn=None, start=None, size=None, pageinfo=None, docinfo=None):
446 """returns single page from the table of contents"""
447 logging.debug("getTocPage mode=%s, pn=%s start=%s size=%s"%(mode,repr(pn),repr(start),repr(size)))
448 fulltoc = self.getToc(mode=mode, docinfo=docinfo)
449 if len(fulltoc) < 1:
450 logging.error("getTocPage: unable to find toc!")
451 return "Error: no table of contents!"
452
453 if size is None:
454 size = pageinfo.get('tocPageSize', 30)
455
456 if start is None:
457 start = (pn - 1) * size
458
459 # paginate
460 first = (start - 1)
461 last = first + size
462 tocs = fulltoc[first:last]
463 tp = '<div>'
464 for toc in tocs:
465 pageurl = self.getLink('pn', toc['pn'])
466 tp += '<div class="tocline">'
467 tp += '<div class="toc name">[%s %s]</div>'%(toc['level-string'], toc['content'])
468 tp += '<div class="toc float right page"><a href="%s">Page: %s</a></div>'%(pageurl, toc['pn'])
469 tp += '</div>\n'
470
471 tp += '</div>\n'
472
473 return tp
474
475
476 def manage_changeHocrTextServer(self,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de/mpdl/interface/",timeout=40,repositoryType=None,RESPONSE=None):
477 """change settings"""
478 self.title=title
479 self.timeout = timeout
480 self.serverUrl = serverUrl
481 if repositoryType:
482 self.repositoryType = repositoryType
483 if RESPONSE is not None:
484 RESPONSE.redirect('manage_main')
485
486 # management methods
487 def manage_addHocrTextServerForm(self):
488 """Form for adding"""
489 pt = PageTemplateFile("zpt/manage_addHocrTextServer", globals()).__of__(self)
490 return pt()
491
492 def manage_addHocrTextServer(self,id,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de/mpdl/interface/",timeout=40,RESPONSE=None):
493 #def manage_addHocrTextServer(self,id,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de:30030/mpdl/interface/",timeout=40,RESPONSE=None):
494 """add zogiimage"""
495 newObj = HocrTextServer(id,title,serverUrl,timeout)
496 self.Destination()._setObject(id, newObj)
497 if RESPONSE is not None:
498 RESPONSE.redirect('manage_main')
499
500