Mercurial > hg > documentViewer
annotate MpdlXmlTextServer.py @ 569:be21250420be
MpdlXmlTextServer more resilient to non-existing doc-info.xql
author | casties |
---|---|
date | Fri, 12 Oct 2012 15:34:37 +0200 |
parents | 9255acc4518d |
children | 61d53ccbdd70 |
rev | line source |
---|---|
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
1 from OFS.SimpleItem import SimpleItem |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
2 from Products.PageTemplates.PageTemplateFile import PageTemplateFile |
455 | 3 |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
4 import xml.etree.ElementTree as ET |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
5 |
455 | 6 import re |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
7 import logging |
134 | 8 import urllib |
511 | 9 import urlparse |
506 | 10 import base64 |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
11 |
458 | 12 from SrvTxtUtils import getInt, getText, getHttpData |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
13 |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
14 def serialize(node): |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
15 """returns a string containing an XML snippet of node""" |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
16 s = ET.tostring(node, 'UTF-8') |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
17 # snip off XML declaration |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
18 if s.startswith('<?xml'): |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
19 i = s.find('?>') |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
20 return s[i+3:] |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
21 |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
22 return s |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
23 |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
24 |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
25 class MpdlXmlTextServer(SimpleItem): |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
26 """TextServer implementation for MPDL-XML eXist server""" |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
27 meta_type="MPDL-XML TextServer" |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
28 |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
29 manage_options=( |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
30 {'label':'Config','action':'manage_changeMpdlXmlTextServerForm'}, |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
31 )+SimpleItem.manage_options |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
32 |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
33 manage_changeMpdlXmlTextServerForm = PageTemplateFile("zpt/manage_changeMpdlXmlTextServer", globals()) |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
34 |
561
9255acc4518d
CLOSED - # 256: display texts from different backends (sandbox)
casties
parents:
559
diff
changeset
|
35 def __init__(self,id,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de/mpdl/interface/", serverName=None, timeout=40, repositoryType='production'): |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
36 """constructor""" |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
37 self.id=id |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
38 self.title=title |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
39 self.timeout = timeout |
561
9255acc4518d
CLOSED - # 256: display texts from different backends (sandbox)
casties
parents:
559
diff
changeset
|
40 self.repositoryType = repositoryType |
132 | 41 if serverName is None: |
42 self.serverUrl = serverUrl | |
43 else: | |
44 self.serverUrl = "http://%s/mpdl/interface/"%serverName | |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
45 |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
46 def getHttpData(self, url, data=None): |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
47 """returns result from url+data HTTP request""" |
458 | 48 return getHttpData(url,data,timeout=self.timeout) |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
49 |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
50 def getServerData(self, method, data=None): |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
51 """returns result from text server for method+data""" |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
52 url = self.serverUrl+method |
458 | 53 return getHttpData(url,data,timeout=self.timeout) |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
54 |
506 | 55 |
561
9255acc4518d
CLOSED - # 256: display texts from different backends (sandbox)
casties
parents:
559
diff
changeset
|
56 def getRepositoryType(self): |
9255acc4518d
CLOSED - # 256: display texts from different backends (sandbox)
casties
parents:
559
diff
changeset
|
57 """returns the repository type, e.g. 'production'""" |
9255acc4518d
CLOSED - # 256: display texts from different backends (sandbox)
casties
parents:
559
diff
changeset
|
58 return self.repositoryType |
9255acc4518d
CLOSED - # 256: display texts from different backends (sandbox)
casties
parents:
559
diff
changeset
|
59 |
559 | 60 def getTextDownloadUrl(self, type='xml', docinfo=None): |
61 """returns a URL to download the current text""" | |
62 docpath = docinfo.get('textURLPath', None) | |
63 if not docpath: | |
64 return None | |
65 | |
66 docpath = docpath.replace('.xml','.'+type) | |
67 url = '%sgetDoc?doc=%s'%(self.serverUrl.replace('interface/',''), docpath) | |
68 return url | |
69 | |
70 | |
506 | 71 def getPlacesOnPage(self, docinfo=None, pn=None): |
72 """Returns list of GIS places of page pn""" | |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
73 docpath = docinfo.get('textURLPath',None) |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
74 if not docpath: |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
75 return None |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
76 |
506 | 77 places=[] |
78 text=self.getServerData("xpath.xql", "document=%s&xpath=//place&pn=%s"%(docpath,pn)) | |
455 | 79 dom = ET.fromstring(text) |
506 | 80 result = dom.findall(".//resultPage/place") |
236 | 81 for l in result: |
506 | 82 id = l.get("id") |
83 name = l.text | |
84 place = {'id': id, 'name': name} | |
85 places.append(place) | |
86 | |
87 return places | |
307 | 88 |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
89 |
517 | 90 def getTextInfo(self, mode='', docinfo=None): |
516 | 91 """reads document info, including page concordance, from text server""" |
518
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
92 logging.debug("getTextInfo mode=%s"%mode) |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
93 if mode not in ['toc', 'figures', '']: |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
94 mode = '' |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
95 # check cached info |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
96 if mode: |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
97 # cached toc-request? |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
98 if 'full_%s'%mode in docinfo: |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
99 return docinfo |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
100 |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
101 else: |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
102 # no toc-request |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
103 if 'numTextPages' in docinfo: |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
104 return docinfo |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
105 |
516 | 106 docpath = docinfo.get('textURLPath', None) |
107 if docpath is None: | |
108 logging.error("getTextInfo: no textURLPath!") | |
109 return docinfo | |
569
be21250420be
MpdlXmlTextServer more resilient to non-existing doc-info.xql
casties
parents:
561
diff
changeset
|
110 |
be21250420be
MpdlXmlTextServer more resilient to non-existing doc-info.xql
casties
parents:
561
diff
changeset
|
111 try: |
be21250420be
MpdlXmlTextServer more resilient to non-existing doc-info.xql
casties
parents:
561
diff
changeset
|
112 # we need to set a result set size |
be21250420be
MpdlXmlTextServer more resilient to non-existing doc-info.xql
casties
parents:
561
diff
changeset
|
113 pagesize = 10000 |
be21250420be
MpdlXmlTextServer more resilient to non-existing doc-info.xql
casties
parents:
561
diff
changeset
|
114 pn = 1 |
be21250420be
MpdlXmlTextServer more resilient to non-existing doc-info.xql
casties
parents:
561
diff
changeset
|
115 # fetch docinfo |
be21250420be
MpdlXmlTextServer more resilient to non-existing doc-info.xql
casties
parents:
561
diff
changeset
|
116 pagexml = self.getServerData("doc-info.xql","document=%s&info=%s&pageSize=%s&pn=%s"%(docpath,mode,pagesize,pn)) |
be21250420be
MpdlXmlTextServer more resilient to non-existing doc-info.xql
casties
parents:
561
diff
changeset
|
117 dom = ET.fromstring(pagexml) |
be21250420be
MpdlXmlTextServer more resilient to non-existing doc-info.xql
casties
parents:
561
diff
changeset
|
118 # all info in tag <document> |
be21250420be
MpdlXmlTextServer more resilient to non-existing doc-info.xql
casties
parents:
561
diff
changeset
|
119 doc = dom.find("document") |
be21250420be
MpdlXmlTextServer more resilient to non-existing doc-info.xql
casties
parents:
561
diff
changeset
|
120 except Exception, e: |
be21250420be
MpdlXmlTextServer more resilient to non-existing doc-info.xql
casties
parents:
561
diff
changeset
|
121 logging.error("getTextInfo: Error getting data: %s"%e) |
be21250420be
MpdlXmlTextServer more resilient to non-existing doc-info.xql
casties
parents:
561
diff
changeset
|
122 return docinfo |
be21250420be
MpdlXmlTextServer more resilient to non-existing doc-info.xql
casties
parents:
561
diff
changeset
|
123 |
516 | 124 if doc is None: |
125 logging.error("getTextInfo: unable to find document-tag!") | |
126 else: | |
127 # go through all child elements | |
128 for tag in doc: | |
129 name = tag.tag | |
130 # numTextPages | |
131 if name == 'countPages': | |
132 np = getInt(tag.text) | |
133 if np > 0: | |
134 docinfo['numTextPages'] = np | |
135 | |
136 # numFigureEntries | |
137 elif name == 'countFigureEntries': | |
138 docinfo['numFigureEntries'] = getInt(tag.text) | |
139 | |
140 # numTocEntries | |
141 elif name == 'countTocEntries': | |
142 # WTF: s1 = int(s)/30+1 | |
143 docinfo['numTocEntries'] = getInt(tag.text) | |
144 | |
145 # numPlaces | |
146 elif name == 'countPlaces': | |
147 docinfo['numPlaces'] = getInt(tag.text) | |
148 | |
149 # pageNumbers | |
150 elif name == 'pageNumbers': | |
151 # contains tags with page numbers | |
152 # <pn><n>4</n><no>4</no><non/></pn> | |
153 # n=scan number, no=original page no, non=normalized original page no | |
154 # pageNumbers is a dict indexed by scan number | |
155 pages = {} | |
156 for pn in tag: | |
157 page = {} | |
158 n = 0 | |
159 for p in pn: | |
160 if p.tag == 'n': | |
161 n = getInt(p.text) | |
518
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
162 page['pn'] = n |
516 | 163 elif p.tag == 'no': |
164 page['no'] = p.text | |
165 elif p.tag == 'non': | |
166 page['non'] = p.text | |
167 | |
168 if n > 0: | |
169 pages[n] = page | |
170 | |
171 docinfo['pageNumbers'] = pages | |
517 | 172 #logging.debug("got pageNumbers=%s"%repr(pages)) |
516 | 173 |
517 | 174 # toc |
175 elif name == 'toc': | |
518
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
176 # contains tags with table of contents/figures |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
177 # <toc-entry><page>13</page><level>3</level><content>Chapter I</content><level-string>1.</level-string><real-level>1</real-level></toc-entry> |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
178 tocs = [] |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
179 for te in tag: |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
180 toc = {} |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
181 for t in te: |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
182 if t.tag == 'page': |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
183 toc['pn'] = getInt(t.text) |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
184 elif t.tag == 'level': |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
185 toc['level'] = t.text |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
186 elif t.tag == 'content': |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
187 toc['content'] = t.text |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
188 elif t.tag == 'level-string': |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
189 toc['level-string'] = t.text |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
190 elif t.tag == 'real-level': |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
191 toc['real-level'] = t.text |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
192 |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
193 tocs.append(toc) |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
194 |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
195 # save as full_toc/full_figures |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
196 docinfo['full_%s'%mode] = tocs |
517 | 197 |
516 | 198 return docinfo |
199 | |
200 | |
455 | 201 def processPageInfo(self, dom, docinfo, pageinfo): |
202 """processes page info divs from dom and stores in docinfo and pageinfo""" | |
469 | 203 # assume first second level div is pageMeta |
204 alldivs = dom.find("div") | |
473 | 205 |
206 if alldivs is None or alldivs.get('class', '') != 'pageMeta': | |
207 logging.error("processPageInfo: pageMeta div not found!") | |
208 return | |
209 | |
455 | 210 for div in alldivs: |
211 dc = div.get('class') | |
212 | |
213 # pageNumberOrig | |
469 | 214 if dc == 'pageNumberOrig': |
455 | 215 pageinfo['pageNumberOrig'] = div.text |
216 | |
217 # pageNumberOrigNorm | |
218 elif dc == 'pageNumberOrigNorm': | |
219 pageinfo['pageNumberOrigNorm'] = div.text | |
220 | |
474 | 221 # pageHeaderTitle |
222 elif dc == 'pageHeaderTitle': | |
223 pageinfo['pageHeaderTitle'] = div.text | |
517 | 224 |
473 | 225 #logging.debug("processPageInfo: pageinfo=%s"%repr(pageinfo)) |
455 | 226 return |
227 | |
388 | 228 |
471 | 229 def getTextPage(self, mode="text", pn=1, docinfo=None, pageinfo=None): |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
230 """returns single page from fulltext""" |
508 | 231 |
455 | 232 logging.debug("getTextPage mode=%s, pn=%s"%(mode,pn)) |
478 | 233 # check for cached text -- but ideally this shouldn't be called twice |
455 | 234 if pageinfo.has_key('textPage'): |
235 logging.debug("getTextPage: using cached text") | |
236 return pageinfo['textPage'] | |
237 | |
530 | 238 docpath = docinfo.get('textURLPath', None) |
239 if not docpath: | |
240 return None | |
241 | |
455 | 242 # just checking |
243 if pageinfo['current'] != pn: | |
244 logging.warning("getTextPage: current!=pn!") | |
245 | |
246 # stuff for constructing full urls | |
478 | 247 selfurl = docinfo['viewerUrl'] |
511 | 248 textParams = {'document': docpath, |
249 'pn': pn} | |
250 if 'characterNormalization' in pageinfo: | |
251 textParams['characterNormalization'] = pageinfo['characterNormalization'] | |
455 | 252 |
508 | 253 if not mode: |
254 # default is dict | |
255 mode = 'text' | |
256 | |
257 modes = mode.split(',') | |
258 # check for multiple layers | |
259 if len(modes) > 1: | |
260 logging.debug("getTextPage: more than one mode=%s"%mode) | |
527 | 261 |
508 | 262 # search mode |
263 if 'search' in modes: | |
264 # add highlighting | |
265 highlightQuery = pageinfo.get('highlightQuery', None) | |
511 | 266 if highlightQuery: |
267 textParams['highlightQuery'] = highlightQuery | |
268 textParams['highlightElement'] = pageinfo.get('highlightElement', '') | |
269 textParams['highlightElementPos'] = pageinfo.get('highlightElementPos', '') | |
508 | 270 |
511 | 271 # ignore mode in the following |
508 | 272 modes.remove('search') |
273 | |
551 | 274 # pundit mode |
275 punditMode = False | |
276 if 'pundit' in modes: | |
277 punditMode = True | |
278 # ignore mode in the following | |
279 modes.remove('pundit') | |
280 | |
508 | 281 # other modes don't combine |
282 if 'dict' in modes: | |
473 | 283 # dict is called textPollux in the backend |
508 | 284 textmode = 'textPollux' |
529 | 285 elif 'xml' in modes: |
286 # xml mode | |
287 textmode = 'xml' | |
288 textParams['characterNormalization'] = 'orig' | |
544 | 289 elif 'gis' in modes: |
290 textmode = 'gis' | |
527 | 291 else: |
508 | 292 # text is default mode |
293 textmode = 'text' | |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
294 |
511 | 295 textParams['mode'] = textmode |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
296 |
455 | 297 # fetch the page |
511 | 298 pagexml = self.getServerData("page-fragment.xql",urllib.urlencode(textParams)) |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
299 dom = ET.fromstring(pagexml) |
455 | 300 # extract additional info |
301 self.processPageInfo(dom, docinfo, pageinfo) | |
302 # page content is in <div class="pageContent"> | |
303 pagediv = None | |
304 # ElementTree 1.2 in Python 2.6 can't do div[@class='pageContent'] | |
469 | 305 # so we look at the second level divs |
554 | 306 alldivs = dom.findall('div') |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
307 for div in alldivs: |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
308 dc = div.get('class') |
455 | 309 # page content div |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
310 if dc == 'pageContent': |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
311 pagediv = div |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
312 break |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
313 |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
314 # plain text mode |
508 | 315 if textmode == "text": |
478 | 316 # get full url assuming documentViewer is parent |
317 selfurl = self.getLink() | |
473 | 318 if pagediv is not None: |
551 | 319 if punditMode: |
320 pagediv = self.addPunditAttributes(pagediv, pageinfo, docinfo) | |
321 | |
554 | 322 # fix empty div tags |
323 divs = pagediv.findall('.//div') | |
324 for d in divs: | |
325 if len(d) == 0 and not d.text: | |
326 # make empty divs non-empty | |
327 d.text = ' ' | |
328 | |
551 | 329 # check all a-tags |
554 | 330 links = pagediv.findall('.//a') |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
331 for l in links: |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
332 href = l.get('href') |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
333 if href and href.startswith('#note-'): |
478 | 334 href = href.replace('#note-',"%s#note-"%selfurl) |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
335 l.set('href', href) |
455 | 336 |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
337 return serialize(pagediv) |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
338 |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
339 # text-with-links mode |
508 | 340 elif textmode == "textPollux": |
473 | 341 if pagediv is not None: |
478 | 342 viewerurl = docinfo['viewerUrl'] |
343 selfurl = self.getLink() | |
551 | 344 if punditMode: |
345 pagediv = self.addPunditAttributes(pagediv, pageinfo, docinfo) | |
346 | |
554 | 347 # fix empty div tags |
348 divs = pagediv.findall('.//div') | |
349 for d in divs: | |
350 if len(d) == 0 and not d.text: | |
351 # make empty divs non-empty | |
352 d.text = ' ' | |
353 | |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
354 # check all a-tags |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
355 links = pagediv.findall(".//a") |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
356 for l in links: |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
357 href = l.get('href') |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
358 |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
359 if href: |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
360 # is link with href |
511 | 361 linkurl = urlparse.urlparse(href) |
362 #logging.debug("getTextPage: linkurl=%s"%repr(linkurl)) | |
363 if linkurl.path.endswith('GetDictionaryEntries'): | |
364 #TODO: replace wordInfo page | |
478 | 365 # is dictionary link - change href (keeping parameters) |
511 | 366 #l.set('href', href.replace('http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/lt/wordInfo.xql','%s/template/viewer_wordinfo'%viewerurl)) |
478 | 367 # add target to open new page |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
368 l.set('target', '_blank') |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
369 |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
370 if href.startswith('#note-'): |
478 | 371 # note link |
372 l.set('href', href.replace('#note-',"%s#note-"%selfurl)) | |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
373 |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
374 return serialize(pagediv) |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
375 |
455 | 376 # xml mode |
508 | 377 elif textmode == "xml": |
473 | 378 if pagediv is not None: |
455 | 379 return serialize(pagediv) |
380 | |
544 | 381 # pureXml mode WTF? |
508 | 382 elif textmode == "pureXml": |
473 | 383 if pagediv is not None: |
455 | 384 return serialize(pagediv) |
385 | |
386 # gis mode | |
508 | 387 elif textmode == "gis": |
473 | 388 if pagediv is not None: |
554 | 389 # fix empty div tags |
390 divs = pagediv.findall('.//div') | |
391 for d in divs: | |
392 if len(d) == 0 and not d.text: | |
393 # make empty divs non-empty | |
394 d.text = ' ' | |
395 | |
455 | 396 # check all a-tags |
397 links = pagediv.findall(".//a") | |
506 | 398 # add our URL as backlink |
399 selfurl = self.getLink() | |
400 doc = base64.b64encode(selfurl) | |
455 | 401 for l in links: |
402 href = l.get('href') | |
403 if href: | |
506 | 404 if href.startswith('http://mappit.mpiwg-berlin.mpg.de'): |
405 l.set('href', re.sub(r'doc=[\w+/=]+', 'doc=%s'%doc, href)) | |
406 l.set('target', '_blank') | |
455 | 407 |
408 return serialize(pagediv) | |
409 | |
501 | 410 return None |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
411 |
551 | 412 def addPunditAttributes(self, pagediv, pageinfo, docinfo): |
413 """add about attributes for pundit annotation tool""" | |
414 textid = docinfo.get('DRI', "fn=%s"%docinfo.get('documentPath', '???')) | |
415 pn = pageinfo.get('pn', '1') | |
416 # TODO: use pn as well? | |
417 # check all div-tags | |
418 divs = pagediv.findall(".//div") | |
419 for d in divs: | |
420 id = d.get('id') | |
421 if id: | |
422 d.set('about', "http://echo.mpiwg-berlin.mpg.de/%s/pn=%s/#%s"%(textid,pn,id)) | |
423 cls = d.get('class','') | |
424 cls += ' pundit-content' | |
425 d.set('class', cls.strip()) | |
426 | |
427 return pagediv | |
455 | 428 |
509
9d05befdd462
try to get characterNormalization in search result working.
casties
parents:
508
diff
changeset
|
429 def getSearchResults(self, mode, query=None, pageinfo=None, docinfo=None): |
508 | 430 """loads list of search results and stores XML in docinfo""" |
511 | 431 |
508 | 432 logging.debug("getSearchResults mode=%s query=%s"%(mode, query)) |
433 if mode == "none": | |
434 return docinfo | |
435 | |
511 | 436 cachedQuery = docinfo.get('cachedQuery', None) |
437 if cachedQuery is not None: | |
438 # cached search result | |
439 if cachedQuery == '%s_%s'%(mode,query): | |
440 # same query | |
441 return docinfo | |
442 | |
443 else: | |
444 # different query | |
445 del docinfo['resultSize'] | |
446 del docinfo['resultXML'] | |
508 | 447 |
511 | 448 # cache query |
449 docinfo['cachedQuery'] = '%s_%s'%(mode,query) | |
509
9d05befdd462
try to get characterNormalization in search result working.
casties
parents:
508
diff
changeset
|
450 |
508 | 451 # fetch full results |
511 | 452 docpath = docinfo['textURLPath'] |
508 | 453 params = {'document': docpath, |
454 'mode': 'text', | |
455 'queryType': mode, | |
456 'query': query, | |
457 'queryResultPageSize': 1000, | |
509
9d05befdd462
try to get characterNormalization in search result working.
casties
parents:
508
diff
changeset
|
458 'queryResultPN': 1, |
9d05befdd462
try to get characterNormalization in search result working.
casties
parents:
508
diff
changeset
|
459 'characterNormalization': pageinfo.get('characterNormalization', 'reg')} |
508 | 460 pagexml = self.getServerData("doc-query.xql",urllib.urlencode(params)) |
461 #pagexml = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&s=%s&viewMode=%s&characterNormalization=%s&highlightElementPos=%s&highlightElement=%s&highlightQuery=%s"%(docpath, 'text', queryType, urllib.quote(query), pagesize, pn, s, viewMode,characterNormalization, highlightElementPos, highlightElement, urllib.quote(highlightQuery))) | |
462 dom = ET.fromstring(pagexml) | |
463 # page content is in <div class="queryResultPage"> | |
464 pagediv = None | |
465 # ElementTree 1.2 in Python 2.6 can't do div[@class='queryResultPage'] | |
466 alldivs = dom.findall("div") | |
467 for div in alldivs: | |
468 dc = div.get('class') | |
469 # page content div | |
470 if dc == 'queryResultPage': | |
471 pagediv = div | |
472 | |
473 elif dc == 'queryResultHits': | |
511 | 474 docinfo['resultSize'] = getInt(div.text) |
508 | 475 |
510 | 476 if pagediv is not None: |
508 | 477 # store XML in docinfo |
511 | 478 docinfo['resultXML'] = ET.tostring(pagediv, 'UTF-8') |
508 | 479 |
480 return docinfo | |
481 | |
482 | |
483 def getResultsPage(self, mode="text", query=None, pn=None, start=None, size=None, pageinfo=None, docinfo=None): | |
484 """returns single page from the table of contents""" | |
485 logging.debug("getResultsPage mode=%s, pn=%s"%(mode,pn)) | |
516 | 486 # get (cached) result |
487 self.getSearchResults(mode=mode, query=query, pageinfo=pageinfo, docinfo=docinfo) | |
508 | 488 |
511 | 489 resultxml = docinfo.get('resultXML', None) |
508 | 490 if not resultxml: |
491 logging.error("getResultPage: unable to find resultXML") | |
492 return "Error: no result!" | |
493 | |
494 if size is None: | |
511 | 495 size = pageinfo.get('resultPageSize', 10) |
508 | 496 |
497 if start is None: | |
498 start = (pn - 1) * size | |
499 | |
500 fullresult = ET.fromstring(resultxml) | |
501 | |
510 | 502 if fullresult is not None: |
508 | 503 # paginate |
511 | 504 first = start-1 |
508 | 505 len = size |
506 del fullresult[:first] | |
507 del fullresult[len:] | |
508 tocdivs = fullresult | |
509 | |
510 # check all a-tags | |
511 links = tocdivs.findall(".//a") | |
512 for l in links: | |
513 href = l.get('href') | |
514 if href: | |
511 | 515 # assume all links go to pages |
516 linkUrl = urlparse.urlparse(href) | |
517 linkParams = urlparse.parse_qs(linkUrl.query) | |
518 # take some parameters | |
519 params = {'pn': linkParams['pn'], | |
520 'highlightQuery': linkParams.get('highlightQuery',''), | |
521 'highlightElement': linkParams.get('highlightElement',''), | |
522 'highlightElementPos': linkParams.get('highlightElementPos','') | |
523 } | |
524 url = self.getLink(params=params) | |
525 l.set('href', url) | |
508 | 526 |
527 return serialize(tocdivs) | |
528 | |
529 return "ERROR: no results!" | |
530 | |
531 | |
518
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
532 def getToc(self, mode='text', docinfo=None): |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
533 """returns list of table of contents from docinfo""" |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
534 logging.debug("getToc mode=%s"%mode) |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
535 if mode == 'text': |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
536 queryType = 'toc' |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
537 else: |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
538 queryType = mode |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
539 |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
540 if not 'full_%s'%queryType in docinfo: |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
541 # get new toc |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
542 docinfo = self.getTextInfo(queryType, docinfo) |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
543 |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
544 return docinfo.get('full_%s'%queryType, []) |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
545 |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
546 def getTocPage(self, mode='text', pn=None, start=None, size=None, pageinfo=None, docinfo=None): |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
547 """returns single page from the table of contents""" |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
548 logging.debug("getTocPage mode=%s, pn=%s start=%s size=%s"%(mode,repr(pn),repr(start),repr(size))) |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
549 fulltoc = self.getToc(mode=mode, docinfo=docinfo) |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
550 if len(fulltoc) < 1: |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
551 logging.error("getTocPage: unable to find toc!") |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
552 return "Error: no table of contents!" |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
553 |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
554 if size is None: |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
555 size = pageinfo.get('tocPageSize', 30) |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
556 |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
557 if start is None: |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
558 start = (pn - 1) * size |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
559 |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
560 # paginate |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
561 first = (start - 1) |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
562 last = first + size |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
563 tocs = fulltoc[first:last] |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
564 tp = '<div>' |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
565 for toc in tocs: |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
566 pageurl = self.getLink('pn', toc['pn']) |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
567 tp += '<div class="tocline">' |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
568 tp += '<div class="toc name">[%s %s]</div>'%(toc['level-string'], toc['content']) |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
569 tp += '<div class="toc float right page"><a href="%s">Page: %s</a></div>'%(pageurl, toc['pn']) |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
570 tp += '</div>\n' |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
571 |
519
9a3cc3732194
uses xml from doc-info.xql for table of contents now.
casties
parents:
518
diff
changeset
|
572 tp += '</div>\n' |
518
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
573 |
91051b36b9cc
uses xml info from doc-info.xql for table of contents now.
casties
parents:
517
diff
changeset
|
574 return tp |
519
9a3cc3732194
uses xml from doc-info.xql for table of contents now.
casties
parents:
518
diff
changeset
|
575 |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
576 |
561
9255acc4518d
CLOSED - # 256: display texts from different backends (sandbox)
casties
parents:
559
diff
changeset
|
577 def manage_changeMpdlXmlTextServer(self,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de/mpdl/interface/",timeout=40,repositoryType=None,RESPONSE=None): |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
578 """change settings""" |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
579 self.title=title |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
580 self.timeout = timeout |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
581 self.serverUrl = serverUrl |
561
9255acc4518d
CLOSED - # 256: display texts from different backends (sandbox)
casties
parents:
559
diff
changeset
|
582 if repositoryType: |
9255acc4518d
CLOSED - # 256: display texts from different backends (sandbox)
casties
parents:
559
diff
changeset
|
583 self.repositoryType = repositoryType |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
584 if RESPONSE is not None: |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
585 RESPONSE.redirect('manage_main') |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
586 |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
587 # management methods |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
588 def manage_addMpdlXmlTextServerForm(self): |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
589 """Form for adding""" |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
590 pt = PageTemplateFile("zpt/manage_addMpdlXmlTextServer", globals()).__of__(self) |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
591 return pt() |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
592 |
453
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
593 def manage_addMpdlXmlTextServer(self,id,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de/mpdl/interface/",timeout=40,RESPONSE=None): |
beb7ccb92564
first version using elementtree instead of 4suite xml
casties
parents:
407
diff
changeset
|
594 #def manage_addMpdlXmlTextServer(self,id,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de:30030/mpdl/interface/",timeout=40,RESPONSE=None): |
129
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
595 """add zogiimage""" |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
596 newObj = MpdlXmlTextServer(id,title,serverUrl,timeout) |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
597 self.Destination()._setObject(id, newObj) |
9404b6c37920
more modular version with separate object MpdlXmlTextServer
casties
parents:
diff
changeset
|
598 if RESPONSE is not None: |
455 | 599 RESPONSE.redirect('manage_main') |
600 | |
601 |