--- documentViewer/MpdlXmlTextServer.py 2010/06/17 17:38:59 1.3 +++ documentViewer/MpdlXmlTextServer.py 2010/12/23 17:05:01 1.220 @@ -1,15 +1,14 @@ from OFS.SimpleItem import SimpleItem from Products.PageTemplates.PageTemplateFile import PageTemplateFile - from Ft.Xml import EMPTY_NAMESPACE, Parse import sys import logging +import urllib import documentViewer from documentViewer import getTextFromNode, serializeNode - class MpdlXmlTextServer(SimpleItem): """TextServer implementation for MPDL-XML eXist server""" meta_type="MPDL-XML TextServer" @@ -29,25 +28,20 @@ class MpdlXmlTextServer(SimpleItem): self.serverUrl = serverUrl else: self.serverUrl = "http://%s/mpdl/interface/"%serverName - def getHttpData(self, url, data=None): """returns result from url+data HTTP request""" return documentViewer.getHttpData(url,data,timeout=self.timeout) - def getServerData(self, method, data=None): """returns result from text server for method+data""" url = self.serverUrl+method return documentViewer.getHttpData(url,data,timeout=self.timeout) - - def getSearch(self, pn=1, pageinfo=None, docinfo=None, query=None, queryType=None, lemma=None): + def getSearch(self, pn=1, pageinfo=None, docinfo=None, query=None, queryType=None, lemma=None, characterNormalization=None, optionToggle=None): """get search list""" docpath = docinfo['textURLPath'] url = docinfo['url'] - logging.debug("documentViewer (gettoc) docpath: %s"%(docpath)) - logging.debug("documentViewer (gettoc) url: %s"%(url)) pagesize = pageinfo['queryPageSize'] pn = pageinfo['searchPN'] sn = pageinfo['sn'] @@ -56,10 +50,12 @@ class MpdlXmlTextServer(SimpleItem): queryType =pageinfo['queryType'] viewMode= pageinfo['viewMode'] tocMode = pageinfo['tocMode'] + characterNormalization = pageinfo['characterNormalization'] + optionToggle = pageinfo['optionToggle'] tocPN = pageinfo['tocPN'] selfurl = self.absolute_url() - data = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&sn=%s&viewMode=%s&highlightQuery=%s"%(docpath, 'text', queryType, query, pagesize, pn, sn, viewMode,highlightQuery)) + data = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&sn=%s&viewMode=%s&characterNormalization=%s&optionToggle=%s&highlightQuery=%s"%(docpath, 'text', queryType, urllib.quote(query), pagesize, pn, sn, viewMode,characterNormalization,optionToggle ,urllib.quote(highlightQuery))) #page=self.template.fulltextclient.eval("/mpdl/interface/doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&sn=%s&viewMode=%s&highlightQuery=%s"%(docpath, 'text', queryType, query, pagesize, pn, sn, viewMode,highlightQuery) ,outputUnicode=False) pagexml = data.replace('?document=%s'%str(docpath),'?url=%s'%url) @@ -75,7 +71,7 @@ class MpdlXmlTextServer(SimpleItem): href = hrefNode.nodeValue if href.startswith('page-fragment.xql'): selfurl = self.absolute_url() - pagexml=href.replace('mode=text','mode=texttool&viewMode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&tocMode=%s&searchPN=%s&tocPN=%s'%(viewMode,queryType,query,pagesize,pn,tocMode,pn,tocPN)) + pagexml=href.replace('mode=text','mode=texttool&viewMode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&tocMode=%s&searchPN=%s&tocPN=%s&optionToggle=%s&characterNormalization=%s'%(viewMode,queryType,urllib.quote(query),pagesize,pn,tocMode,pn,tocPN,optionToggle,characterNormalization)) hrefNode.nodeValue = pagexml.replace('page-fragment.xql','%s'%selfurl) return serializeNode(pagenode) if (queryType=="fulltextMorph"): @@ -89,13 +85,13 @@ class MpdlXmlTextServer(SimpleItem): href = hrefNode.nodeValue if href.startswith('page-fragment.xql'): selfurl = self.absolute_url() - pagexml=href.replace('mode=text','mode=texttool&viewMode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&tocMode=%s&searchPN=%s&tocPN=%s'%(viewMode,queryType,query,pagesize,pn,tocMode,pn,tocPN)) + pagexml=href.replace('mode=text','mode=texttool&viewMode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&tocMode=%s&searchPN=%s&tocPN=%s&optionToggle=%s&characterNormalization=%s'%(viewMode,queryType,urllib.quote(query),pagesize,pn,tocMode,pn,tocPN,optionToggle,characterNormalization)) hrefNode.nodeValue = pagexml.replace('page-fragment.xql','%s'%selfurl) if href.startswith('../lt/lemma.xql'): hrefNode.nodeValue = href.replace('../lt/lemma.xql','%s/template/head_main_lemma_New'%(selfurl)) l.setAttributeNS(None, 'target', '_blank') l.setAttributeNS(None, 'onClick',"popupWin = window.open(this.href, 'contacts', 'location,width=500,height=600,top=180, left=400, scrollbars=1'); return false;") - l.setAttributeNS(None, 'onDblclick', 'popupWin.focus();') + l.setAttributeNS(None, 'onClick', 'popupWin.focus();') pagedivs = pagedom.xpath("//div[@class='queryResultMorphExpansion']") return serializeNode(pagenode) if (queryType=="ftIndex")or(queryType=="ftIndexMorph"): @@ -107,51 +103,181 @@ class MpdlXmlTextServer(SimpleItem): hrefNode = l.getAttributeNodeNS(None, u"href") if hrefNode: href = hrefNode.nodeValue - hrefNode.nodeValue=href.replace('mode=text','mode=texttool&viewMode=%s&tocMode=%s&tocPN=%s&pn=%s'%(viewMode,tocMode,tocPN,pn)) + hrefNode.nodeValue=href.replace('mode=text','mode=texttool&viewMode=%s&tocMode=%s&tocPN=%s&pn=%s&optionToggle=%s&characterNormalization=%s'%(viewMode,tocMode,tocPN,pn,optionToggle,characterNormalization)) if href.startswith('../lt/lex.xql'): hrefNode.nodeValue = href.replace('../lt/lex.xql','%s/template/head_main_voc'%selfurl) l.setAttributeNS(None, 'target', '_blank') l.setAttributeNS(None, 'onClick',"popupWin = window.open(this.href, 'contacts', 'location,width=500,height=600,top=180, left=400, scrollbars=1'); return false;") - l.setAttributeNS(None, 'onDblclick', 'popupWin.focus();') + l.setAttributeNS(None, 'onClick', 'popupWin.focus();') if href.startswith('../lt/lemma.xql'): hrefNode.nodeValue = href.replace('../lt/lemma.xql','%s/template/head_main_lemma'%selfurl) l.setAttributeNS(None, 'target', '_blank') l.setAttributeNS(None, 'onClick',"popupWin = window.open(this.href, 'contacts', 'location,width=500,height=600,top=180, left=400, scrollbars=1'); return false;") - l.setAttributeNS(None, 'onDblclick', 'popupWin.focus();') + l.setAttributeNS(None, 'onClick', 'popupWin.focus();') return serializeNode(pagenode) return "no text here" - def getNumPages(self,docinfo=None): - """get list of pages from fulltext and put in docinfo""" + + def getNumTextPages (self, docinfo=None): + """get list of pages from fulltext (texts without images) and put in docinfo""" + logging.debug("getNumTextPages") + docpath = docinfo['textURLPath'] + selfurl = self.absolute_url() + #viewMode= pageinfo['viewMode'] if 'numPages' in docinfo: - # already there + # allredy there return docinfo - xquery = '//pb' - text = self.getServerData("xquery.xql","document=%s&xquery=%s"%(docinfo['textURLPath'],xquery)) - #text = self.template.fulltextclient.eval("/mpdl/interface/xquery.xql", "document=%s&xquery=%s"%(docinfo['textURLPath'],xquery)) - docinfo['numPages'] = text.count("0: + docinfo['numPages']= int(getTextFromNode(pagedivs[0])) + return docinfo + + def getTocEntries (self, docinfo=None): + """ number of text entries""" + self.getInfoFromPage(docinfo) + return docinfo['tocEntries'] + + def getFigureEntries (self, docinfo=None): + """ number of figure entries""" + self.getInfoFromPage(docinfo) + return docinfo['figureEntries'] + + def getGisPlaces(self, docinfo=None, pageinfo=None): + """ Show all Gis Places of whole Page""" + xpath='//place' + docpath = docinfo.get('textURLPath',None) + if not docpath: + return None + + url = docinfo['url'] + selfurl = self.absolute_url() + pn = pageinfo['current'] + hrefList=[] + myList= "" + text=self.getServerData("xpath.xql", "document=%s&xpath=%s&pn=%s"%(docinfo['textURLPath'],xpath,pn)) + dom = Parse(text) + result = dom.xpath("//result/resultPage/place") + for l in result: + hrefNode= l.getAttributeNodeNS(None, u"id") + href= hrefNode.nodeValue + hrefList.append(href) + myList = ",".join(hrefList) + logging.debug("getGisPlaces :%s"%(myList)) + return myList + + def getAllGisPlaces (self, docinfo=None, pageinfo=None): + """Show all Gis Places of whole Book """ + xpath ='//echo:place' + docpath =docinfo['textURLPath'] + url = docinfo['url'] + selfurl =self.absolute_url() + pn =pageinfo['current'] + hrefList=[] + myList="" + text=self.getServerData("xpath.xql", "document=%s&xpath=%s"%(docinfo['textURLPath'],xpath)) + dom =Parse(text) + result = dom.xpath("//result/resultPage/place") + + for l in result: + hrefNode = l.getAttributeNodeNS(None, u"id") + href= hrefNode.nodeValue + hrefList.append(href) + myList = ",".join(hrefList) + logging.debug("getALLGisPlaces :%s"%(myList)) + return myList + + def getOrigPages (self, docinfo=None, pageinfo=None): + """Show original page """ + docpath = docinfo.get('textURLPath',None) + if not docpath: + return None + selfurl = self.absolute_url() + pn =pageinfo['current'] - def getTextPage(self, mode="text", pn=1, docinfo=None, pageinfo=None, highlightQuery=None,sn=None, viewMode=None, tocMode=None, tocPN=None): + viewMode= pageinfo['viewMode'] + text = self.getServerData("page-fragment.xql","document=%s&mode=%s&pn=%s"%(docinfo['textURLPath'], 'text', pn)) + dom =Parse(text) + pagedivs = dom.xpath("//div[@class='pageNumberOrig']") + if len(pagedivs)>0: + originalPage= getTextFromNode(pagedivs[0]) + #return docinfo['originalPage'] + return originalPage + + def getAllPlaces (self, docinfo=None): + """Show all Places if no places than 0""" + self.getInfoFromPage(docinfo) + return docinfo['allPlaces'] + + def getInfoFromPage(self, docinfo=None): + """ extract diverse info from page-fragment""" + docpath = docinfo['textURLPath'] + if 'allPlaces' in docinfo: + # allredy there + return docinfo + + if (docpath is not None): + text = self.getServerData("page-fragment.xql","document=%s"%(docinfo['textURLPath'])) + dom = Parse(text) + # figureEntries + pagedivs = dom.xpath("//div[@class='countFigureEntries']") + s = getTextFromNode(pagedivs[0]) + try: + docinfo['figureEntries'] = int(s) + except: + docinfo['figureEntries'] = 0 + # tocEntries + pagedivs = dom.xpath("//div[@class='countTocEntries']") + s = getTextFromNode(pagedivs[0]) + try: + docinfo['tocEntries'] = int(s) + except: + docinfo['tocEntries'] = 0 + # allPlaces + pagedivs = dom.xpath("//div[@class='countPlaces']") + s = getTextFromNode(pagedivs[0]) + try: + docinfo['allPlaces'] = int(s) + except: + docinfo['allPlaces'] = 0 + + else: + # no full text -- init to 0 + docinfo['figureEntries'] = 0 + docinfo['tocEntries'] = 0 + docinfo['allPlaces'] = 0 + + return docinfo + + + def getTextPage(self, mode="text", pn=1, docinfo=None, pageinfo=None, viewMode=None, tocMode=None, tocPN=None, characterNormalization="reg", highlightQuery=None, sn=None, optionToggle=None): """returns single page from fulltext""" docpath = docinfo['textURLPath'] path = docinfo['textURLPath'] url = docinfo['url'] + name = docinfo['name'] viewMode= pageinfo['viewMode'] + sn = pageinfo['sn'] + highlightQuery = pageinfo['highlightQuery'] + tocMode = pageinfo['tocMode'] + characterNormalization=pageinfo['characterNormalization'] tocPN = pageinfo['tocPN'] selfurl = self.absolute_url() if mode == "text_dict": textmode = "textPollux" else: textmode = mode - - textParam = "document=%s&mode=%s&pn=%s"%(docpath,textmode,pn) + #logging.debug("documentViewer highlightQuery: %s"%(highlightQuery)) + textParam = "document=%s&mode=%s&pn=%s&characterNormalization=%s"%(docpath,textmode,pn,characterNormalization) if highlightQuery is not None: - textParam +="&highlightQuery=%s&sn=%s"%(highlightQuery,sn) - + textParam +="&highlightQuery=%s&sn=%s"%(urllib.quote(highlightQuery),sn) + #logging.debug("documentViewer highlightQuery: %s"%(highlightQuery)) pagexml = self.getServerData("page-fragment.xql",textParam) + logging.debug("documentViewer highlightQuery: %s"%(highlightQuery)) #pagexml=self.template.fulltextclient.eval("/mpdl/interface/page-fragment.xql", textParam, outputUnicode=False) pagedom = Parse(pagexml) @@ -175,6 +301,21 @@ class MpdlXmlTextServer(SimpleItem): if len(pagedivs) > 0: pagenode = pagedivs[0] return serializeNode(pagenode) + if mode == "gis": + # first div contains text + pagedivs = pagedom.xpath("/div") + if len(pagedivs) > 0: + pagenode = pagedivs[0] + links =pagenode.xpath("//a") + for l in links: + hrefNode =l.getAttributeNodeNS(None, u"href") + if hrefNode: + href=hrefNode.nodeValue + if href.startswith('http://chinagis.mpiwg-berlin.mpg.de'): + hrefNode.nodeValue =href.replace('chinagis_REST/REST/db/chgis/mpdl','chinagis/REST/db/mpdl/%s'%name) + l.setAttributeNS(None, 'target', '_blank') + return serializeNode(pagenode) + if mode == "pureXml": # first div contains text pagedivs = pagedom.xpath("/div") @@ -202,13 +343,13 @@ class MpdlXmlTextServer(SimpleItem): # add target l.setAttributeNS(None, 'target', '_blank') l.setAttributeNS(None, 'onClick',"popupWin = window.open(this.href, 'contacts', 'location,width=500,height=600,top=180, left=700, scrollbars=1'); return false;") - l.setAttributeNS(None, 'onDblclick', 'popupWin.focus();') + l.setAttributeNS(None, 'onClick', 'popupWin.focus();') if href.startswith('lt/lemma.xql'): selfurl = self.absolute_url() hrefNode.nodeValue = href.replace('lt/lemma.xql','%s/template/head_main_lemma'%selfurl) l.setAttributeNS(None, 'target', '_blank') l.setAttributeNS(None, 'onClick',"popupWin = window.open(this.href, 'contacts', 'location,width=500,height=600,top=180, left=700, scrollbars=1'); return false;") - l.setAttributeNS(None, 'onDblclick', 'popupWin.focus();') + l.setAttributeNS(None, 'onClick', 'popupWin.focus();') if href.startswith('#note-'): hrefNode.nodeValue = href.replace('#note-',"?url=%s&viewMode=%s&tocMode=%s&tocPN=%s&pn=%s#note-"%(url,viewMode,tocMode,tocPN,pn)) return serializeNode(pagenode) @@ -216,23 +357,23 @@ class MpdlXmlTextServer(SimpleItem): def getTranslate(self, query=None, language=None): """translate into another languages""" - data = self.getServerData("lt/lex.xql","document=&language="+str(language)+"&query="+url_quote(str(query))) + data = self.getServerData("lt/lex.xql","document=&language="+str(language)+"&query="+urllib.quote(query)) #pagexml=self.template.fulltextclient.eval("/mpdl/interface/lt/lex.xql","document=&language="+str(language)+"&query="+url_quote(str(query))) return data def getLemma(self, lemma=None, language=None): """simular words lemma """ - data = self.getServerData("lt/lemma.xql","document=&language="+str(language)+"&lemma="+url_quote(str(lemma))) + data = self.getServerData("lt/lemma.xql","document=&language="+str(language)+"&lemma="+urllib.quote(lemma)) #pagexml=self.template.fulltextclient.eval("/mpdl/interface/lt/lemma.xql","document=&language="+str(language)+"&lemma="+url_quote(str(lemma))) return data def getLemmaNew(self, query=None, language=None): """simular words lemma """ - data = self.getServerData("lt/lemma.xql","document=&language="+str(language)+"&lemma="+url_quote(str(query))) + data = self.getServerData("lt/lemma.xql","document=&language="+str(language)+"&lemma="+urllib.quote(query)) #pagexml=self.template.fulltextclient.eval("/mpdl/interface/lt/lemma.xql","document=&language="+str(language)+"&lemma="+url_quote(str(query))) return data - - def getQuery (self, docinfo=None, pageinfo=None, query=None, queryType=None, pn=1): + + def getQuery (self, docinfo=None, pageinfo=None, query=None, queryType=None, pn=1, optionToggle=None): """number of""" docpath = docinfo['textURLPath'] pagesize = pageinfo['queryPageSize'] @@ -242,15 +383,67 @@ class MpdlXmlTextServer(SimpleItem): tocSearch = 0 tocDiv = None - pagexml = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath, 'text', queryType, query, pagesize, pn)) + pagexml = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath, 'text', queryType, urllib.quote(query), pagesize, pn)) #pagexml=self.template.fulltextclient.eval("/mpdl/interface/doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath, 'text', queryType, query, pagesize, pn) ,outputUnicode=False) pagedom = Parse(pagexml) numdivs = pagedom.xpath("//div[@class='queryResultHits']") tocSearch = int(getTextFromNode(numdivs[0])) + logging.debug("documentViewer (gettoc) tocSearch: %s"%(tocSearch)) tc=int((tocSearch/10)+1) logging.debug("documentViewer (gettoc) tc: %s"%(tc)) return tc + def getQueryResultHits(self, docinfo=None, pageinfo=None, query=None, queryType=None, pn=1, optionsClose=None): + + """number of hits in Search mode""" + docpath = docinfo['textURLPath'] + pagesize = pageinfo['queryPageSize'] + pn = pageinfo['searchPN'] + query =pageinfo['query'] + queryType =pageinfo['queryType'] + tocSearch = 0 + tocDiv = None + + pagexml = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath, 'text', queryType, urllib.quote(query), pagesize, pn)) + #pagexml=self.template.fulltextclient.eval("/mpdl/interface/doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath, 'text', queryType, query, pagesize, pn) ,outputUnicode=False) + pagedom = Parse(pagexml) + numdivs = pagedom.xpath("//div[@class='queryResultHits']") + tocSearch = int(getTextFromNode(numdivs[0])) + tc=int((tocSearch/10)+1) + return tc + + def getQueryResultHitsText(self, docinfo=None, pageinfo=None): + """number of hits in Text of Contents mode""" + selfurl = self.absolute_url() + docpath = docinfo['textURLPath'] + viewMode= pageinfo['viewMode'] + text = self.getServerData("page-fragment.xql","document=%s&mode=%s"%(docinfo['textURLPath'], 'text')) + dom =Parse(text) + pagedivs = dom.xpath("//div[@class='countTocEntries']") + logging.debug ("pagedivs=%s"%(pagedivs)) + if len(pagedivs)>0: + originalPage= (getTextFromNode(pagedivs[0])) + tc = int (originalPage) + tc1 =tc/30+1 + return tc1 + + def getQueryResultHitsFigures(self, docinfo=None, pageinfo=None): + """number of hits in Text of Figures mode""" + + selfurl = self.absolute_url() + docpath = docinfo['textURLPath'] + viewMode= pageinfo['viewMode'] + text = self.getServerData("page-fragment.xql","document=%s&mode=%s"%(docinfo['textURLPath'], 'text')) + dom =Parse(text) + pagedivs = dom.xpath("//div[@class='countFigureEntries']") + logging.debug ("pagedivs=%s"%(pagedivs)) + if len(pagedivs)>0: + originalPage= (getTextFromNode(pagedivs[0])) + tc = int (originalPage) + tc1 =tc/30+1 + return tc1 + + def getToc(self, mode="text", docinfo=None): """loads table of contents and stores in docinfo""" logging.debug("documentViewer (gettoc) mode: %s"%(mode)) @@ -297,13 +490,16 @@ class MpdlXmlTextServer(SimpleItem): url = docinfo['url'] selfurl = self.absolute_url() viewMode= pageinfo['viewMode'] + characterNormalization = pageinfo ['characterNormalization'] + optionToggle =pageinfo ['optionToggle'] tocMode = pageinfo['tocMode'] tocPN = pageinfo['tocPN'] - data = self.getServerData("doc-query.xql","document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath,queryType, pagesize, pn)) - - page = data.replace('page-fragment.xql?document=%s'%str(path),'%s?url=%s&viewMode=%s&tocMode=%s&tocPN=%s'%(selfurl,url, viewMode, tocMode, tocPN)) + data = self.getServerData("doc-query.xql","document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s&characterNormalization=regPlusNorm&optionToggle=1"%(docpath,queryType, pagesize, pn)) + page = data.replace('page-fragment.xql?document=%s'%str(path),'%s?url=%s&viewMode=%s&tocMode=%s&tocPN=%s&optionToggle=1'%(selfurl,url, viewMode, tocMode, tocPN)) text = page.replace('mode=image','mode=texttool') + logging.debug("documentViewer (characterNormalization) characterNormalization: %s"%(characterNormalization)) + #logging.debug("documentViewer (characterNormalization) text: %s"%(text)) return text def manage_changeMpdlXmlTextServer(self,title="",serverUrl="http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/",timeout=40,RESPONSE=None): @@ -328,4 +524,4 @@ def manage_addMpdlXmlTextServer(self,id, RESPONSE.redirect('manage_main') - \ No newline at end of file +