--- documentViewer/MpdlXmlTextServer.py 2010/10/11 13:58:31 1.72 +++ documentViewer/MpdlXmlTextServer.py 2011/03/07 14:08:07 1.230 @@ -3,6 +3,7 @@ from OFS.SimpleItem import SimpleItem from Products.PageTemplates.PageTemplateFile import PageTemplateFile from Ft.Xml import EMPTY_NAMESPACE, Parse +import md5 import sys import logging import urllib @@ -38,14 +39,12 @@ class MpdlXmlTextServer(SimpleItem): url = self.serverUrl+method return documentViewer.getHttpData(url,data,timeout=self.timeout) - def getSearch(self, pn=1, pageinfo=None, docinfo=None, query=None, queryType=None, lemma=None): + def getSearch(self, pageinfo=None, docinfo=None, lemma=None): """get search list""" docpath = docinfo['textURLPath'] url = docinfo['url'] - logging.debug("documentViewer (gettoc) docpath: %s"%(docpath)) - logging.debug("documentViewer (gettoc) url: %s"%(url)) pagesize = pageinfo['queryPageSize'] - pn = pageinfo['searchPN'] + pn = pageinfo.get('searchPN',1) sn = pageinfo['sn'] highlightQuery = pageinfo['highlightQuery'] query =pageinfo['query'] @@ -53,14 +52,26 @@ class MpdlXmlTextServer(SimpleItem): viewMode= pageinfo['viewMode'] tocMode = pageinfo['tocMode'] characterNormalization = pageinfo['characterNormalization'] + optionToggle = pageinfo['optionToggle'] tocPN = pageinfo['tocPN'] selfurl = self.absolute_url() - - data = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&sn=%s&viewMode=%s&highlightQuery=%s"%(docpath, 'text', queryType, urllib.quote(query), pagesize, pn, sn, viewMode,urllib.quote(highlightQuery))) - #page=self.template.fulltextclient.eval("/mpdl/interface/doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&sn=%s&viewMode=%s&highlightQuery=%s"%(docpath, 'text', queryType, query, pagesize, pn, sn, viewMode,highlightQuery) ,outputUnicode=False) - + data = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&sn=%s&viewMode=%s&characterNormalization=%s&optionToggle=%s&highlightQuery=%s"%(docpath, 'text', queryType, urllib.quote(query), pagesize, pn, sn, viewMode,characterNormalization,optionToggle ,urllib.quote(highlightQuery))) pagexml = data.replace('?document=%s'%str(docpath),'?url=%s'%url) pagedom = Parse(pagexml) + + """ + pagedivs = pagedom.xpath("//div[@class='queryResultHits']") + if (pagedivs == pagedom.xpath("//div[@class='queryResultHits']")): + if len(pagedivs)>0: + docinfo['queryResultHits'] = int(getTextFromNode(pagedivs[0])) + s = getTextFromNode(pagedivs[0]) + s1 = int(s)/10+1 + try: + docinfo['queryResultHits'] = int(s1) + logging.debug("SEARCH ENTRIES: %s"%(s1)) + except: + docinfo['queryResultHits'] = 0 + """ if (queryType=="fulltext")or(queryType=="xpath")or(queryType=="xquery")or(queryType=="fulltextMorphLemma"): pagedivs = pagedom.xpath("//div[@class='queryResultPage']") if len(pagedivs)>0: @@ -72,8 +83,9 @@ class MpdlXmlTextServer(SimpleItem): href = hrefNode.nodeValue if href.startswith('page-fragment.xql'): selfurl = self.absolute_url() - pagexml=href.replace('mode=text','mode=texttool&viewMode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&tocMode=%s&searchPN=%s&tocPN=%s'%(viewMode,queryType,urllib.quote(query),pagesize,pn,tocMode,pn,tocPN)) + pagexml=href.replace('mode=text','mode=texttool&viewMode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&tocMode=%s&searchPN=%s&tocPN=%s&optionToggle=%s&characterNormalization=%s'%(viewMode,queryType,urllib.quote(query),pagesize,pn,tocMode,pn,tocPN,optionToggle,characterNormalization)) hrefNode.nodeValue = pagexml.replace('page-fragment.xql','%s'%selfurl) + #logging.debug("PUREXML :%s"%(serializeNode(pagenode))) return serializeNode(pagenode) if (queryType=="fulltextMorph"): pagedivs = pagedom.xpath("//div[@class='queryResult']") @@ -86,7 +98,7 @@ class MpdlXmlTextServer(SimpleItem): href = hrefNode.nodeValue if href.startswith('page-fragment.xql'): selfurl = self.absolute_url() - pagexml=href.replace('mode=text','mode=texttool&viewMode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&tocMode=%s&searchPN=%s&tocPN=%s'%(viewMode,queryType,urllib.quote(query),pagesize,pn,tocMode,pn,tocPN)) + pagexml=href.replace('mode=text','mode=texttool&viewMode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&tocMode=%s&searchPN=%s&tocPN=%s&optionToggle=%s&characterNormalization=%s'%(viewMode,queryType,urllib.quote(query),pagesize,pn,tocMode,pn,tocPN,optionToggle,characterNormalization)) hrefNode.nodeValue = pagexml.replace('page-fragment.xql','%s'%selfurl) if href.startswith('../lt/lemma.xql'): hrefNode.nodeValue = href.replace('../lt/lemma.xql','%s/template/head_main_lemma_New'%(selfurl)) @@ -104,7 +116,7 @@ class MpdlXmlTextServer(SimpleItem): hrefNode = l.getAttributeNodeNS(None, u"href") if hrefNode: href = hrefNode.nodeValue - hrefNode.nodeValue=href.replace('mode=text','mode=texttool&viewMode=%s&tocMode=%s&tocPN=%s&pn=%s'%(viewMode,tocMode,tocPN,pn)) + hrefNode.nodeValue=href.replace('mode=text','mode=texttool&viewMode=%s&tocMode=%s&tocPN=%s&pn=%s&optionToggle=%s&characterNormalization=%s'%(viewMode,tocMode,tocPN,pn,optionToggle,characterNormalization)) if href.startswith('../lt/lex.xql'): hrefNode.nodeValue = href.replace('../lt/lex.xql','%s/template/head_main_voc'%selfurl) l.setAttributeNS(None, 'target', '_blank') @@ -117,47 +129,62 @@ class MpdlXmlTextServer(SimpleItem): l.setAttributeNS(None, 'onClick', 'popupWin.focus();') return serializeNode(pagenode) return "no text here" - - """def getNumPages(self, docinfo): - ""get list of pages from fulltext and put in docinfo"" - if 'numPages' in docinfo: - # already there - return docinfo - xquery = '//pb' - text = self.getServerData("xquery.xql","document=%s&xquery=%s"%(docinfo['textURLPath'],xquery)) - docinfo['numPages'] = text.count("0: + docinfo['pageNumberOrig']= getTextFromNode(pagedivs[0]) + logging.debug("ORIGINAL PAGE: %s"%(docinfo['pageNumberOrig'])) + + #original Pages Norm + pagedivs = dom.xpath("//div[@class='pageNumberOrigNorm']") + if pagedivs == dom.xpath("//div[@class='pageNumberOrigNorm']"): + if len(pagedivs)>0: + docinfo['pageNumberOrigNorm']= getTextFromNode(pagedivs[0]) + logging.debug("ORIGINAL PAGE NORM: %s"%(docinfo['pageNumberOrigNorm'])) + """ + #figureEntries + pagedivs = dom.xpath("//div[@class='countFigureEntries']") + if pagedivs == dom.xpath("//div[@class='countFigureEntries']"): + if len(pagedivs)>0: + docinfo['countFigureEntries'] = getTextFromNode(pagedivs[0]) + s = getTextFromNode(pagedivs[0]) + if s=='0': + try: + docinfo['countFigureEntries'] = int(s) + except: + docinfo['countFigureEntries'] = 0 + else: + s1 = int(s)/30+1 + try: + docinfo['countFigureEntries'] = int(s1) + except: + docinfo['countFigureEntries'] = 0 + + #allPlaces + pagedivs = dom.xpath("//div[@class='countPlaces']") + if pagedivs == dom.xpath("//div[@class='countPlaces']"): + if len(pagedivs)>0: + docinfo['countPlaces']= getTextFromNode(pagedivs[0]) + s = getTextFromNode(pagedivs[0]) + try: + docinfo['countPlaces'] = int(s) + except: + docinfo['countPlaces'] = 0 + + #tocEntries + pagedivs = dom.xpath("//div[@class='countTocEntries']") + if pagedivs == dom.xpath("//div[@class='countTocEntries']"): + if len(pagedivs)>0: + docinfo['countTocEntries'] = int(getTextFromNode(pagedivs[0])) + s = getTextFromNode(pagedivs[0]) + if s=='0': + try: + docinfo['countTocEntries'] = int(s) + except: + docinfo['countTocEntries'] = 0 + else: + s1 = int(s)/30+1 + try: + docinfo['countTocEntries'] = int(s1) + except: + docinfo['countTocEntries'] = 0 + + #numTextPages + pagedivs = dom.xpath("//div[@class='countPages']") + if pagedivs == dom.xpath("//div[@class='countPages']"): + if len(pagedivs)>0: + docinfo['numPages'] = getTextFromNode(pagedivs[0]) + s = getTextFromNode(pagedivs[0]) + + try: + docinfo['numPages'] = int(s) + #logging.debug("PAGE NUMBER: %s"%(s)) + + np = docinfo['numPages'] + pageinfo['end'] = min(pageinfo['end'], np) + pageinfo['numgroups'] = int(np / pageinfo['groupsize']) + if np % pageinfo['groupsize'] > 0: + pageinfo['numgroups'] += 1 + except: + docinfo['numPages'] = 0 + + else: + #no full text -- init to 0 + docinfo['pageNumberOrig'] = 0 + docinfo['countFigureEntries'] = 0 + docinfo['countPlaces'] = 0 + docinfo['countTocEntries'] = 0 + docinfo['numPages'] = 0 + docinfo['pageNumberOrigNorm'] = 0 + #return docinfo - pagedom = Parse(pagexml) # plain text mode if mode == "text": # first div contains text - pagedivs = pagedom.xpath("/div") + pagedivs = dom.xpath("/div") if len(pagedivs) > 0: pagenode = pagedivs[0] links = pagenode.xpath("//a") @@ -187,17 +302,17 @@ class MpdlXmlTextServer(SimpleItem): if hrefNode: href= hrefNode.nodeValue if href.startswith('#note-'): - hrefNode.nodeValue = href.replace('#note-',"?url=%s&viewMode=%s&tocMode=%s&tocPN=%s&pn=%s#note-"%(url,viewMode,tocMode,tocPN,pn)) + hrefNode.nodeValue = href.replace('#note-',"?url=%s&viewMode=text&tocMode=%s&tocPN=%s&pn=%s#note-"%(url,tocMode,tocPN,pn)) return serializeNode(pagenode) if mode == "xml": # first div contains text - pagedivs = pagedom.xpath("/div") + pagedivs = dom.xpath("/div") if len(pagedivs) > 0: pagenode = pagedivs[0] return serializeNode(pagenode) if mode == "gis": # first div contains text - pagedivs = pagedom.xpath("/div") + pagedivs = dom.xpath("/div") if len(pagedivs) > 0: pagenode = pagedivs[0] links =pagenode.xpath("//a") @@ -206,21 +321,21 @@ class MpdlXmlTextServer(SimpleItem): if hrefNode: href=hrefNode.nodeValue if href.startswith('http://chinagis.mpiwg-berlin.mpg.de'): - hrefNode.nodeValue =href.replace('chinagis_REST/REST/db/chgis/mpdl','chinagis/REST/db/mpdl/'%nameR) - #hrefNode.nodeValue =href.replace('chinagis_REST/REST/db/chgis/mpdl','chinagis/REST/db/mpdl%s'%url) + hrefNode.nodeValue =href.replace('chinagis_REST/REST/db/chgis/mpdl','chinagis/REST/db/mpdl/%s'%name) l.setAttributeNS(None, 'target', '_blank') return serializeNode(pagenode) if mode == "pureXml": # first div contains text - pagedivs = pagedom.xpath("/div") + pagedivs = dom.xpath("/div") if len(pagedivs) > 0: pagenode = pagedivs[0] return serializeNode(pagenode) # text-with-links mode if mode == "text_dict": # first div contains text - pagedivs = pagedom.xpath("/div") + #mode = pageinfo ['viewMode'] + pagedivs = dom.xpath("/div") if len(pagedivs) > 0: pagenode = pagedivs[0] # check all a-tags @@ -246,10 +361,35 @@ class MpdlXmlTextServer(SimpleItem): l.setAttributeNS(None, 'onClick',"popupWin = window.open(this.href, 'contacts', 'location,width=500,height=600,top=180, left=700, scrollbars=1'); return false;") l.setAttributeNS(None, 'onClick', 'popupWin.focus();') if href.startswith('#note-'): - hrefNode.nodeValue = href.replace('#note-',"?url=%s&viewMode=%s&tocMode=%s&tocPN=%s&pn=%s#note-"%(url,viewMode,tocMode,tocPN,pn)) + hrefNode.nodeValue = href.replace('#note-',"?url=%s&viewMode=text_dict&tocMode=%s&tocPN=%s&pn=%s#note-"%(url,tocMode,tocPN,pn)) return serializeNode(pagenode) return "no text here" + + def getOrigPages(self, docinfo=None, pageinfo=None): + docpath = docinfo['textURLPath'] + pn =pageinfo['current'] + selfurl = self.absolute_url() + pagexml = self.getServerData("page-fragment.xql","document=%s&pn=%s"%(docpath, pn)) + dom = Parse(pagexml) + pagedivs = dom.xpath("//div[@class='pageNumberOrig']") + if pagedivs == dom.xpath("//div[@class='pageNumberOrig']"): + if len(pagedivs)>0: + docinfo['pageNumberOrig']= getTextFromNode(pagedivs[0]) + return docinfo['pageNumberOrig'] + + def getOrigPagesNorm(self, docinfo=None, pageinfo=None): + docpath = docinfo['textURLPath'] + pn =pageinfo['current'] + selfurl = self.absolute_url() + pagexml = self.getServerData("page-fragment.xql","document=%s&pn=%s"%(docpath, pn)) + dom = Parse(pagexml) + pagedivs = dom.xpath("//div[@class='pageNumberOrigNorm']") + if pagedivs == dom.xpath("//div[@class='pageNumberOrigNorm']"): + if len(pagedivs)>0: + docinfo['pageNumberOrigNorm']= getTextFromNode(pagedivs[0]) + return docinfo['pageNumberOrigNorm'] + def getTranslate(self, query=None, language=None): """translate into another languages""" data = self.getServerData("lt/lex.xql","document=&language="+str(language)+"&query="+urllib.quote(query)) @@ -268,8 +408,8 @@ class MpdlXmlTextServer(SimpleItem): #pagexml=self.template.fulltextclient.eval("/mpdl/interface/lt/lemma.xql","document=&language="+str(language)+"&lemma="+url_quote(str(query))) return data - def getQuery (self, docinfo=None, pageinfo=None, query=None, queryType=None, pn=1): - """number of""" + def getQuery (self, docinfo=None, pageinfo=None, query=None, queryType=None, pn=1, optionToggle=None): + #number of docpath = docinfo['textURLPath'] pagesize = pageinfo['queryPageSize'] pn = pageinfo['searchPN'] @@ -279,17 +419,14 @@ class MpdlXmlTextServer(SimpleItem): tocDiv = None pagexml = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath, 'text', queryType, urllib.quote(query), pagesize, pn)) - #pagexml=self.template.fulltextclient.eval("/mpdl/interface/doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath, 'text', queryType, query, pagesize, pn) ,outputUnicode=False) pagedom = Parse(pagexml) numdivs = pagedom.xpath("//div[@class='queryResultHits']") tocSearch = int(getTextFromNode(numdivs[0])) tc=int((tocSearch/10)+1) - logging.debug("documentViewer (gettoc) tc: %s"%(tc)) return tc - + def getToc(self, mode="text", docinfo=None): """loads table of contents and stores in docinfo""" - logging.debug("documentViewer (gettoc) mode: %s"%(mode)) if mode == "none": return docinfo if 'tocSize_%s'%mode in docinfo: @@ -309,7 +446,7 @@ class MpdlXmlTextServer(SimpleItem): tocDiv = None pagexml = self.getServerData("doc-query.xql","document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath,queryType, pagesize, pn)) - #pagexml=self.template.fulltextclient.eval("/mpdl/interface/doc-query.xql", "document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath,queryType,pagesize,pn), outputUnicode=False) + # post-processing downloaded xml pagedom = Parse(pagexml) # get number of entries @@ -334,14 +471,13 @@ class MpdlXmlTextServer(SimpleItem): selfurl = self.absolute_url() viewMode= pageinfo['viewMode'] characterNormalization = pageinfo ['characterNormalization'] + optionToggle =pageinfo ['optionToggle'] tocMode = pageinfo['tocMode'] tocPN = pageinfo['tocPN'] - data = self.getServerData("doc-query.xql","document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s&characterNormalization=%s"%(docpath,queryType, pagesize, pn,characterNormalization)) - page = data.replace('page-fragment.xql?document=%s'%str(path),'%s?url=%s&viewMode=%s&tocMode=%s&tocPN=%s'%(selfurl,url, viewMode, tocMode, tocPN)) + data = self.getServerData("doc-query.xql","document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s&characterNormalization=regPlusNorm&optionToggle=1"%(docpath,queryType, pagesize, pn)) + page = data.replace('page-fragment.xql?document=%s'%str(path),'%s?url=%s&viewMode=%s&tocMode=%s&tocPN=%s&optionToggle=1'%(selfurl,url, viewMode, tocMode, tocPN)) text = page.replace('mode=image','mode=texttool') - logging.debug("documentViewer (characterNormalization) characterNormalization: %s"%(characterNormalization)) - #logging.debug("documentViewer (characterNormalization) text: %s"%(text)) return text def manage_changeMpdlXmlTextServer(self,title="",serverUrl="http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/",timeout=40,RESPONSE=None): @@ -363,7 +499,4 @@ def manage_addMpdlXmlTextServer(self,id, newObj = MpdlXmlTextServer(id,title,serverUrl,timeout) self.Destination()._setObject(id, newObj) if RESPONSE is not None: - RESPONSE.redirect('manage_main') - - - + RESPONSE.redirect('manage_main') \ No newline at end of file