--- documentViewer/MpdlXmlTextServer.py 2010/10/15 14:23:03 1.162 +++ documentViewer/MpdlXmlTextServer.py 2011/02/14 14:41:12 1.225 @@ -3,6 +3,7 @@ from OFS.SimpleItem import SimpleItem from Products.PageTemplates.PageTemplateFile import PageTemplateFile from Ft.Xml import EMPTY_NAMESPACE, Parse +import md5 import sys import logging import urllib @@ -38,12 +39,12 @@ class MpdlXmlTextServer(SimpleItem): url = self.serverUrl+method return documentViewer.getHttpData(url,data,timeout=self.timeout) - def getSearch(self, pn=1, pageinfo=None, docinfo=None, query=None, queryType=None, lemma=None): + def getSearch(self, pageinfo=None, docinfo=None, lemma=None): """get search list""" docpath = docinfo['textURLPath'] url = docinfo['url'] pagesize = pageinfo['queryPageSize'] - pn = pageinfo['searchPN'] + pn = pageinfo.get('searchPN',1) sn = pageinfo['sn'] highlightQuery = pageinfo['highlightQuery'] query =pageinfo['query'] @@ -51,14 +52,26 @@ class MpdlXmlTextServer(SimpleItem): viewMode= pageinfo['viewMode'] tocMode = pageinfo['tocMode'] characterNormalization = pageinfo['characterNormalization'] + optionToggle = pageinfo['optionToggle'] tocPN = pageinfo['tocPN'] selfurl = self.absolute_url() - - data = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&sn=%s&viewMode=%s&highlightQuery=%s"%(docpath, 'text', queryType, urllib.quote(query), pagesize, pn, sn, viewMode,urllib.quote(highlightQuery))) - #page=self.template.fulltextclient.eval("/mpdl/interface/doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&sn=%s&viewMode=%s&highlightQuery=%s"%(docpath, 'text', queryType, query, pagesize, pn, sn, viewMode,highlightQuery) ,outputUnicode=False) - + data = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&sn=%s&viewMode=%s&characterNormalization=%s&optionToggle=%s&highlightQuery=%s"%(docpath, 'text', queryType, urllib.quote(query), pagesize, pn, sn, viewMode,characterNormalization,optionToggle ,urllib.quote(highlightQuery))) pagexml = data.replace('?document=%s'%str(docpath),'?url=%s'%url) pagedom = Parse(pagexml) + + """ + pagedivs = pagedom.xpath("//div[@class='queryResultHits']") + if (pagedivs == pagedom.xpath("//div[@class='queryResultHits']")): + if len(pagedivs)>0: + docinfo['queryResultHits'] = int(getTextFromNode(pagedivs[0])) + s = getTextFromNode(pagedivs[0]) + s1 = int(s)/10+1 + try: + docinfo['queryResultHits'] = int(s1) + logging.debug("SEARCH ENTRIES: %s"%(s1)) + except: + docinfo['queryResultHits'] = 0 + """ if (queryType=="fulltext")or(queryType=="xpath")or(queryType=="xquery")or(queryType=="fulltextMorphLemma"): pagedivs = pagedom.xpath("//div[@class='queryResultPage']") if len(pagedivs)>0: @@ -70,8 +83,9 @@ class MpdlXmlTextServer(SimpleItem): href = hrefNode.nodeValue if href.startswith('page-fragment.xql'): selfurl = self.absolute_url() - pagexml=href.replace('mode=text','mode=texttool&viewMode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&tocMode=%s&searchPN=%s&tocPN=%s'%(viewMode,queryType,urllib.quote(query),pagesize,pn,tocMode,pn,tocPN)) + pagexml=href.replace('mode=text','mode=texttool&viewMode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&tocMode=%s&searchPN=%s&tocPN=%s&optionToggle=%s&characterNormalization=%s'%(viewMode,queryType,urllib.quote(query),pagesize,pn,tocMode,pn,tocPN,optionToggle,characterNormalization)) hrefNode.nodeValue = pagexml.replace('page-fragment.xql','%s'%selfurl) + logging.debug("PUREXML :%s"%(serializeNode(pagenode))) return serializeNode(pagenode) if (queryType=="fulltextMorph"): pagedivs = pagedom.xpath("//div[@class='queryResult']") @@ -84,7 +98,7 @@ class MpdlXmlTextServer(SimpleItem): href = hrefNode.nodeValue if href.startswith('page-fragment.xql'): selfurl = self.absolute_url() - pagexml=href.replace('mode=text','mode=texttool&viewMode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&tocMode=%s&searchPN=%s&tocPN=%s'%(viewMode,queryType,urllib.quote(query),pagesize,pn,tocMode,pn,tocPN)) + pagexml=href.replace('mode=text','mode=texttool&viewMode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&tocMode=%s&searchPN=%s&tocPN=%s&optionToggle=%s&characterNormalization=%s'%(viewMode,queryType,urllib.quote(query),pagesize,pn,tocMode,pn,tocPN,optionToggle,characterNormalization)) hrefNode.nodeValue = pagexml.replace('page-fragment.xql','%s'%selfurl) if href.startswith('../lt/lemma.xql'): hrefNode.nodeValue = href.replace('../lt/lemma.xql','%s/template/head_main_lemma_New'%(selfurl)) @@ -102,7 +116,7 @@ class MpdlXmlTextServer(SimpleItem): hrefNode = l.getAttributeNodeNS(None, u"href") if hrefNode: href = hrefNode.nodeValue - hrefNode.nodeValue=href.replace('mode=text','mode=texttool&viewMode=%s&tocMode=%s&tocPN=%s&pn=%s'%(viewMode,tocMode,tocPN,pn)) + hrefNode.nodeValue=href.replace('mode=text','mode=texttool&viewMode=%s&tocMode=%s&tocPN=%s&pn=%s&optionToggle=%s&characterNormalization=%s'%(viewMode,tocMode,tocPN,pn,optionToggle,characterNormalization)) if href.startswith('../lt/lex.xql'): hrefNode.nodeValue = href.replace('../lt/lex.xql','%s/template/head_main_voc'%selfurl) l.setAttributeNS(None, 'target', '_blank') @@ -116,32 +130,14 @@ class MpdlXmlTextServer(SimpleItem): return serializeNode(pagenode) return "no text here" - """def getNumPages(self, docinfo): - ""get list of pages from fulltext and put in docinfo"" - if 'numPages' in docinfo: - # already there - return docinfo - xquery = '//pb' - text = self.getServerData("xquery.xql","document=%s&xquery=%s"%(docinfo['textURLPath'],xquery)) - docinfo['numPages'] = text.count("0: - docinfo['originalPage'] = getTextFromNode(pagedivs[0]) - return docinfo - - def getTextPage(self, mode="text", pn=1, docinfo=None, pageinfo=None, highlightQuery=None,sn=None, viewMode=None, tocMode=None, tocPN=None, characterNormalization=""): + + + def getTextPage(self, mode="text", pn=1, docinfo=None, pageinfo=None): """returns single page from fulltext""" docpath = docinfo['textURLPath'] path = docinfo['textURLPath'] - url = docinfo['url'] - name = docinfo['name'] + url = docinfo.get('url',None) + name = docinfo.get('name',None) + pn =pageinfo['current'] viewMode= pageinfo['viewMode'] + sn = pageinfo['sn'] + highlightQuery = pageinfo['highlightQuery'] + #mode = pageinfo ['viewMode'] tocMode = pageinfo['tocMode'] characterNormalization=pageinfo['characterNormalization'] tocPN = pageinfo['tocPN'] @@ -208,19 +195,105 @@ class MpdlXmlTextServer(SimpleItem): textmode = "textPollux" else: textmode = mode - #logging.debug("documentViewer (characterNormalization) characterNormalization: %s"%(characterNormalization)) + textParam = "document=%s&mode=%s&pn=%s&characterNormalization=%s"%(docpath,textmode,pn,characterNormalization) if highlightQuery is not None: textParam +="&highlightQuery=%s&sn=%s"%(urllib.quote(highlightQuery),sn) pagexml = self.getServerData("page-fragment.xql",textParam) - #pagexml=self.template.fulltextclient.eval("/mpdl/interface/page-fragment.xql", textParam, outputUnicode=False) + dom = Parse(pagexml) + + #original Pages + pagedivs = dom.xpath("//div[@class='pageNumberOrig']") + if pagedivs == dom.xpath("//div[@class='pageNumberOrig']"): + if len(pagedivs)>0: + docinfo['pageNumberOrig']= getTextFromNode(pagedivs[0]) + logging.debug("ORIGINAL PAGES: %s"%(docinfo['pageNumberOrig'])) + + #figureEntries + pagedivs = dom.xpath("//div[@class='countFigureEntries']") + if pagedivs == dom.xpath("//div[@class='countFigureEntries']"): + if len(pagedivs)>0: + docinfo['countFigureEntries'] = getTextFromNode(pagedivs[0]) + s = getTextFromNode(pagedivs[0]) + if s=='0': + try: + docinfo['countFigureEntries'] = int(s) + logging.debug("FIGURE ENTRIES: %s"%(s)) + except: + docinfo['countFigureEntries'] = 0 + else: + s1 = int(s)/30+1 + try: + docinfo['countFigureEntries'] = int(s1) + logging.debug("FIGURE ENTRIES: %s"%(s1)) + except: + docinfo['countFigureEntries'] = 0 + + #allPlaces + pagedivs = dom.xpath("//div[@class='countPlaces']") + if pagedivs == dom.xpath("//div[@class='countPlaces']"): + if len(pagedivs)>0: + docinfo['countPlaces']= getTextFromNode(pagedivs[0]) + s = getTextFromNode(pagedivs[0]) + try: + docinfo['countPlaces'] = int(s) + logging.debug("PLACES HERE: %s"%(s)) + except: + docinfo['countPlaces'] = 0 + + #tocEntries + pagedivs = dom.xpath("//div[@class='countTocEntries']") + if pagedivs == dom.xpath("//div[@class='countTocEntries']"): + if len(pagedivs)>0: + docinfo['countTocEntries'] = int(getTextFromNode(pagedivs[0])) + s = getTextFromNode(pagedivs[0]) + if s=='0': + try: + docinfo['countTocEntries'] = int(s) + logging.debug("TEXT ENTRIES: %s"%(s)) + except: + docinfo['countTocEntries'] = 0 + else: + s1 = int(s)/30+1 + try: + docinfo['countTocEntries'] = int(s1) + logging.debug("TEXT ENTRIES: %s"%(s1)) + except: + docinfo['countTocEntries'] = 0 + + #numTextPages + pagedivs = dom.xpath("//div[@class='countPages']") + if pagedivs == dom.xpath("//div[@class='countPages']"): + if len(pagedivs)>0: + docinfo['numPages'] = getTextFromNode(pagedivs[0]) + s = getTextFromNode(pagedivs[0]) + + try: + docinfo['numPages'] = int(s) + logging.debug("PAGE NUMBER: %s"%(s)) + + np = docinfo['numPages'] + pageinfo['end'] = min(pageinfo['end'], np) + pageinfo['numgroups'] = int(np / pageinfo['groupsize']) + if np % pageinfo['groupsize'] > 0: + pageinfo['numgroups'] += 1 + except: + docinfo['numPages'] = 0 + + else: + #no full text -- init to 0 + docinfo['pageNumberOrig'] = 0 + docinfo['countFigureEntries'] = 0 + docinfo['countPlaces'] = 0 + docinfo['countTocEntries'] = 0 + docinfo['numPages'] = 0 + #return docinfo - pagedom = Parse(pagexml) # plain text mode if mode == "text": # first div contains text - pagedivs = pagedom.xpath("/div") + pagedivs = dom.xpath("/div") if len(pagedivs) > 0: pagenode = pagedivs[0] links = pagenode.xpath("//a") @@ -233,13 +306,13 @@ class MpdlXmlTextServer(SimpleItem): return serializeNode(pagenode) if mode == "xml": # first div contains text - pagedivs = pagedom.xpath("/div") + pagedivs = dom.xpath("/div") if len(pagedivs) > 0: pagenode = pagedivs[0] return serializeNode(pagenode) if mode == "gis": # first div contains text - pagedivs = pagedom.xpath("/div") + pagedivs = dom.xpath("/div") if len(pagedivs) > 0: pagenode = pagedivs[0] links =pagenode.xpath("//a") @@ -254,14 +327,14 @@ class MpdlXmlTextServer(SimpleItem): if mode == "pureXml": # first div contains text - pagedivs = pagedom.xpath("/div") + pagedivs = dom.xpath("/div") if len(pagedivs) > 0: pagenode = pagedivs[0] return serializeNode(pagenode) # text-with-links mode if mode == "text_dict": # first div contains text - pagedivs = pagedom.xpath("/div") + pagedivs = dom.xpath("/div") if len(pagedivs) > 0: pagenode = pagedivs[0] # check all a-tags @@ -290,7 +363,22 @@ class MpdlXmlTextServer(SimpleItem): hrefNode.nodeValue = href.replace('#note-',"?url=%s&viewMode=%s&tocMode=%s&tocPN=%s&pn=%s#note-"%(url,viewMode,tocMode,tocPN,pn)) return serializeNode(pagenode) return "no text here" - + + def getText(self, mode="text", pn=1, docinfo=None, pageinfo=None): + data = self.getTextPage(mode="text", pn=1 ,docinfo=docinfo, pageinfo=pageinfo) + logging.debug("TEXT ALL data: %s"%(data)) + return data + + def getXML(self, mode="xml", pn=1, docinfo=None, pageinfo=None): + data = self.getTextPage(mode="xml", pn=1 ,docinfo=docinfo, pageinfo=pageinfo) + logging.debug("TEXT ALL data: %s"%(data)) + return data + + def getDictionary(self, mode="text_dict", pn=1, docinfo=None, pageinfo=None): + data = self.getTextPage(mode="text_dict", pn=1 ,docinfo=docinfo, pageinfo=pageinfo) + logging.debug("TEXT ALL data: %s"%(data)) + return data + def getTranslate(self, query=None, language=None): """translate into another languages""" data = self.getServerData("lt/lex.xql","document=&language="+str(language)+"&query="+urllib.quote(query)) @@ -309,8 +397,8 @@ class MpdlXmlTextServer(SimpleItem): #pagexml=self.template.fulltextclient.eval("/mpdl/interface/lt/lemma.xql","document=&language="+str(language)+"&lemma="+url_quote(str(query))) return data - def getQuery (self, docinfo=None, pageinfo=None, query=None, queryType=None, pn=1): - """number of""" + def getQuery (self, docinfo=None, pageinfo=None, query=None, queryType=None, pn=1, optionToggle=None): + #number of docpath = docinfo['textURLPath'] pagesize = pageinfo['queryPageSize'] pn = pageinfo['searchPN'] @@ -320,14 +408,14 @@ class MpdlXmlTextServer(SimpleItem): tocDiv = None pagexml = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath, 'text', queryType, urllib.quote(query), pagesize, pn)) - #pagexml=self.template.fulltextclient.eval("/mpdl/interface/doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath, 'text', queryType, query, pagesize, pn) ,outputUnicode=False) pagedom = Parse(pagexml) numdivs = pagedom.xpath("//div[@class='queryResultHits']") tocSearch = int(getTextFromNode(numdivs[0])) + logging.debug("documentViewer (gettoc) tocSearch: %s"%(tocSearch)) tc=int((tocSearch/10)+1) logging.debug("documentViewer (gettoc) tc: %s"%(tc)) return tc - + def getToc(self, mode="text", docinfo=None): """loads table of contents and stores in docinfo""" logging.debug("documentViewer (gettoc) mode: %s"%(mode)) @@ -350,7 +438,7 @@ class MpdlXmlTextServer(SimpleItem): tocDiv = None pagexml = self.getServerData("doc-query.xql","document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath,queryType, pagesize, pn)) - #pagexml=self.template.fulltextclient.eval("/mpdl/interface/doc-query.xql", "document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath,queryType,pagesize,pn), outputUnicode=False) + # post-processing downloaded xml pagedom = Parse(pagexml) # get number of entries @@ -375,14 +463,15 @@ class MpdlXmlTextServer(SimpleItem): selfurl = self.absolute_url() viewMode= pageinfo['viewMode'] characterNormalization = pageinfo ['characterNormalization'] + optionToggle =pageinfo ['optionToggle'] tocMode = pageinfo['tocMode'] tocPN = pageinfo['tocPN'] - data = self.getServerData("doc-query.xql","document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s&characterNormalization=%s"%(docpath,queryType, pagesize, pn,characterNormalization)) - page = data.replace('page-fragment.xql?document=%s'%str(path),'%s?url=%s&viewMode=%s&tocMode=%s&tocPN=%s'%(selfurl,url, viewMode, tocMode, tocPN)) + data = self.getServerData("doc-query.xql","document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s&characterNormalization=regPlusNorm&optionToggle=1"%(docpath,queryType, pagesize, pn)) + page = data.replace('page-fragment.xql?document=%s'%str(path),'%s?url=%s&viewMode=%s&tocMode=%s&tocPN=%s&optionToggle=1'%(selfurl,url, viewMode, tocMode, tocPN)) text = page.replace('mode=image','mode=texttool') - logging.debug("documentViewer (characterNormalization) characterNormalization: %s"%(characterNormalization)) - #logging.debug("documentViewer (characterNormalization) text: %s"%(text)) + #logging.debug("documentViewer (characterNormalization) characterNormalization: %s"%(characterNormalization)) + logging.debug("TEXT %s"%(text)) return text def manage_changeMpdlXmlTextServer(self,title="",serverUrl="http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/",timeout=40,RESPONSE=None): @@ -404,7 +493,4 @@ def manage_addMpdlXmlTextServer(self,id, newObj = MpdlXmlTextServer(id,title,serverUrl,timeout) self.Destination()._setObject(id, newObj) if RESPONSE is not None: - RESPONSE.redirect('manage_main') - - - + RESPONSE.redirect('manage_main') \ No newline at end of file