--- documentViewer/MpdlXmlTextServer.py 2010/12/23 17:13:01 1.221 +++ documentViewer/MpdlXmlTextServer.py 2011/01/06 15:44:18 1.222 @@ -38,12 +38,12 @@ class MpdlXmlTextServer(SimpleItem): url = self.serverUrl+method return documentViewer.getHttpData(url,data,timeout=self.timeout) - def getSearch(self, pn=1, pageinfo=None, docinfo=None, query=None, queryType=None, lemma=None, characterNormalization=None, optionToggle=None): + def getSearch(self, pageinfo=None, docinfo=None, lemma=None): """get search list""" docpath = docinfo['textURLPath'] url = docinfo['url'] pagesize = pageinfo['queryPageSize'] - pn = pageinfo['searchPN'] + pn = pageinfo.get('searchPN',1) sn = pageinfo['sn'] highlightQuery = pageinfo['highlightQuery'] query =pageinfo['query'] @@ -56,10 +56,23 @@ class MpdlXmlTextServer(SimpleItem): selfurl = self.absolute_url() data = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&sn=%s&viewMode=%s&characterNormalization=%s&optionToggle=%s&highlightQuery=%s"%(docpath, 'text', queryType, urllib.quote(query), pagesize, pn, sn, viewMode,characterNormalization,optionToggle ,urllib.quote(highlightQuery))) - #page=self.template.fulltextclient.eval("/mpdl/interface/doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&sn=%s&viewMode=%s&highlightQuery=%s"%(docpath, 'text', queryType, query, pagesize, pn, sn, viewMode,highlightQuery) ,outputUnicode=False) - + pagexml = data.replace('?document=%s'%str(docpath),'?url=%s'%url) pagedom = Parse(pagexml) + + """ + pagedivs = pagedom.xpath("//div[@class='queryResultHits']") + if (pagedivs == pagedom.xpath("//div[@class='queryResultHits']")): + if len(pagedivs)>0: + docinfo['queryResultHits'] = int(getTextFromNode(pagedivs[0])) + s = getTextFromNode(pagedivs[0]) + s1 = int(s)/10+1 + try: + docinfo['queryResultHits'] = int(s1) + logging.debug("SEARCH ENTRIES: %s"%(s1)) + except: + docinfo['queryResultHits'] = 0 + """ if (queryType=="fulltext")or(queryType=="xpath")or(queryType=="xquery")or(queryType=="fulltextMorphLemma"): pagedivs = pagedom.xpath("//div[@class='queryResultPage']") if len(pagedivs)>0: @@ -117,36 +130,7 @@ class MpdlXmlTextServer(SimpleItem): return serializeNode(pagenode) return "no text here" - - def getNumTextPages (self, docinfo=None): - """get list of pages from fulltext (texts without images) and put in docinfo""" - logging.debug("getNumTextPages") - docpath = docinfo['textURLPath'] - selfurl = self.absolute_url() - #viewMode= pageinfo['viewMode'] - if 'numPages' in docinfo: - # allredy there - return docinfo - - text = self.getServerData("page-fragment.xql","document=%s"%(docinfo['textURLPath'])) - dom =Parse(text) - pagedivs = dom.xpath("//div[@class='countPages']") - logging.debug ("pagedivs=%s"%(pagedivs)) - if len(pagedivs)>0: - docinfo['numPages']= int(getTextFromNode(pagedivs[0])) - return docinfo - - def getTocEntries (self, docinfo=None): - """ number of text entries""" - docinfo = self.getInfoFromPage(docinfo) - logging.debug("gettocentries: docinfo=%s"%docinfo) - return docinfo['tocEntries'] - - def getFigureEntries (self, docinfo=None): - """ number of figure entries""" - docinfo = self.getInfoFromPage(docinfo) - return docinfo['figureEntries'] - + def getGisPlaces(self, docinfo=None, pageinfo=None): """ Show all Gis Places of whole Page""" xpath='//place' @@ -190,80 +174,19 @@ class MpdlXmlTextServer(SimpleItem): myList = ",".join(hrefList) logging.debug("getALLGisPlaces :%s"%(myList)) return myList - - def getOrigPages (self, docinfo=None, pageinfo=None): - """Show original page """ - docpath = docinfo.get('textURLPath',None) - if not docpath: - return None - selfurl = self.absolute_url() - pn =pageinfo['current'] - - viewMode= pageinfo['viewMode'] - text = self.getServerData("page-fragment.xql","document=%s&mode=%s&pn=%s"%(docinfo['textURLPath'], 'text', pn)) - dom =Parse(text) - pagedivs = dom.xpath("//div[@class='pageNumberOrig']") - if len(pagedivs)>0: - originalPage= getTextFromNode(pagedivs[0]) - #return docinfo['originalPage'] - return originalPage - - def getAllPlaces (self, docinfo=None): - """Show all Places if no places than 0""" - docinfo = self.getInfoFromPage(docinfo) - return docinfo['allPlaces'] - - def getInfoFromPage(self, docinfo=None): - """ extract diverse info from page-fragment""" - docpath = docinfo['textURLPath'] - if 'allPlaces' in docinfo: - # allredy there - return docinfo - - if (docpath is not None): - text = self.getServerData("page-fragment.xql","document=%s"%(docinfo['textURLPath'])) - dom = Parse(text) - # figureEntries - pagedivs = dom.xpath("//div[@class='countFigureEntries']") - s = getTextFromNode(pagedivs[0]) - try: - docinfo['figureEntries'] = int(s) - except: - docinfo['figureEntries'] = 0 - # tocEntries - pagedivs = dom.xpath("//div[@class='countTocEntries']") - s = getTextFromNode(pagedivs[0]) - try: - docinfo['tocEntries'] = int(s) - except: - docinfo['tocEntries'] = 0 - # allPlaces - pagedivs = dom.xpath("//div[@class='countPlaces']") - s = getTextFromNode(pagedivs[0]) - try: - docinfo['allPlaces'] = int(s) - except: - docinfo['allPlaces'] = 0 - - else: - # no full text -- init to 0 - docinfo['figureEntries'] = 0 - docinfo['tocEntries'] = 0 - docinfo['allPlaces'] = 0 - - return docinfo - + - def getTextPage(self, mode="text", pn=1, docinfo=None, pageinfo=None, viewMode=None, tocMode=None, tocPN=None, characterNormalization="reg", highlightQuery=None, sn=None, optionToggle=None): + def getTextPage(self, docinfo=None, pageinfo=None): """returns single page from fulltext""" docpath = docinfo['textURLPath'] path = docinfo['textURLPath'] - url = docinfo['url'] - name = docinfo['name'] - viewMode= pageinfo['viewMode'] + url = docinfo.get('url',None) + name = docinfo.get('name',None) + pn =pageinfo['current'] + #viewMode= pageinfo['viewMode'] sn = pageinfo['sn'] highlightQuery = pageinfo['highlightQuery'] - + mode = pageinfo ['viewMode'] tocMode = pageinfo['tocMode'] characterNormalization=pageinfo['characterNormalization'] tocPN = pageinfo['tocPN'] @@ -272,20 +195,105 @@ class MpdlXmlTextServer(SimpleItem): textmode = "textPollux" else: textmode = mode - #logging.debug("documentViewer highlightQuery: %s"%(highlightQuery)) + textParam = "document=%s&mode=%s&pn=%s&characterNormalization=%s"%(docpath,textmode,pn,characterNormalization) if highlightQuery is not None: textParam +="&highlightQuery=%s&sn=%s"%(urllib.quote(highlightQuery),sn) - #logging.debug("documentViewer highlightQuery: %s"%(highlightQuery)) + pagexml = self.getServerData("page-fragment.xql",textParam) - logging.debug("documentViewer highlightQuery: %s"%(highlightQuery)) - #pagexml=self.template.fulltextclient.eval("/mpdl/interface/page-fragment.xql", textParam, outputUnicode=False) + dom = Parse(pagexml) + + #original Pages + pagedivs = dom.xpath("//div[@class='pageNumberOrig']") + if pagedivs == dom.xpath("//div[@class='pageNumberOrig']"): + if len(pagedivs)>0: + docinfo['pageNumberOrig']= getTextFromNode(pagedivs[0]) + logging.debug("ORIGINAL PAGES: %s"%(docinfo['pageNumberOrig'])) + + #figureEntries + pagedivs = dom.xpath("//div[@class='countFigureEntries']") + if pagedivs == dom.xpath("//div[@class='countFigureEntries']"): + if len(pagedivs)>0: + docinfo['countFigureEntries'] = getTextFromNode(pagedivs[0]) + s = getTextFromNode(pagedivs[0]) + if s=='0': + try: + docinfo['countFigureEntries'] = int(s) + logging.debug("FIGURE ENTRIES: %s"%(s)) + except: + docinfo['countFigureEntries'] = 0 + else: + s1 = int(s)/30+1 + try: + docinfo['countFigureEntries'] = int(s1) + logging.debug("FIGURE ENTRIES: %s"%(s1)) + except: + docinfo['countFigureEntries'] = 0 + + #allPlaces + pagedivs = dom.xpath("//div[@class='countPlaces']") + if pagedivs == dom.xpath("//div[@class='countPlaces']"): + if len(pagedivs)>0: + docinfo['countPlaces']= getTextFromNode(pagedivs[0]) + s = getTextFromNode(pagedivs[0]) + try: + docinfo['countPlaces'] = int(s) + logging.debug("PLACES HERE: %s"%(s)) + except: + docinfo['countPlaces'] = 0 + + #tocEntries + pagedivs = dom.xpath("//div[@class='countTocEntries']") + if pagedivs == dom.xpath("//div[@class='countTocEntries']"): + if len(pagedivs)>0: + docinfo['countTocEntries'] = int(getTextFromNode(pagedivs[0])) + s = getTextFromNode(pagedivs[0]) + if s=='0': + try: + docinfo['countTocEntries'] = int(s) + logging.debug("TEXT ENTRIES: %s"%(s)) + except: + docinfo['countTocEntries'] = 0 + else: + s1 = int(s)/30+1 + try: + docinfo['countTocEntries'] = int(s1) + logging.debug("TEXT ENTRIES: %s"%(s1)) + except: + docinfo['countTocEntries'] = 0 + + #numTextPages + pagedivs = dom.xpath("//div[@class='countPages']") + if pagedivs == dom.xpath("//div[@class='countPages']"): + if len(pagedivs)>0: + docinfo['numPages'] = getTextFromNode(pagedivs[0]) + s = getTextFromNode(pagedivs[0]) + + try: + docinfo['numPages'] = int(s) + logging.debug("PAGE NUMBER: %s"%(s)) + + np = docinfo['numPages'] + pageinfo['end'] = min(pageinfo['end'], np) + pageinfo['numgroups'] = int(np / pageinfo['groupsize']) + if np % pageinfo['groupsize'] > 0: + pageinfo['numgroups'] += 1 + except: + docinfo['numPages'] = 0 + + else: + #no full text -- init to 0 + docinfo['pageNumberOrig'] = 0 + docinfo['countFigureEntries'] = 0 + docinfo['countPlaces'] = 0 + docinfo['countTocEntries'] = 0 + docinfo['numPages'] = 0 + #return docinfo - pagedom = Parse(pagexml) # plain text mode if mode == "text": # first div contains text - pagedivs = pagedom.xpath("/div") + pagedivs = dom.xpath("/div") if len(pagedivs) > 0: pagenode = pagedivs[0] links = pagenode.xpath("//a") @@ -298,13 +306,13 @@ class MpdlXmlTextServer(SimpleItem): return serializeNode(pagenode) if mode == "xml": # first div contains text - pagedivs = pagedom.xpath("/div") + pagedivs = dom.xpath("/div") if len(pagedivs) > 0: pagenode = pagedivs[0] return serializeNode(pagenode) if mode == "gis": # first div contains text - pagedivs = pagedom.xpath("/div") + pagedivs = dom.xpath("/div") if len(pagedivs) > 0: pagenode = pagedivs[0] links =pagenode.xpath("//a") @@ -319,14 +327,14 @@ class MpdlXmlTextServer(SimpleItem): if mode == "pureXml": # first div contains text - pagedivs = pagedom.xpath("/div") + pagedivs = dom.xpath("/div") if len(pagedivs) > 0: pagenode = pagedivs[0] return serializeNode(pagenode) # text-with-links mode if mode == "text_dict": # first div contains text - pagedivs = pagedom.xpath("/div") + pagedivs = dom.xpath("/div") if len(pagedivs) > 0: pagenode = pagedivs[0] # check all a-tags @@ -375,7 +383,7 @@ class MpdlXmlTextServer(SimpleItem): return data def getQuery (self, docinfo=None, pageinfo=None, query=None, queryType=None, pn=1, optionToggle=None): - """number of""" + #number of docpath = docinfo['textURLPath'] pagesize = pageinfo['queryPageSize'] pn = pageinfo['searchPN'] @@ -385,7 +393,6 @@ class MpdlXmlTextServer(SimpleItem): tocDiv = None pagexml = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath, 'text', queryType, urllib.quote(query), pagesize, pn)) - #pagexml=self.template.fulltextclient.eval("/mpdl/interface/doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath, 'text', queryType, query, pagesize, pn) ,outputUnicode=False) pagedom = Parse(pagexml) numdivs = pagedom.xpath("//div[@class='queryResultHits']") tocSearch = int(getTextFromNode(numdivs[0])) @@ -393,58 +400,7 @@ class MpdlXmlTextServer(SimpleItem): tc=int((tocSearch/10)+1) logging.debug("documentViewer (gettoc) tc: %s"%(tc)) return tc - - def getQueryResultHits(self, docinfo=None, pageinfo=None, query=None, queryType=None, pn=1, optionsClose=None): - - """number of hits in Search mode""" - docpath = docinfo['textURLPath'] - pagesize = pageinfo['queryPageSize'] - pn = pageinfo['searchPN'] - query =pageinfo['query'] - queryType =pageinfo['queryType'] - tocSearch = 0 - tocDiv = None - - pagexml = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath, 'text', queryType, urllib.quote(query), pagesize, pn)) - #pagexml=self.template.fulltextclient.eval("/mpdl/interface/doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath, 'text', queryType, query, pagesize, pn) ,outputUnicode=False) - pagedom = Parse(pagexml) - numdivs = pagedom.xpath("//div[@class='queryResultHits']") - tocSearch = int(getTextFromNode(numdivs[0])) - tc=int((tocSearch/10)+1) - return tc - - def getQueryResultHitsText(self, docinfo=None, pageinfo=None): - """number of hits in Text of Contents mode""" - selfurl = self.absolute_url() - docpath = docinfo['textURLPath'] - viewMode= pageinfo['viewMode'] - text = self.getServerData("page-fragment.xql","document=%s&mode=%s"%(docinfo['textURLPath'], 'text')) - dom =Parse(text) - pagedivs = dom.xpath("//div[@class='countTocEntries']") - logging.debug ("pagedivs=%s"%(pagedivs)) - if len(pagedivs)>0: - originalPage= (getTextFromNode(pagedivs[0])) - tc = int (originalPage) - tc1 =tc/30+1 - return tc1 - - def getQueryResultHitsFigures(self, docinfo=None, pageinfo=None): - """number of hits in Text of Figures mode""" - - selfurl = self.absolute_url() - docpath = docinfo['textURLPath'] - viewMode= pageinfo['viewMode'] - text = self.getServerData("page-fragment.xql","document=%s&mode=%s"%(docinfo['textURLPath'], 'text')) - dom =Parse(text) - pagedivs = dom.xpath("//div[@class='countFigureEntries']") - logging.debug ("pagedivs=%s"%(pagedivs)) - if len(pagedivs)>0: - originalPage= (getTextFromNode(pagedivs[0])) - tc = int (originalPage) - tc1 =tc/30+1 - return tc1 - - + def getToc(self, mode="text", docinfo=None): """loads table of contents and stores in docinfo""" logging.debug("documentViewer (gettoc) mode: %s"%(mode)) @@ -467,7 +423,7 @@ class MpdlXmlTextServer(SimpleItem): tocDiv = None pagexml = self.getServerData("doc-query.xql","document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath,queryType, pagesize, pn)) - #pagexml=self.template.fulltextclient.eval("/mpdl/interface/doc-query.xql", "document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath,queryType,pagesize,pn), outputUnicode=False) + # post-processing downloaded xml pagedom = Parse(pagexml) # get number of entries @@ -499,8 +455,8 @@ class MpdlXmlTextServer(SimpleItem): data = self.getServerData("doc-query.xql","document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s&characterNormalization=regPlusNorm&optionToggle=1"%(docpath,queryType, pagesize, pn)) page = data.replace('page-fragment.xql?document=%s'%str(path),'%s?url=%s&viewMode=%s&tocMode=%s&tocPN=%s&optionToggle=1'%(selfurl,url, viewMode, tocMode, tocPN)) text = page.replace('mode=image','mode=texttool') - logging.debug("documentViewer (characterNormalization) characterNormalization: %s"%(characterNormalization)) - #logging.debug("documentViewer (characterNormalization) text: %s"%(text)) + #logging.debug("documentViewer (characterNormalization) characterNormalization: %s"%(characterNormalization)) + return text def manage_changeMpdlXmlTextServer(self,title="",serverUrl="http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/",timeout=40,RESPONSE=None):