Index: MpiwgXmlTextServer.py
===================================================================
--- MpiwgXmlTextServer.py	(revision 564)
+++ MpiwgXmlTextServer.py	(revision 564)
@@ -0,0 +1,579 @@
+from OFS.SimpleItem import SimpleItem
+from Products.PageTemplates.PageTemplateFile import PageTemplateFile 
+
+import xml.etree.ElementTree as ET
+
+import re
+import logging
+import urllib
+import urlparse
+import base64
+
+from SrvTxtUtils import getInt, getText, getHttpData
+
+def serialize(node):
+    """returns a string containing an XML snippet of node"""
+    s = ET.tostring(node, 'UTF-8')
+    # snip off XML declaration
+    if s.startswith('<?xml'):
+        i = s.find('?>')
+        return s[i+3:]
+
+    return s
+
+
+class MpiwgXmlTextServer(SimpleItem):
+    """TextServer implementation for MPIWG-XML server"""
+    meta_type="MPIWG-XML TextServer"
+
+    manage_options=(
+        {'label':'Config','action':'manage_changeMpiwgXmlTextServerForm'},
+       )+SimpleItem.manage_options
+    
+    manage_changeMpiwgXmlTextServerForm = PageTemplateFile("zpt/manage_changeMpiwgXmlTextServer", globals())
+        
+    def __init__(self,id,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de/mpiwg-mpdl-cms-web/", timeout=40, serverName=None, repositoryType='production'):
+        """constructor"""
+        self.id=id
+        self.title=title
+        self.timeout = timeout
+        self.repositoryType = repositoryType
+        if serverName is None:
+            self.serverUrl = serverUrl
+        else:
+            self.serverUrl = "http://%s/mpiwg-mpdl-cms-web/"%serverName
+        
+    def getHttpData(self, url, data=None):
+        """returns result from url+data HTTP request"""
+        return getHttpData(url,data,timeout=self.timeout)
+    
+    def getServerData(self, method, data=None):
+        """returns result from text server for method+data"""
+        url = self.serverUrl+method
+        return getHttpData(url,data,timeout=self.timeout)
+
+
+    def getRepositoryType(self):
+        """returns the repository type, e.g. 'production'"""
+        return self.repositoryType
+
+    def getTextDownloadUrl(self, type='xml', docinfo=None):
+        """returns a URL to download the current text"""
+        docpath = docinfo.get('textURLPath', None)
+        if not docpath:
+            return None
+
+        docpath = docpath.replace('.xml','.'+type)
+        url = '%sdoc/GetDocument?id=%s'%(self.serverUrl.replace('interface/',''), docpath)
+        return url
+
+
+    def getPlacesOnPage(self, docinfo=None, pn=None):
+        """Returns list of GIS places of page pn"""
+        #FIXME!
+        docpath = docinfo.get('textURLPath',None)
+        if not docpath:
+            return None
+
+        places=[]
+        text=self.getServerData("xpath.xql", "document=%s&xpath=//place&pn=%s"%(docpath,pn))
+        dom = ET.fromstring(text)
+        result = dom.findall(".//resultPage/place")
+        for l in result:
+            id = l.get("id")
+            name = l.text
+            place = {'id': id, 'name': name}
+            places.append(place)
+
+        return places
+    
+          
+    def getTextInfo(self, mode='', docinfo=None):
+        """reads document info, including page concordance, from text server"""
+        logging.debug("getTextInfo mode=%s"%mode)
+        if mode not in ['toc', 'figures', '']:
+            mode = ''
+        # check cached info
+        if mode:
+            # cached toc-request?
+            if 'full_%s'%mode in docinfo:
+                return docinfo
+            
+        else:
+            # no toc-request
+            if 'numTextPages' in docinfo:
+                return docinfo
+                
+        docpath = docinfo.get('textURLPath', None)
+        if docpath is None:
+            logging.error("getTextInfo: no textURLPath!")
+            return docinfo
+                
+        # fetch docinfo            
+        pagexml = self.getServerData("query/GetDocInfo","docId=%s&field=%s"%(docpath,mode))
+        dom = ET.fromstring(pagexml)
+        # all info in tag <document>
+        doc = dom.find("doc")
+        if doc is None:
+            logging.error("getTextInfo: unable to find document-tag!")
+        else:
+            # result is in list-tag
+            l = doc.find('list')
+            if l is not None:
+                lt = l.get('type')
+                # pageNumbers
+                if lt == 'pages':
+                    # contains tags with page numbers
+                    # <item n="14" o="2" o-norm="2" file="0014"/>
+                    # n=scan number, o=original page no, on=normalized original page no
+                    # pageNumbers is a dict indexed by scan number
+                    pages = {}
+                    for i in l:
+                        page = {}
+                        pn = getInt(i.get('n'))
+                        page['pn'] = pn
+                        no = getInt(i.get('o'))
+                        page['no'] = no
+                        non = getInt(i.get('o-norm'))
+                        page['non'] = non
+                                
+                        if pn > 0:
+                            pages[pn] = page
+                        
+                    docinfo['numTextPages'] = len(pages)
+                    docinfo['pageNumbers'] = pages
+                    logging.debug("got pageNumbers=%s"%repr(pages))
+                                
+                # toc
+                elif name == 'toc':
+                    # contains tags with table of contents/figures
+                    # <toc-entry><page>13</page><level>3</level><content>Chapter I</content><level-string>1.</level-string><real-level>1</real-level></toc-entry>
+                    tocs = []
+                    for te in tag:
+                        toc = {}
+                        for t in te:
+                            if t.tag == 'page':
+                                toc['pn'] = getInt(t.text)
+                            elif t.tag == 'level':
+                                toc['level'] = t.text
+                            elif t.tag == 'content':
+                                toc['content'] = t.text
+                            elif t.tag == 'level-string':
+                                toc['level-string'] = t.text
+                            elif t.tag == 'real-level':
+                                toc['real-level'] = t.text
+                                
+                        tocs.append(toc)
+                    
+                    # save as full_toc/full_figures
+                    docinfo['full_%s'%mode] = tocs
+
+        return docinfo
+        
+          
+    def processPageInfo(self, dom, docinfo, pageinfo):
+        """processes page info divs from dom and stores in docinfo and pageinfo"""
+        # assume first second level div is pageMeta
+        alldivs = dom.find("div")
+        
+        if alldivs is None or alldivs.get('class', '') != 'pageMeta':
+            logging.error("processPageInfo: pageMeta div not found!")
+            return
+        
+        for div in alldivs:
+            dc = div.get('class')
+            
+            # pageNumberOrig  
+            if dc == 'pageNumberOrig':
+                pageinfo['pageNumberOrig'] = div.text
+                
+            # pageNumberOrigNorm
+            elif dc == 'pageNumberOrigNorm':
+                pageinfo['pageNumberOrigNorm'] = div.text
+                
+            # pageHeaderTitle
+            elif dc == 'pageHeaderTitle':
+                pageinfo['pageHeaderTitle'] = div.text
+                        
+        #logging.debug("processPageInfo: pageinfo=%s"%repr(pageinfo))
+        return
+         
+           
+    def getTextPage(self, mode="text", pn=1, docinfo=None, pageinfo=None):
+        """returns single page from fulltext"""
+        
+        logging.debug("getTextPage mode=%s, pn=%s"%(mode,pn))
+        # check for cached text -- but ideally this shouldn't be called twice
+        if pageinfo.has_key('textPage'):
+            logging.debug("getTextPage: using cached text")
+            return pageinfo['textPage']
+        
+        docpath = docinfo.get('textURLPath', None)
+        if not docpath:
+            return None
+        
+        # just checking
+        if pageinfo['current'] != pn:
+            logging.warning("getTextPage: current!=pn!")
+            
+        # stuff for constructing full urls
+        selfurl = docinfo['viewerUrl']
+        textParams = {'docId': docpath,
+                      'page': pn}
+        if 'characterNormalization' in pageinfo:
+            textParams['normalization'] = pageinfo['characterNormalization']
+        
+        if not mode:
+            # default is dict
+            mode = 'text'
+
+        modes = mode.split(',')
+        # check for multiple layers
+        if len(modes) > 1:
+            logging.debug("getTextPage: more than one mode=%s"%mode)
+                        
+        # search mode
+        if 'search' in modes:
+            # add highlighting
+            highlightQuery = pageinfo.get('highlightQuery', None)
+            if highlightQuery:
+                textParams['highlightQuery'] = highlightQuery
+                textParams['highlightElem'] = pageinfo.get('highlightElement', '')
+                textParams['highlightElemPos'] = pageinfo.get('highlightElementPos', '')
+                
+            # ignore mode in the following
+            modes.remove('search')
+                            
+        # pundit mode
+        punditMode = False
+        if 'pundit' in modes:
+            punditMode = True
+            # ignore mode in the following
+            modes.remove('pundit')
+                            
+        # other modes don't combine
+        if 'dict' in modes:
+            textmode = 'dict'
+            textParams['mode'] = 'tokenized'
+            textParams['outputFormat'] = 'html'
+        elif 'xml' in modes:
+            textmode = 'xml'
+            textParams['mode'] = 'untokenized'
+            textParams['outputFormat'] = 'xmlDisplay'
+            textParams['normalization'] = 'orig'
+        elif 'gis' in modes:
+            #FIXME!
+            textmode = 'gis'
+        else:
+            # text is default mode
+            textmode = 'text'
+            textParams['mode'] = 'untokenized'
+            textParams['outputFormat'] = 'html'
+        
+        # fetch the page
+        pagexml = self.getServerData("query/GetPage",urllib.urlencode(textParams))
+        dom = ET.fromstring(pagexml)
+        # extract additional info
+        #self.processPageInfo(dom, docinfo, pageinfo)
+        # page content is in <div class="pageContent">
+        pagediv = None
+        body = dom.find('.//body')
+        if body is None:
+            logging.error("getTextPage: no body!")
+            return None
+        
+        # currently there's lots of divs...
+        textspan = body.find('span/span')
+        divs = textspan.findall('div')
+        logging.debug("textdivs: %s"%repr(divs))
+        pagediv = divs[0]
+        logging.debug("pagediv: %s"%serialize(pagediv))
+        
+        # plain text mode
+        if textmode == "text":
+            # get full url assuming documentViewer is parent
+            selfurl = self.getLink()
+            if pagediv is not None:
+                if punditMode:
+                    pagediv = self.addPunditAttributes(pagediv, pageinfo, docinfo)
+                    
+                # fix empty div tags
+                divs = pagediv.findall('.//div')
+                for d in divs:
+                    if len(d) == 0 and not d.text:
+                        # make empty divs non-empty
+                        d.text = ' '
+                    
+                # check all a-tags
+                links = pagediv.findall('.//a')
+                for l in links:
+                    href = l.get('href')
+                    if href and href.startswith('#note-'):
+                        href = href.replace('#note-',"%s#note-"%selfurl)
+                        l.set('href', href)
+
+                return serialize(pagediv)
+            
+        # text-with-links mode
+        elif textmode == "dict":
+            if pagediv is not None:
+                viewerurl = docinfo['viewerUrl']
+                selfurl = self.getLink()
+                if punditMode:
+                    pagediv = self.addPunditAttributes(pagediv, pageinfo, docinfo)
+                    
+                # fix empty div tags
+                divs = pagediv.findall('.//div')
+                for d in divs:
+                    if len(d) == 0 and not d.text:
+                        # make empty divs non-empty
+                        d.text = ' '
+                    
+                # check all a-tags
+                links = pagediv.findall(".//a")
+                for l in links:
+                    href = l.get('href')
+                    
+                    if href:
+                        # is link with href
+                        linkurl = urlparse.urlparse(href)
+                        #logging.debug("getTextPage: linkurl=%s"%repr(linkurl))
+                        if linkurl.path.endswith('GetDictionaryEntries'):
+                            #TODO: replace wordInfo page
+                            # is dictionary link - change href (keeping parameters)
+                            #l.set('href', href.replace('http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/lt/wordInfo.xql','%s/template/viewer_wordinfo'%viewerurl))
+                            # add target to open new page
+                            l.set('target', '_blank')
+                                                          
+                        if href.startswith('#note-'):
+                            # note link
+                            l.set('href', href.replace('#note-',"%s#note-"%selfurl))
+                              
+                return serialize(pagediv)
+            
+        # xml mode
+        elif textmode == "xml":
+            if pagediv is not None:
+                return serialize(pagediv)
+            
+        # pureXml mode WTF?
+        elif textmode == "pureXml":
+            if pagediv is not None:
+                return serialize(pagediv)
+                  
+        # gis mode
+        elif textmode == "gis":
+            if pagediv is not None:
+                # fix empty div tags
+                divs = pagediv.findall('.//div')
+                for d in divs:
+                    if len(d) == 0 and not d.text:
+                        # make empty divs non-empty
+                        d.text = ' '
+                    
+                # check all a-tags
+                links = pagediv.findall(".//a")
+                # add our URL as backlink
+                selfurl = self.getLink()
+                doc = base64.b64encode(selfurl)
+                for l in links:
+                    href = l.get('href')
+                    if href:
+                        if href.startswith('http://mappit.mpiwg-berlin.mpg.de'):
+                            l.set('href', re.sub(r'doc=[\w+/=]+', 'doc=%s'%doc, href))
+                            l.set('target', '_blank')
+                            
+                return serialize(pagediv)
+                    
+        return None
+    
+    def addPunditAttributes(self, pagediv, pageinfo, docinfo):
+        """add about attributes for pundit annotation tool"""
+        textid = docinfo.get('DRI', "fn=%s"%docinfo.get('documentPath', '???'))
+        pn = pageinfo.get('pn', '1')
+        #  TODO: use pn as well?
+        # check all div-tags
+        divs = pagediv.findall(".//div")
+        for d in divs:
+            id = d.get('id')
+            if id:
+                d.set('about', "http://echo.mpiwg-berlin.mpg.de/%s/pn=%s/#%s"%(textid,pn,id))
+                cls = d.get('class','')
+                cls += ' pundit-content'
+                d.set('class', cls.strip())
+
+        return pagediv
+
+    def getSearchResults(self, mode, query=None, pageinfo=None, docinfo=None):
+        """loads list of search results and stores XML in docinfo"""
+        
+        logging.debug("getSearchResults mode=%s query=%s"%(mode, query))
+        if mode == "none":
+            return docinfo
+              
+        cachedQuery = docinfo.get('cachedQuery', None)
+        if cachedQuery is not None:
+            # cached search result
+            if cachedQuery == '%s_%s'%(mode,query):
+                # same query
+                return docinfo
+            
+            else:
+                # different query
+                del docinfo['resultSize']
+                del docinfo['resultXML']
+        
+        # cache query
+        docinfo['cachedQuery'] = '%s_%s'%(mode,query)
+        
+        # fetch full results
+        docpath = docinfo['textURLPath']
+        params = {'document': docpath,
+                  'mode': 'text',
+                  'queryType': mode,
+                  'query': query,
+                  'queryResultPageSize': 1000,
+                  'queryResultPN': 1,
+                  'characterNormalization': pageinfo.get('characterNormalization', 'reg')}
+        pagexml = self.getServerData("doc-query.xql",urllib.urlencode(params))
+        #pagexml = self.getServerData("doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&s=%s&viewMode=%s&characterNormalization=%s&highlightElementPos=%s&highlightElement=%s&highlightQuery=%s"%(docpath, 'text', queryType, urllib.quote(query), pagesize, pn, s, viewMode,characterNormalization, highlightElementPos, highlightElement, urllib.quote(highlightQuery)))
+        dom = ET.fromstring(pagexml)
+        # page content is in <div class="queryResultPage">
+        pagediv = None
+        # ElementTree 1.2 in Python 2.6 can't do div[@class='queryResultPage']
+        alldivs = dom.findall("div")
+        for div in alldivs:
+            dc = div.get('class')
+            # page content div
+            if dc == 'queryResultPage':
+                pagediv = div
+                
+            elif dc == 'queryResultHits':
+                docinfo['resultSize'] = getInt(div.text)
+
+        if pagediv is not None:
+            # store XML in docinfo
+            docinfo['resultXML'] = ET.tostring(pagediv, 'UTF-8')
+
+        return docinfo
+    
+
+    def getResultsPage(self, mode="text", query=None, pn=None, start=None, size=None, pageinfo=None, docinfo=None):
+        """returns single page from the table of contents"""
+        logging.debug("getResultsPage mode=%s, pn=%s"%(mode,pn))
+        # get (cached) result
+        self.getSearchResults(mode=mode, query=query, pageinfo=pageinfo, docinfo=docinfo)
+            
+        resultxml = docinfo.get('resultXML', None)
+        if not resultxml:
+            logging.error("getResultPage: unable to find resultXML")
+            return "Error: no result!"
+        
+        if size is None:
+            size = pageinfo.get('resultPageSize', 10)
+            
+        if start is None:
+            start = (pn - 1) * size
+
+        fullresult = ET.fromstring(resultxml)
+        
+        if fullresult is not None:
+            # paginate
+            first = start-1
+            len = size
+            del fullresult[:first]
+            del fullresult[len:]
+            tocdivs = fullresult
+            
+            # check all a-tags
+            links = tocdivs.findall(".//a")
+            for l in links:
+                href = l.get('href')
+                if href:
+                    # assume all links go to pages
+                    linkUrl = urlparse.urlparse(href)
+                    linkParams = urlparse.parse_qs(linkUrl.query)
+                    # take some parameters
+                    params = {'pn': linkParams['pn'],
+                              'highlightQuery': linkParams.get('highlightQuery',''),
+                              'highlightElement': linkParams.get('highlightElement',''),
+                              'highlightElementPos': linkParams.get('highlightElementPos','')
+                              }
+                    url = self.getLink(params=params)
+                    l.set('href', url)
+                        
+            return serialize(tocdivs)
+        
+        return "ERROR: no results!"
+
+
+    def getToc(self, mode='text', docinfo=None):
+        """returns list of table of contents from docinfo"""
+        logging.debug("getToc mode=%s"%mode)
+        if mode == 'text':
+            queryType = 'toc'
+        else:
+            queryType = mode
+            
+        if not 'full_%s'%queryType in docinfo:
+            # get new toc
+            docinfo = self.getTextInfo(queryType, docinfo)
+            
+        return docinfo.get('full_%s'%queryType, [])
+
+    def getTocPage(self, mode='text', pn=None, start=None, size=None, pageinfo=None, docinfo=None):
+        """returns single page from the table of contents"""
+        logging.debug("getTocPage mode=%s, pn=%s start=%s size=%s"%(mode,repr(pn),repr(start),repr(size)))
+        fulltoc = self.getToc(mode=mode, docinfo=docinfo)
+        if len(fulltoc) < 1:
+            logging.error("getTocPage: unable to find toc!")
+            return "Error: no table of contents!"        
+        
+        if size is None:
+            size = pageinfo.get('tocPageSize', 30)
+            
+        if start is None:
+            start = (pn - 1) * size
+
+        # paginate
+        first = (start - 1)
+        last = first + size
+        tocs = fulltoc[first:last]
+        tp = '<div>'
+        for toc in tocs:
+            pageurl = self.getLink('pn', toc['pn'])
+            tp += '<div class="tocline">'
+            tp += '<div class="toc name">[%s %s]</div>'%(toc['level-string'], toc['content'])
+            tp += '<div class="toc float right page"><a href="%s">Page: %s</a></div>'%(pageurl, toc['pn'])
+            tp += '</div>\n'
+            
+        tp += '</div>\n'
+        
+        return tp
+           
+    
+    def manage_changeMpiwgXmlTextServer(self,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de/mpdl/interface/",timeout=40,repositoryType=None,RESPONSE=None):
+        """change settings"""
+        self.title=title
+        self.timeout = timeout
+        self.serverUrl = serverUrl
+        if repositoryType:
+            self.repositoryType = repositoryType
+        if RESPONSE is not None:
+            RESPONSE.redirect('manage_main')
+        
+# management methods
+def manage_addMpiwgXmlTextServerForm(self):
+    """Form for adding"""
+    pt = PageTemplateFile("zpt/manage_addMpiwgXmlTextServer", globals()).__of__(self)
+    return pt()
+
+def manage_addMpiwgXmlTextServer(self,id,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de/mpdl/interface/",timeout=40,RESPONSE=None):
+#def manage_addMpiwgXmlTextServer(self,id,title="",serverUrl="http://mpdl-text.mpiwg-berlin.mpg.de:30030/mpdl/interface/",timeout=40,RESPONSE=None):    
+    """add zogiimage"""
+    newObj = MpiwgXmlTextServer(id=id,title=title,serverUrl=serverUrl,timeout=timeout)
+    self.Destination()._setObject(id, newObj)
+    if RESPONSE is not None:
+        RESPONSE.redirect('manage_main')
+        
+        
Index: __init__.py
===================================================================
--- __init__.py	(revision 522)
+++ __init__.py	(revision 564)
@@ -1,4 +1,5 @@
 import documentViewer
 import MpdlXmlTextServer
+import MpiwgXmlTextServer
 
 def initialize(context):
@@ -19,3 +20,11 @@
           )
         )
+
+    context.registerClass(
+        MpiwgXmlTextServer.MpiwgXmlTextServer,
+        constructors = (
+          MpiwgXmlTextServer.manage_addMpiwgXmlTextServerForm,
+          MpiwgXmlTextServer.manage_addMpiwgXmlTextServer
+          )
+        )
     
Index: documentViewer.py
===================================================================
--- documentViewer.py	(revision 562)
+++ documentViewer.py	(revision 564)
@@ -587,5 +587,5 @@
                 # document info (including toc) from full text
                 if docinfo.get('textURLPath', None):
-                    docinfo = self.getTextInfo(mode=tocMode, docinfo=docinfo)
+                    docinfo = self.getTextInfo(mode='pages', docinfo=docinfo)
             
             # bib info
@@ -724,4 +724,5 @@
                         docinfo['textURLPath'] = textUrl
                         docinfo['textURLRepository'] = textRepo
+                        break
                 
                 else:
Index: zpt/manage_addMpiwgXmlTextServer.zpt
===================================================================
--- zpt/manage_addMpiwgXmlTextServer.zpt	(revision 564)
+++ zpt/manage_addMpiwgXmlTextServer.zpt	(revision 564)
@@ -0,0 +1,11 @@
+<h1 tal:replace="structure here/manage_page_header">Header</h1>
+	<h2>Add an MPIWG-XML TextServer</h2>
+		<form action="manage_addMpiwgXmlTextServer">
+		<p class="form-label">Id<input name="id"></p>
+		<p class="form-optional">Title<input size="80" name="title"></p>
+		<p class="form-label">Text Server URL<input size="80" name="serverUrl" value="http://mpdl-text.mpiwg-berlin.mpg.de/mpiwg-mpdl-cms-web/"></p>
+		<p class="form-optional">Timeout (s)<input size="10" name="timeout" value="30"></p>
+		<p><input type="submit" value="add"></p>
+		</form>
+			
+<h1 tal:replace="structure here/manage_page_footer">Footer</h1>
Index: zpt/manage_changeMpiwgXmlTextServer.zpt
===================================================================
--- zpt/manage_changeMpiwgXmlTextServer.zpt	(revision 564)
+++ zpt/manage_changeMpiwgXmlTextServer.zpt	(revision 564)
@@ -0,0 +1,19 @@
+<div tal:replace="structure here/manage_page_header">Header</div>
+<!-- ZOPE management tabs -->
+<h2 tal:define="manage_tabs_message options/manage_tabs_message | nothing"
+    tal:replace="structure here/manage_tabs">Tabs</h2>
+<!-- end of ZOPE management tabs -->
+		<form action="manage_changeMpiwgXmlTextServer">
+		
+		<p class="form-optional">Title</p>
+        <p class="form-element"><input size="80" tal:attributes="value here/title" name="title"></p>
+        <p class="form-optional">Text Server URL</p>
+        <p class="form-element"><input size="80" tal:attributes="value here/serverUrl | default" name="serverUrl"></p>
+        <p class="form-optional">Timeout (s)</p>
+        <p class="form-element"><input size="3" tal:attributes="value here/timeout | default" name="timeout"></p>
+        <p class="form-optional">Repository type</p>
+        <p class="form-element"><input size="10" tal:attributes="value here/repositoryType | default" name="repositoryType"></p>
+		<p><input type="submit" value="change"></p>
+		</form>
+			
+<div tal:replace="structure here/manage_page_footer">Footer</div>
