Index: extraFunction.py
===================================================================
--- extraFunction.py	(revision 3)
+++ extraFunction.py	(revision 4)
@@ -54,4 +54,49 @@
         self.id=id
         self.title=title
+        
+    def getHttpData(self, url, data=None, num_tries=3, timeout=40):
+        """returns result from url+data HTTP request"""
+        # we do GET (by appending data to url)
+        if isinstance(data, str) or isinstance(data, unicode):
+            # if data is string then append
+            url = "%s?%s"%(url,data)
+        else:
+            # we assume its a dict
+            url = "%s?%s"%(url,urllib.urlencode(data))
+        
+        response = None
+        errmsg = None
+        for cnt in range(num_tries):
+            try:
+                logging.debug("getHttpData(%s) url=%s"%(cnt+1,url))
+                if sys.version_info < (2, 6):
+                    # set timeout on socket -- ugly :-(
+                    import socket
+                    socket.setdefaulttimeout(timeout)
+                    response = urllib2.urlopen(url)
+                else:
+                    response = urllib2.urlopen(url,timeout=timeout)
+                # check result?
+                break
+            except urllib2.HTTPError, e:
+                logging.error("getHttpData: HTTP error(%s): %s"%(e.code,e))
+                errmsg = str(e)
+                # stop trying
+                break
+            except urllib2.URLError, e:
+                logging.error("getHttpData: URLLIB error(%s): %s"%(e.reason,e))
+                errmsg = str(e)
+                # stop trying
+                #break
+
+        if response is not None:
+            data = response.read()
+            response.close()
+            return data
+        
+        raise IOError("ERROR fetching HTTP data from %s: %s"%(url,errmsg))
+        #return None
+
+                
     
     def getSearch(self, pn=1, pageinfo=None,  docinfo=None, query=None, queryType=None, lemma=None):
@@ -72,8 +117,8 @@
         selfurl = self.absolute_url()
         
-        page = urllib2.urlopen("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&sn=%s&viewMode=%s&highlightQuery=%s"%(docpath, 'text', queryType, query, pagesize, pn, sn, viewMode,highlightQuery))
+        data = self.getHttpData("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&sn=%s&viewMode=%s&highlightQuery=%s"%(docpath, 'text', queryType, query, pagesize, pn, sn, viewMode,highlightQuery))
         #page=self.template.fulltextclient.eval("/mpdl/interface/doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s&sn=%s&viewMode=%s&highlightQuery=%s"%(docpath, 'text', queryType, query, pagesize, pn, sn, viewMode,highlightQuery) ,outputUnicode=False)                
-        data = page.read()
-        page.close()
+        #data = page.read()
+        #page.close()
         
         pagexml = data.replace('?document=%s'%str(docpath),'?url=%s'%url)
@@ -139,5 +184,5 @@
         """get list of pages from fulltext and put in docinfo"""
         xquery = '//pb'
-        text = urllib2.urlopen("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/xquery.xql","document=%s&xquery=%s"%(docinfo['textURLPath'],xquery))
+        text = self.getHttpData("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/xquery.xql","document=%s&xquery=%s"%(docinfo['textURLPath'],xquery))
         #text = self.template.fulltextclient.eval("/mpdl/interface/xquery.xql", "document=%s&xquery=%s"%(docinfo['textURLPath'],xquery))
         docinfo['numPages'] = text.count("<pb ")
@@ -162,5 +207,5 @@
             textParam +="&highlightQuery=%s&sn=%s"%(highlightQuery,sn)           
         
-        pagexml = urllib2.urlopen("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/page-fragment.xql",textParam)
+        pagexml = self.getHttpData("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/page-fragment.xql",textParam)
         """pagexml=self.template.fulltextclient.eval("/mpdl/interface/page-fragment.xql", textParam, outputUnicode=False)"""
         
@@ -227,25 +272,24 @@
     def getTranslate(self, query=None, language=None):
         """translate into another languages"""
-        pagexml = urllib2.urlopen("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/lt/lex.xql","document=&language="+str(language)+"&query="+url_quote(str(query)))
+        data = self.getHttpData("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/lt/lex.xql","document=&language="+str(language)+"&query="+url_quote(str(query)))
         #pagexml=self.template.fulltextclient.eval("/mpdl/interface/lt/lex.xql","document=&language="+str(language)+"&query="+url_quote(str(query)))
-        data = pagexml.read()
-        pagexml.close()
+        #data = pagexml.read()
+        #pagexml.close()
         return data
     
     def getLemma(self, lemma=None, language=None):
         """simular words lemma """
-        pagexml = urllib2.urlopen("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/lt/lemma.xql","document=&language="+str(language)+"&lemma="+url_quote(str(lemma)))
+        data = self.getHttpData("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/lt/lemma.xql","document=&language="+str(language)+"&lemma="+url_quote(str(lemma)))
         #pagexml=self.template.fulltextclient.eval("/mpdl/interface/lt/lemma.xql","document=&language="+str(language)+"&lemma="+url_quote(str(lemma)))
-        data = pagexml.read()
-        pagexml.close()
+        #data = pagexml.read()
+        #pagexml.close()
         return data
     
     def getLemmaNew(self, query=None, language=None):
         """simular words lemma """
-        
-        pagexml = urllib2.urlopen("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/lt/lemma.xql","document=&language="+str(language)+"&lemma="+url_quote(str(query)))
+        data = self.getHttpData("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/lt/lemma.xql","document=&language="+str(language)+"&lemma="+url_quote(str(query)))
         #pagexml=self.template.fulltextclient.eval("/mpdl/interface/lt/lemma.xql","document=&language="+str(language)+"&lemma="+url_quote(str(query)))
-        data = pagexml.read()
-        pagexml.close()
+        #data = pagexml.read()
+        #pagexml.close()
         return data
 
@@ -260,5 +304,5 @@
          tocDiv = None
          
-         pagexml = urllib2.urlopen("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath, 'text', queryType, query, pagesize, pn))
+         pagexml = self.getHttpData("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath, 'text', queryType, query, pagesize, pn))
          #pagexml=self.template.fulltextclient.eval("/mpdl/interface/doc-query.xql","document=%s&mode=%s&queryType=%s&query=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath, 'text', queryType, query, pagesize, pn) ,outputUnicode=False)
          pagedom = Parse(pagexml)
@@ -290,5 +334,5 @@
         tocDiv = None
         
-        pagexml = urllib2.urlopen("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/doc-query.xql","document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath,queryType, pagesize, pn))
+        pagexml = self.getHttpData("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/doc-query.xql","document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath,queryType, pagesize, pn))
         #pagexml=self.template.fulltextclient.eval("/mpdl/interface/doc-query.xql", "document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath,queryType,pagesize,pn), outputUnicode=False)
         # post-processing downloaded xml
@@ -318,7 +362,7 @@
         tocPN = pageinfo['tocPN']  
         
-        pagexml = urllib2.urlopen("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/doc-query.xql","document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath,queryType, pagesize, pn))  
-        data = pagexml.read()
-        pagexml.close()
+        data = self.getHttpData("http://mpdl-proto.mpiwg-berlin.mpg.de/mpdl/interface/doc-query.xql","document=%s&queryType=%s&queryResultPageSize=%s&queryResultPN=%s"%(docpath,queryType, pagesize, pn))  
+        #data = pagexml.read()
+        #pagexml.close()
 
         page = data.replace('page-fragment.xql?document=%s'%str(path),'%s?url=%s&viewMode=%s&tocMode=%s&tocPN=%s'%(selfurl,url, viewMode, tocMode, tocPN))
