|
0
|
1 """Utility methods for handling XML, reading HTTP, etc"""
|
|
|
2
|
|
|
3 import sys
|
|
|
4 import urllib
|
|
|
5 import urllib2
|
|
|
6 import logging
|
|
|
7
|
|
|
8
|
|
|
9 srvTxtUtilsVersion = "1.0"
|
|
|
10
|
|
|
11 def getInt(number, default=0):
|
|
|
12 """returns always an int (0 in case of problems)"""
|
|
|
13 try:
|
|
|
14 return int(number)
|
|
|
15 except:
|
|
|
16 return int(default)
|
|
|
17
|
|
|
18 def getText(node):
|
|
|
19 """returns all text content of a node and its subnodes"""
|
|
|
20 if node is None:
|
|
|
21 return ""
|
|
|
22 # ElementTree:
|
|
|
23 text = node.text or ""
|
|
|
24 for e in node:
|
|
|
25 text += gettext(e)
|
|
|
26 if e.tail:
|
|
|
27 text += e.tail
|
|
|
28
|
|
|
29 # 4Suite:
|
|
|
30 #nodelist=node.childNodes
|
|
|
31 #text = ""
|
|
|
32 #for n in nodelist:
|
|
|
33 # if n.nodeType == node.TEXT_NODE:
|
|
|
34 # text = text + n.data
|
|
|
35
|
|
|
36 return text
|
|
|
37
|
|
|
38
|
|
|
39
|
|
|
40 def getHttpData(url, data=None, num_tries=3, timeout=10):
|
|
|
41 """returns result from url+data HTTP request"""
|
|
|
42 # we do GET (by appending data to url)
|
|
|
43 if isinstance(data, str) or isinstance(data, unicode):
|
|
|
44 # if data is string then append
|
|
|
45 url = "%s?%s"%(url,data)
|
|
|
46 elif isinstance(data, dict) or isinstance(data, list) or isinstance(data, tuple):
|
|
|
47 # urlencode
|
|
|
48 url = "%s?%s"%(url,urllib.urlencode(data))
|
|
|
49
|
|
|
50 response = None
|
|
|
51 errmsg = None
|
|
|
52 for cnt in range(num_tries):
|
|
|
53 try:
|
|
|
54 logging.debug("getHttpData(#%s %ss) url=%s"%(cnt+1,timeout,url))
|
|
|
55 if sys.version_info < (2, 6):
|
|
|
56 # set timeout on socket -- ugly :-(
|
|
|
57 import socket
|
|
|
58 socket.setdefaulttimeout(float(timeout))
|
|
|
59 response = urllib2.urlopen(url)
|
|
|
60 else:
|
|
|
61 # timeout as parameter
|
|
|
62 response = urllib2.urlopen(url,timeout=float(timeout))
|
|
|
63 # check result?
|
|
|
64 break
|
|
|
65 except urllib2.HTTPError, e:
|
|
|
66 logging.error("getHttpData: HTTP error(%s): %s"%(e.code,e))
|
|
|
67 errmsg = str(e)
|
|
|
68 # stop trying
|
|
|
69 break
|
|
|
70 except urllib2.URLError, e:
|
|
|
71 logging.error("getHttpData: URLLIB error(%s): %s"%(e.reason,e))
|
|
|
72 errmsg = str(e)
|
|
|
73 # stop trying
|
|
|
74 #break
|
|
|
75
|
|
|
76 if response is not None:
|
|
|
77 data = response.read()
|
|
|
78 response.close()
|
|
|
79 return data
|
|
|
80
|
|
|
81 raise IOError("ERROR fetching HTTP data from %s: %s"%(url,errmsg))
|
|
|
82 #return None
|
|
|
83
|
