Annotation of MPIWGWeb/updatePersonalWWW.py, revision 1.1.2.8
1.1.2.1 dwinter 1: try:
2: import psycopg2 as psycopg
3: psyco = 2
4: except:
5: import psycopg
6: psyco = 1
7:
8: import logging
1.1.2.8 ! casties 9: from MPIWGHelper import unicodify, utf8ify
1.1.2.1 dwinter 10:
11: from xml import sax
12: from amara import saxtools
13:
1.1.2.4 casties 14: # namespace for FileMaker8
1.1.2.1 dwinter 15: fm_ns = 'http://www.filemaker.com/fmpxmlresult'
1.1.2.4 casties 16:
17: # list of fields that are taken from XML and checked against DB as conflicts
1.1.2.1 dwinter 18: #checkFields=['key','first_name','last_name','title','home_inst','current_work','e_mail2']
1.1.2.7 casties 19: checkFields=['key','first_name','last_name','title','titles_new','home_inst','e_mail2']
1.1.2.1 dwinter 20:
21: def sql_quote(v):
22: # quote dictionary
23: quote_dict = {"\'": "''", "\\": "\\\\"}
24: for dkey in quote_dict.keys():
25: if v.find(dkey) >= 0:
26: v=quote_dict[dkey].join(v.split(dkey))
27: return "'%s'"%v
28:
29: def SimpleSearch(curs,query, args=None):
30: """execute sql query and return data"""
31: logging.debug("executing: "+query)
32: if psyco == 1:
33: query = query.encode("UTF-8")
34: #if args is not None:
35: # args = [ sql_quote(a) for a in args ]
36: #logging.debug(query)
37: #logging.debug(args)
38:
39: curs.execute(query, args)
40: logging.debug("sql done")
41: try:
42: return curs.fetchall()
43: except:
44: return None
45:
46: class xml_handler:
47:
48: def __init__(self):
49: '''
50: SAX handler to import FileMaker XML file (FMPXMLRESULT format) into the table.
51: @param dsn: database connection string
52: @param table: name of the table the xml shall be imported into
53: '''
54:
55: # set up parser
56: self.result={}
57: self.event = None
58: self.top_dispatcher = {
59: (saxtools.START_ELEMENT, fm_ns, u'METADATA'):
60: self.handle_meta_fields,
61: (saxtools.START_ELEMENT, fm_ns, u'RESULTSET'):
62: self.handle_data,
63: }
64:
65: # connect database
66:
67:
68:
69:
70: self.dbIDs = {}
71: self.rowcnt = 0
72:
73:
74:
75: self.newDataset = []
76: self.conflicts = []
77: self.ok = []
78: self.fieldNames=[]
79: return
80:
81: def handle_meta_fields(self, end_condition):
82: dispatcher = {
83: (saxtools.START_ELEMENT, fm_ns, u'FIELD'):
84: self.handle_meta_field,
85: }
86: #First round through the generator corresponds to the
87: #start element event
88: logging.debug("START METADATA")
89: yield None
90:
91: #delegate is a generator that handles all the events "within"
92: #this element
93: delegate = None
94: while not self.event == end_condition:
95: delegate = saxtools.tenorsax.event_loop_body(
96: dispatcher, delegate, self.event)
97: yield None
98:
99: #Element closed. Wrap up
100: logging.debug("END METADATA")
101:
102: self.update_fields = self.fieldNames
103:
104: logging.debug("xml-fieldnames:"+repr(self.fieldNames))
105: # get list of fields in db table
106:
107: #print "upQ: ", self.updQuery
108: #print "adQ: ", self.addQuery
109:
110: return
111:
112: def handle_meta_field(self, end_condition):
113: name = self.params.get((None, u'NAME'))
114: yield None
115: #Element closed. Wrap up
116: name=name.replace(" ","_")# make sure no spaces
117: self.fieldNames.append(name)
118: logging.debug("FIELD name: "+name)
119: return
120:
121: def handle_data(self, end_condition):
122: dispatcher = {
123: (saxtools.START_ELEMENT, fm_ns, u'ROW'):
124: self.handle_row,
125: }
126: #First round through the generator corresponds to the
127: #start element event
128: logging.debug("START RESULTSET")
129: self.rowcnt = 0
130: yield None
131:
132: #delegate is a generator that handles all the events "within"
133: #this element
134: delegate = None
135: while not self.event == end_condition:
136: delegate = saxtools.tenorsax.event_loop_body(
137: dispatcher, delegate, self.event)
138: yield None
139:
140: #Element closed. Wrap up
141: logging.debug("END RESULTSET")
142:
143:
144:
145: return
146:
147: def handle_row(self, end_condition):
148: dispatcher = {
149: (saxtools.START_ELEMENT, fm_ns, u'COL'):
150: self.handle_col,
151: }
152: logging.debug("START ROW")
153: self.dataSet = {}
154: self.colIdx = 0
155: yield None
156:
157: #delegate is a generator that handles all the events "within"
158: #this element
159: delegate = None
160: while not self.event == end_condition:
161: delegate = saxtools.tenorsax.event_loop_body(
162: dispatcher, delegate, self.event)
163: yield None
164:
165: #Element closed. Wrap up
166: logging.debug("END ROW")
167: self.rowcnt += 1
168: # process collected row data
169: update=False
170: id_val=''
171:
172: if self.result.has_key(self.dataSet['key']):
173: logging.error("Key %s not unique"%self.dataSet['key'])
174:
175: self.result[self.dataSet['key']]=self.dataSet
176:
177:
178: return
179:
180: def handle_col(self, end_condition):
181: dispatcher = {
182: (saxtools.START_ELEMENT, fm_ns, u'DATA'):
183: self.handle_data_tag,
184: }
185: #print "START COL"
186: yield None
187: #delegate is a generator that handles all the events "within"
188: #this element
189: delegate = None
190: while not self.event == end_condition:
191: delegate = saxtools.tenorsax.event_loop_body(
192: dispatcher, delegate, self.event)
193: yield None
194: #Element closed. Wrap up
195: #print "END COL"
196: self.colIdx += 1
197: return
198:
199: def handle_data_tag(self, end_condition):
200: #print "START DATA"
201: content = u''
202: yield None
203: # gather child elements
204: while not self.event == end_condition:
205: if self.event[0] == saxtools.CHARACTER_DATA:
206: content += self.params
207: yield None
208: #Element closed. Wrap up
209: field = self.fieldNames[self.colIdx]
210: self.dataSet[field.lower()] = content
211: #print " DATA(", field, ") ", repr(content)
212: return
213:
214:
215: def checkImport(dsn,resultSet):
216: #now connect to the database
1.1.2.3 casties 217: logging.info("dsn: %s"%dsn)
1.1.2.1 dwinter 218: dbCon = psycopg.connect(dsn)
219: db = dbCon.cursor()
220:
221:
222: qstr="select key from personal_www"
223:
224: results=SimpleSearch(db,qstr)
225:
226: keys=[]
227: for x in results:
228: if x[0]:
1.1.2.8 ! casties 229: keys.append(unicodify(x[0]))
1.1.2.1 dwinter 230:
231:
232: #first step detect new entries and conflicts
233: new=[]
234: conflicts={}
235:
236: for x in resultSet.iterkeys():
237:
238: if x not in keys:
239:
240: new.append(x)
241:
242: else:
243:
244: conflict,ret=checkForConflicts(db,resultSet[x],x)
245: if conflict:
246: conflicts[x]=ret
247:
248: return new,conflicts
249:
250: def importFMPXML(filename):
251: '''
252: method to import FileMaker XML file (FMPXMLRESULT format) into the table.
253: @param filename: xmlfile filename
254:
255: '''
256:
257: parser = sax.make_parser()
258: #The "consumer" is our own handler
259: consumer = xml_handler()
260: #Initialize Tenorsax with handler
261: handler = saxtools.tenorsax(consumer)
262: #Resulting tenorsax instance is the SAX handler
263: parser.setContentHandler(handler)
264: parser.setFeature(sax.handler.feature_namespaces, 1)
265: parser.parse(filename)
266: resultSet=consumer.result # xml now transformed into an dictionary
267:
268: return resultSet
269:
270:
271:
272: def checkForConflicts(cursor,dataSet,key):
273:
274: ret=[]
275: fields=",".join(checkFields)
276:
277: qstr="select %s from personal_www where key='%s'"%(fields,key)
278:
279:
280: sr=SimpleSearch(cursor,qstr)
281:
282: if not sr:
283: return True, None
284:
285: i=0
286: retValue=False
287:
288: for checkField in checkFields:
289: dbValueR=sr[0][i]
290: if dbValueR:
291: dbValue=dbValueR.decode('utf-8')
292: else:
293: dbValue=""
294:
1.1.2.7 casties 295: if checkField in dataSet:
296: setValue=dataSet[checkField]
297: logging.debug( " %s %s %s %s"%(repr(key),checkField,repr(dbValue),repr(setValue)))
298: if dbValue.strip().rstrip()!=setValue.lstrip().rstrip():
299: ret.append((checkField,dbValue,setValue))
300: retValue=True
301:
302: else:
303: logging.warning("unknown field %s in data file!"%checkField)
304:
1.1.2.1 dwinter 305: i+=1
306:
307: return retValue,ret
308:
309:
310: ##
311: ## public static int main()
312: ##
313:
314: if __name__ == "__main__":
315:
316:
317:
318: loglevel = logging.DEBUG
319:
320:
321: logging.basicConfig(level=loglevel,
322: format='%(asctime)s %(levelname)s %(message)s',
323: datefmt='%H:%M:%S')
324:
325: resultSet=importFMPXML(filename="/Users/dwinter/Desktop/personalwww.xml")
1.1.2.2 dwinter 326: news,conflicts=checkImport(dsn="dbname=personalwww host=xserve02a user=mysql password=e1nste1n", resultSet=resultSet)
1.1.2.1 dwinter 327:
328:
329: print "new"
330: print len(news),news
331: print "-----------"
332: print "conflicts"
333: print conflicts
334:
335: # update_fields = None
336: #
337: # if options.update_fields:
338: # update_fields = [string.strip(s) for s in options.update_fields.split(',')]
339: #
340: # parser = sax.make_parser()
341: # #The "consumer" is our own handler
342: # consumer = xml_handler(dsn=options.dsn,table=options.table,
343: # update_fields=update_fields,id_field=options.id_field,
344: # sync_mode=options.sync_mode)
345: # #Initialize Tenorsax with handler
346: # handler = saxtools.tenorsax(consumer)
347: # #Resulting tenorsax instance is the SAX handler
348: # parser.setContentHandler(handler)
349: # parser.setFeature(sax.handler.feature_namespaces, 1)
350: # parser.parse(options.filename)
351: #
352: #
353: # print "DONE!"
354:
355:
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>
![[BACK]](/cvs/cvsweb/icons/back.gif){kind=icon}
Return to updatePersonalWWW.py CVS log ![[TXT]](/cvs/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvs/cvsweb/icons/dir.gif){kind=icon}
Up to [Repository] / MPIWGWeb