--- MPIWGWeb/MPIWGProjects.py 2005/10/17 10:59:06 1.47.2.34 +++ MPIWGWeb/MPIWGProjects.py 2006/09/13 08:17:33 1.47.2.59 @@ -33,6 +33,7 @@ from Ft.Xml.XPath import Evaluate from Ft.Xml.XPath.Context import Context from Ft.Xml.Domlette import NonvalidatingReader,PrettyPrint, Print from Ft.Xml import EMPTY_NAMESPACE +import psycopg definedFields=['WEB_title','xdata_01','xdata_02','xdata_03','xdata_04','xdata_05','xdata_06','xdata_07','xdata_08','xdata_09','xdata_10','xdata_11','xdata_12','xdata_13','WEB_project_header','WEB_project_description','WEB_related_pub'] @@ -47,6 +48,30 @@ def getTextFromNode(nodename): rc = rc + node.data return rc + +def sortStopWordsF(self,xo,yo): + if not hasattr(self,'_v_stopWords'): + self._v_stopWords=self.stopwords_en.data.split("\n") + + x=str(xo[1]) + y=str(yo[1]) + + strx=x.split(" ") + stry=y.split(" ") + + for tmp in strx: + if tmp.lower() in self._v_stopWords: + del strx[strx.index(tmp)] + + for tmp in stry: + if tmp.lower() in self._v_stopWords: + del stry[stry.index(tmp)] + + return cmp(" ".join(strx)," ".join(stry)) + +def sortStopWords(self): + return lambda x,y : sortStopWordsF(self,x,y) + def sortF(x,y): try: return cmp(x[1],y[1]) @@ -270,6 +295,36 @@ class MPIWGRoot(ZSQLExtendFolder): folders=['MPIWGProject','Folder','ECHO_Navigation'] meta_type='MPIWGRoot' + def harvestProjects(self): + """harvest""" + folder="/tmp" + try: + os.mkdir("/tmp/harvest_MPIWG") + except: + pass + founds=self.ZopeFind(self.aq_parent.projects,obj_metatypes=['MPIWGProject'],search_sub=1) + for found in founds: + txt=found[1].harvest_page() + + if txt and (txt != ""): + name=found[0].replace("/","_") + fh=file("/tmp/harvest_MPIWG/"+name,"w") + fh.write(txt) + fh.close() + + def decode(self,str): + """decoder""" + if not str: + return "" + if type(str) is StringType: + try: + return str.decode('utf-8') + except: + return str.decode('latin-1') + else: + + return str + def versionHeaderEN(self): """version header text""" @@ -340,7 +395,23 @@ class MPIWGRoot(ZSQLExtendFolder): return True else: return False - + + def subNavStatic(self,obj): + """subnav" von self""" + def sortWeight(x,y): + x1=int(getattr(x[1],'weight','0')) + y1=int(getattr(y[1],'weight','0')) + return cmp(x1,y1) + + subs=self.ZopeFind(obj,obj_metatypes=['MPIWGTemplate','MPIWGLink']) + subret=[] + + for x in subs: + if not(x[1].title==""): + subret.append(x) + subret.sort(sortWeight) + return subret + def subNav(self,obj): """return subnav elemente""" def sortWeight(x,y): @@ -355,12 +426,14 @@ class MPIWGRoot(ZSQLExtendFolder): #suche die zweite ebene + if not obj.aq_parent.getId() in ['de','en']: obj=obj.aq_parent - + while not self.ZopeFind(self,obj_ids=[obj.getId()]): obj=obj.aq_parent + if hasattr(self,obj.getId()): subs=self.ZopeFind(getattr(self,obj.getId()),obj_metatypes=['MPIWGTemplate','MPIWGLink']) @@ -489,6 +562,22 @@ class MPIWGRoot(ZSQLExtendFolder): self.id=id self.title=title + def removeStopWords(self,xo): + """remove stop words from xo""" + if not hasattr(self,'_v_stopWords'): + self._v_stopWords=self.stopwords_en.data.split("\n") + + x=str(xo) + + strx=x.split(" ") + + for tmp in strx: + + if tmp.lower() in self._v_stopWords: + del strx[strx.index(tmp)] + + return " ".join(strx) + def urlQuote(self,str): """quote""" return urllib.quote(str) @@ -634,7 +723,7 @@ class MPIWGRoot(ZSQLExtendFolder): """test""" return self.getProjectsByFieldContent('xdata_09',['biology'])[0].absolute_url - def getContexts(self,childs=None,parents=None,depth=None,date=None): + def getContexts(self,childs=None,parents=None,depth=None,date=None,onlyActive=True): """childs alle childs, alle parents""" ret=[] @@ -649,15 +738,17 @@ class MPIWGRoot(ZSQLExtendFolder): if childs: for project in self.getProjectFields('xdata_05',sort='int',date=date): searchStr=childs+"(\..*)" - if re.match(searchStr,project[1]): - - if depth: - - if int(depth)>=len(project[1].split("."))-len(childs.split(".")): - + + if (onlyActive and project[0].isActiveProject()) or (not onlyActive): + if re.match(searchStr,project[1]): + + if depth: + + if int(depth)>=len(project[1].split("."))-len(childs.split(".")): + + ret.append(project) + else: ret.append(project) - else: - ret.append(project) return ret def getProjectFields(self,fieldName,date=None,folder=None,sort=None): @@ -685,6 +776,10 @@ class MPIWGRoot(ZSQLExtendFolder): if sort=="int": ret.sort(sortI) + elif sort=="stopWords": + + ret.sort(sortStopWords(self)) + else: ret.sort(sortF) @@ -762,13 +857,17 @@ class MPIWGRoot(ZSQLExtendFolder): def updateHomepages(self,RESPONSE=None): """ update""" - #FIXME: seite wird hinter apache mehrfach ausgefŸhrt + RESPONSE.setHeader('Content-type', 'text/html') RESPONSE.write("
\n") - url="http://itgroup.mpiwg-berlin.mpg.de:8050/FMPro?-db=personal-www&-format=-dso_xml&-lay=sql_export&-max=10000&-findall" + RESPONSE.write("Update Institutsbibliography\n") + ret=self.upDateSQL('personalwww.xml') + RESPONSE.write("done Insitutsbibliography:%s\n"%ret) + url="http://itgroup.mpiwg-berlin.mpg.de:8050/FMPro?-db=personal-www&-format=-dso_xml&-lay=sql_export&-max=20000&-findall" dom = NonvalidatingReader.parseUri(url) #fh=urllib.urlopen(url) #dom=xml.dom.minidom.parse(fh) + RESPONSE.write("got_xml_File\n") @@ -787,6 +886,7 @@ class MPIWGRoot(ZSQLExtendFolder): name=getTextFromNode(row.xpath('./dn:Name',explicitNss={'dn':'http://www.filemaker.com/fmpdsoresult'})[0]) vorname=getTextFromNode(row.xpath('./dn:Vorname',explicitNss={'dn':'http://www.filemaker.com/fmpdsoresult'})[0]) title=getTextFromNode(row.xpath('./dn:Title',explicitNss={'dn':'http://www.filemaker.com/fmpdsoresult'})[0]) + position=getTextFromNode(row.xpath('./dn:Position',explicitNss={'dn':'http://www.filemaker.com/fmpdsoresult'})[0]) e_mail=getTextFromNode(row.xpath('./dn:e_mail',explicitNss={'dn':'http://www.filemaker.com/fmpdsoresult'})[0]) e_mail_p=getTextFromNode(row.xpath('./dn:e_mail_p',explicitNss={'dn':'http://www.filemaker.com/fmpdsoresult'})[0]) date_from=getTextFromNode(row.xpath('./dn:Date_from',explicitNss={'dn':'http://www.filemaker.com/fmpdsoresult'})[0]) @@ -796,7 +896,7 @@ class MPIWGRoot(ZSQLExtendFolder): funded_by=getTextFromNode(row.xpath('./dn:funded_by',explicitNss={'dn':'http://www.filemaker.com/fmpdsoresult'})[0]) e_mail2=getTextFromNode(row.xpath('./dn:e_mail2',explicitNss={'dn':'http://www.filemaker.com/fmpdsoresult'})[0]) publish_the_data=getTextFromNode(row.xpath('./dn:publish_the_data',explicitNss={'dn':'http://www.filemaker.com/fmpdsoresult'})[0]) - + stay_at_mpiwg=getTextFromNode(row.xpath('./dn:stay_at_mpiwg',explicitNss={'dn':'http://www.filemaker.com/fmpdsoresult'})[0]) #cwNode=row.xpath('./dn:current_work.current',explicitNss={'dn':'http://www.filemaker.com/fmpdsoresult'})[0] cw=row.xpath('./dn:current_work/dn:DATA',explicitNss={'dn':'http://www.filemaker.com/fmpdsoresult'}) if cw: @@ -822,10 +922,10 @@ class MPIWGRoot(ZSQLExtendFolder): RESPONSE.write("new:%s
\n"%username.encode('utf-8')) obj=getattr(memberFolder,username) ret=obj.createNewDBEntry(publish_the_data,id,name, - vorname,username,title,e_mail, + vorname,username,title,position,e_mail, e_mail_p,date_from,date_to, abteilung,heimat_inst,funded_by, - e_mail2,txt,txt_p) + e_mail2,txt,txt_p,stay_at_mpiwg) RESPONSE.write("""%s
"""%ret[1].encode('utf-8')) except: RESPONSE.write("ERROR:%s %s %s
\n"%(username.encode('utf-8'),name.encode('utf-8'),vorname.encode('utf-8'))) @@ -849,17 +949,31 @@ class MPIWGRoot(ZSQLExtendFolder): obj=getattr(memberFolder,username) - obj.updateDBEntry(DBid=id,publish_the_data=publish_the_data, + done= obj.updateDBEntry(DBid=id,publish_the_data=publish_the_data, date_from=date_from, - date_to=date_to) + date_to=date_to,stay_at_mpiwg=stay_at_mpiwg,position=position,abteilung=abteilung) + if not done and (publish_the_data=='yes'): + + ret=obj.createNewDBEntry(publish_the_data,id,name, + vorname,username,title,position,e_mail, + e_mail_p,date_from,date_to, + abteilung,heimat_inst,funded_by, + e_mail2,txt,txt_p,stay_at_mpiwg) + if not ret[0]: - self.reindexCatalogs(RESPONSE) + RESPONSE.write("Error: %s
\n"%repr(ret[1])) + else: + RESPONSE.write("New: %s
\n"%repr(ret[1])) + + #TODO: reindexCatlogs and updatePublications wieder einbaue + #self.reindexCatalogs(RESPONSE) - self.updatePublicationDB() - - RESPONSE.write("