version 1.1.2.32, 2012/01/03 09:26:45
|
version 1.1.2.41, 2012/03/13 07:04:24
|
Line 21 from bibliography import *
|
Line 21 from bibliography import *
|
import time |
import time |
import xml.dom.minidom |
import xml.dom.minidom |
import sys |
import sys |
from Ft.Xml.XPath import Evaluate |
import transaction |
from Ft.Xml.XPath.Context import Context |
|
from Ft.Xml.Domlette import NonvalidatingReader,PrettyPrint, Print |
#from Ft.Xml.XPath import Evaluate |
from Ft.Xml import EMPTY_NAMESPACE |
#from Ft.Xml.XPath.Context import Context |
|
#from Ft.Xml.Domlette import NonvalidatingReader,PrettyPrint, Print |
|
#from Ft.Xml import EMPTY_NAMESPACE |
import copy |
import copy |
import updatePersonalWWW |
import updatePersonalWWW |
import MPIWGStaff |
import MPIWGStaff |
from MPIWGHelper import * |
from MPIWGHelper import * |
|
from BeautifulSoup import BeautifulSoup, Comment |
|
from ZODB import FileStorage, DB |
|
from ZEO import ClientStorage |
|
|
def sortWeight(x,y): |
def sortWeight(x,y): |
x1=int(getattr(x[1],'weight','0')) |
x1=int(getattr(x[1],'weight','0')) |
Line 40 def sortWeight(x,y):
|
Line 44 def sortWeight(x,y):
|
class MPIWGRoot(ZSQLExtendFolder): |
class MPIWGRoot(ZSQLExtendFolder): |
"""Stammordner fuer den Web-Server""" |
"""Stammordner fuer den Web-Server""" |
|
|
|
_v_harvestCache=None |
meta_type='MPIWGRoot' |
meta_type='MPIWGRoot' |
|
|
fieldLabels={'WEB_title':'WEB_Title', |
fieldLabels={'WEB_title':'WEB_Title', |
Line 78 class MPIWGRoot(ZSQLExtendFolder):
|
Line 83 class MPIWGRoot(ZSQLExtendFolder):
|
) |
) |
|
|
|
|
|
def getHarvestCachePort(self): |
|
return getattr(self,"harvestPort",29999) |
|
|
|
def getHarvestCacheServer(self): |
|
return getattr(self,"harvestServer","localhost") |
|
|
|
|
|
def getHarvestCache(self): |
|
logging.debug("CACHE:"+repr(self._v_harvestCache)) |
|
if self._v_harvestCache==None: |
|
#storage = FileStorage.FileStorage('/tmp/'+self.getId()+'test-filestorage.fs') |
|
addr = self.getHarvestCacheServer(), self.getHarvestCachePort() |
|
storage = ClientStorage.ClientStorage(addr) |
|
db = DB(storage) |
|
self._v_harvestDV=db |
|
self._v_harvestDV=db |
|
conn = db.open() |
|
dbroot = conn.root() |
|
if not dbroot.has_key('templates'): |
|
from BTrees.OOBTree import OOBTree |
|
dbroot['templates'] = OOBTree() |
|
|
|
self._v_harvestCache = dbroot['templates'] |
|
logging.debug("CACHE2:"+repr(self._v_harvestCache)) |
|
return self._v_harvestCache |
|
|
|
|
|
|
|
def __del__(self): |
|
if self._v_harvestCache!=None: |
|
self._v_harvestDV.close(); |
|
|
def getGetNeighbourhood(self,obj, wordStr, length=100,tagging=True): |
def getGetNeighbourhood(self,obj, wordStr, length=100,tagging=True): |
"""finde umgebung um die worte in wordStr, zurueckgegeben wird eine Array mit den Umgebungen von Fundstellen der Worte |
"""finde umgebung um die worte in wordStr, zurueckgegeben wird eine Array mit den Umgebungen von Fundstellen der Worte |
alle Tags werden entfernt, die Fundstellen werden mit <span class="found">XX</span> getaggt, die Umgebungen werden |
alle Tags werden entfernt, die Fundstellen werden mit <span class="found">XX</span> getaggt, die Umgebungen werden |
Line 119 class MPIWGRoot(ZSQLExtendFolder):
|
Line 156 class MPIWGRoot(ZSQLExtendFolder):
|
#if not words is ListType: |
#if not words is ListType: |
# words=[words] |
# words=[words] |
|
|
txt=obj.harvest_page() |
|
|
txtCache = self.en.getHarvestCache(); |
|
txt= txtCache.get(obj.absolute_url(),None) |
|
|
|
if txt==None: |
|
|
|
logging.debug("NO CACHE for: "+obj.absolute_url()) |
|
txt=obj.harvest_page(mode="slim") |
|
|
|
|
if not txt: |
if not txt: |
return ret |
return ret |
txt=re.sub("<.*?>", "", txt) # loesche alle Tags |
|
|
soup = BeautifulSoup(txt) |
|
|
|
comments = soup.findAll(text=lambda text:isinstance(text, Comment)) |
|
[comment.extract() for comment in comments] |
|
|
|
txt = ''.join(soup.findAll(text=True)) |
|
|
|
|
|
#txt=re.sub("<.*?>", "", txt) # loesche alle Tags |
for word in words: |
for word in words: |
word=re.sub("_"," ",word) # ersetze zurueck "_" durch " " |
word=re.sub("_"," ",word) # ersetze zurueck "_" durch " " |
pos=0 |
pos=0 |
Line 144 class MPIWGRoot(ZSQLExtendFolder):
|
Line 199 class MPIWGRoot(ZSQLExtendFolder):
|
y=max(ranges[nr][1],y) |
y=max(ranges[nr][1],y) |
|
|
str=txt[x:y] |
str=txt[x:y] |
|
if x!=0: #add dots if in the middle of text |
|
str="..."+str |
|
|
|
if y!=len(txt): #add dots if in the middle of text |
|
str=str+"..." |
|
|
|
|
|
|
if nr >=0: # word ist in einer schon gefunden Umgebung |
if nr >=0: # word ist in einer schon gefunden Umgebung |
ranges[nr]=(x,y) # neue Position der Umgebung |
ranges[nr]=(x,y) # neue Position der Umgebung |
Line 600 class MPIWGRoot(ZSQLExtendFolder):
|
Line 662 class MPIWGRoot(ZSQLExtendFolder):
|
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','changeMPIWGRootForm')).__of__(self) |
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','changeMPIWGRootForm')).__of__(self) |
return pt() |
return pt() |
|
|
def changeMPIWGRoot(self,title,connection_id,coneServiceURL,lang=None,autocommit=None,RESPONSE=None): |
def changeMPIWGRoot(self,title,connection_id,coneServiceURL,harvestPort,harvestServer,lang=None,autocommit=None,RESPONSE=None): |
"""change""" |
"""change""" |
self.title=title |
self.title=title |
self.connection_id=connection_id |
self.connection_id=connection_id |
#self.disciplineList=disciplineList |
#self.disciplineList=disciplineList |
#self.themesList=themesList |
#self.themesList=themesList |
self.coneServiceURL=coneServiceURL |
self.coneServiceURL=coneServiceURL |
|
self.harvestServer=harvestServer |
|
try: |
|
self.harvestPort=int(harvestPort) |
|
except: |
|
logging.error("couldn't change port!: no number:"+harvestPort) |
|
|
if lang is not None: |
if lang is not None: |
self.lang = lang |
self.lang = lang |
Line 653 class MPIWGRoot(ZSQLExtendFolder):
|
Line 720 class MPIWGRoot(ZSQLExtendFolder):
|
return ret |
return ret |
|
|
|
|
|
def getAllProjectsAndTagsAsCSV(self,archived=1,RESPONSE=None): |
|
"""alle projekte auch die nicht getaggten""" |
|
retList=[] |
|
headers=['projectId','sortingNumber','projectName','scholars','startedAt','completedAt','lastChangeThesaurusAt','lastChangeProjectAt','projectCreatedAt','persons','places','objects'] |
|
headers.extend(list(self.thesaurus.tags.keys())) |
|
retList.append("\t".join(headers)) |
|
if not hasattr(self,'thesaurus'): |
|
return "NON thesaurus (there have to be a MPIWGthesaurus object, with object ID thesaurus)" |
|
|
|
projectTags = self.thesaurus.getProjectsAndTags() |
|
for project in self.getProjectFields('WEB_title_or_short'): |
|
proj = project[0] |
|
p_name = project[1] |
|
retProj=[] |
|
#if (not proj.isArchivedProject() and archived==1) or (proj.isArchivedProject() and archived==2): |
|
retProj.append(self.utf8ify(proj.getId())) |
|
retProj.append(self.utf8ify(proj.getContent('xdata_05'))) |
|
retProj.append(self.utf8ify(p_name)) |
|
retProj.append(self.utf8ify(proj.getContent('xdata_01'))) |
|
retProj.append(self.utf8ify(proj.getStartedAt())) |
|
retProj.append(self.utf8ify(proj.getCompletedAt())) |
|
changeDate=self.thesaurus.lastChangeInThesaurus.get(proj.getId(),'') |
|
n = re.sub("[:\- ]","",str(changeDate)) |
|
retProj.append(n) |
|
retProj.append(self.utf8ify(getattr(proj,'creationTime','20050101000000'))) |
|
retProj.append("")#TODO: project created at |
|
retProj.append(";".join([person[1] for person in self.thesaurus.getPersonsFromProject(proj.getId())])) |
|
retProj.append(";".join([person[1] for person in self.thesaurus.getHistoricalPlacesFromProject(proj.getId())])) |
|
retProj.append(";".join([person[1] for person in self.thesaurus.getObjectsFromProject(proj.getId())])) |
|
retProj+=self.thesaurus.getTags(proj.getId(),projectTags) |
|
retList.append("\t".join(retProj)) |
|
|
|
if RESPONSE: |
|
|
|
RESPONSE.setHeader('Content-Disposition','attachment; filename="ProjectsAndTags.tsv"') |
|
RESPONSE.setHeader('Content-Type', "application/octet-stream") |
|
|
|
return "\n".join(retList); |
|
|
|
|
|
|
|
|
def getProjectFields(self,fieldName,date=None,folder=None,sort=None): |
def getProjectFields(self,fieldName,date=None,folder=None,sort=None): |
"""getListofFieldNames""" |
"""getListofFieldNames""" |
ret=[] |
ret=[] |
Line 961 class MPIWGRoot(ZSQLExtendFolder):
|
Line 1070 class MPIWGRoot(ZSQLExtendFolder):
|
|
|
|
|
|
|
|
|
if RESPONSE: |
if RESPONSE: |
RESPONSE.redirect('manage_main') |
RESPONSE.redirect('manage_main') |
|
|
Line 1232 class MPIWGRoot(ZSQLExtendFolder):
|
Line 1342 class MPIWGRoot(ZSQLExtendFolder):
|
"""give tuple member /projects""" |
"""give tuple member /projects""" |
ret=[] |
ret=[] |
members=self.getAllMembers() |
members=self.getAllMembers() |
logging.error("X %s"%repr(members)) |
logging.debug("X %s"%repr(members)) |
#return str(members) |
#return str(members) |
for x in members: |
for x in members: |
logging.error("X %s"%repr(x)) |
#logging.debug("X %s"%repr(x)) |
projects=self.getProjectsOfMember(key=x[1],date=date) |
projects=self.getProjectsOfMember(key=x[1],date=date) |
if len(projects)>0: |
if len(projects)>0: |
ret.append((x[0],projects)) |
ret.append((x[0],projects)) |
Line 1259 class MPIWGRoot(ZSQLExtendFolder):
|
Line 1369 class MPIWGRoot(ZSQLExtendFolder):
|
|
|
ret=[] |
ret=[] |
if key: |
if key: |
|
logging.debug("MPIWGROOT (getProjectsOfMember):"+key) |
proj=self.ProjectCatalog({'getPersonKeyList':utf8ify(key)}) |
proj=self.ProjectCatalog({'getPersonKeyList':utf8ify(key)}) |
else: |
else: |
return ret # key muss definiert sein |
return ret # key muss definiert sein |
|
|
|
#logging.debug("MPIWGROOT (getProjectsOfMember):"+repr(proj)) |
if proj: |
if proj: |
proj2=[] |
proj2=[] |
for x in proj: |
for x in proj: |
Line 1504 class MPIWGRoot(ZSQLExtendFolder):
|
Line 1615 class MPIWGRoot(ZSQLExtendFolder):
|
return "done" |
return "done" |
|
|
|
|
|
def sortResults(self,results): |
|
"""search the catalog and give results back sorted by meta_type""" |
|
ret = {} |
|
logging.debug(results()) |
|
for result in results(): |
|
metaType = result.meta_type |
|
resultList= ret.get(metaType,[]) |
|
resultList.append(result) |
|
ret[metaType]=resultList |
|
|
|
logging.debug(ret) |
|
return ret |
|
|
|
|
def manage_addMPIWGRootForm(self): |
def manage_addMPIWGRootForm(self): |
"""form for adding the root""" |
"""form for adding the root""" |