from Products.PageTemplates.PageTemplateFile import PageTemplateFile
from Products.PageTemplates.PageTemplate import PageTemplate
from Products.PageTemplates.ZopePageTemplate import ZopePageTemplate
from Products.ZSQLExtend.ZSQLExtend import ZSQLExtendFolder
from Products.ZCatalog.CatalogPathAwareness import CatalogAware
from OFS.Image import Image
from Globals import package_home
import urllib
import MPIWGStaff
import string
import re
import os
from types import *
import logging
import xmlhelper # Methoden zur Verwaltung der projekt xml
from OFS.SimpleItem import SimpleItem
from OFS.Folder import Folder
from Products.ZSQLMethods.SQL import SQLConnectionIDs
from AccessControl import ClassSecurityInfo
from bibliography import *
import time
import xml.dom.minidom
import sys
import transaction

#from Ft.Xml.XPath import Evaluate
#from Ft.Xml.XPath.Context import Context
#from Ft.Xml.Domlette import NonvalidatingReader,PrettyPrint, Print
#from Ft.Xml import EMPTY_NAMESPACE
import copy
import updatePersonalWWW
import MPIWGStaff
from MPIWGHelper import *
from BeautifulSoup import BeautifulSoup, Comment
from ZODB import FileStorage, DB
from ZEO import ClientStorage

def sortWeight(x,y):
    x1=int(getattr(x[1],'weight','0'))
    y1=int(getattr(y[1],'weight','0'))
    return cmp(x1,y1)


class MPIWGRoot(ZSQLExtendFolder):
    """Stammordner fuer den Web-Server"""

    _v_harvestCache=None
    meta_type='MPIWGRoot'

    fieldLabels={'WEB_title':'WEB_Title',
                 'xdata_01':'Responsible Scientists',
                 'xdata_02':'Department',
                 'xdata_03':'Historical Persons',
                 'xdata_04':'Time period',
                 'xdata_05':'Sorting number',
                 'xdata_06':'Keywords',
                 'xdata_07':'Short title',
                 'xdata_08':'Other involved scholars' ,
                 'xdata_09':'Disciplines',
                 'xdata_10':'Themes',
                 'xdata_11':'Object Digitallibrary',
                 'xdata_12':'Cooperation partners',
                 'xdata_13':'Funding institutions',
                 'WEB_project_header':'WEB_project_header',
                 'WEB_project_description':'WEB_project_description',
                 'WEB_related_pub':'WEB_related_pub'}
    
    # (is this used?)
    folders=['MPIWGProject','Folder','ECHO_Navigation']
    # language of this instance
    lang = 'en'
    # types of objects that show up in navigation
    nav_meta_types = ['MPIWGTemplate','MPIWGLink','MPIWGFolder']

    manage_options = Folder.manage_options+(
        {'label':'Update personal homepages','action':'updatePersonalwww_html'},
        {'label':'Reindex catalogs','action':'reindexCatalogs'},
        {'label':'Main config','action':'changeMPIWGRootForm'},
        {'label':'add e-mails','action':'showNewDBEntries'},
        #{'label':'update the institutsbibliography','action':'updateInstitutsbiliography'},
        #{'label':'Edit Historical Persons','action':'editHistoricalPersonsForm'},
        #{'label':'Store Historical Persons','action':'storeHistoricalPersons'},
        )
    

    def getHarvestCachePort(self):
        return getattr(self,"harvestPort",29999)
    
    def getHarvestCacheServer(self):
        return getattr(self,"harvestServer","localhost")
   
    
    def getHarvestCache(self):
        logging.debug("CACHE:"+repr(self._v_harvestCache))
        if self._v_harvestCache==None:
             #storage = FileStorage.FileStorage('/tmp/'+self.getId()+'test-filestorage.fs')
             addr = self.getHarvestCacheServer(), self.getHarvestCachePort()
             storage = ClientStorage.ClientStorage(addr)
             db = DB(storage)
             self._v_harvestDV=db
             self._v_harvestDV=db
             conn = db.open()
             dbroot = conn.root()
             if not dbroot.has_key('templates'):
                 from BTrees.OOBTree import OOBTree
                 dbroot['templates'] = OOBTree()
                 
             self._v_harvestCache = dbroot['templates']
        logging.debug("CACHE2:"+repr(self._v_harvestCache))            
        return self._v_harvestCache
    
    
    
    def __del__(self):
         if self._v_harvestCache!=None:
             self._v_harvestDV.close();
             
    def getGetNeighbourhood(self,obj, wordStr, length=100,tagging=True):
        """finde umgebung um die worte in wordStr, zurueckgegeben wird eine Array mit den Umgebungen von Fundstellen der Worte
        alle Tags werden entfernt, die Fundstellen werden mit <span class="found">XX</span> getaggt, die Umgebungen werden 
        case insensitive gesucht
        @param wordStr: string mit Worten getrennt durch Leerzeichen, Phrasen sind mit " gekennzeichnet
                        "eine phrase", "*"  bezeichnet wildcards und wird ignoriert"
        @param length: optional, default wert 100, 2*length ist die groesse der Umgebung
        @param tagging: optional default wert true, kein span tag wird erzweugt falls tag=false
        """
        
        ret=[] # nimmt das Array auf, dass spaeter zurueckgegeben wird
        ranges=[] #Array mit tupeln x,y wobei x die Position des Anfang und y des Endes der i-ten Umgebung angiebt
        
        wordStr=wordStr.lstrip().rstrip()
        
        def isInRanges(nr,length):
            """test ob eine gegeben Position nr schon irgendwo in einer Umgebung ist, gibt den Index des ersten Wertes aus ranges zurueck, 
            -1, wenn kein Treffer
            
            @param nr: Position die geprueft werden soll
            @param length: Laenge des Wortes das geprueft werden soll
            """
            for x in ranges:
                if (x[0]<=nr) and (nr < (x[1]-length)):
                    return ranges.index(x)
            return -1
                
        # deal with phrases, in Phrasen werden die Leerzeichen durch "_" ersetzt.
        def rep_empty(str):
            x= re.sub(" ","_",str.group(0))
            return re.sub("\"","",x)
            
        wordStr=re.sub("\".*?\"", rep_empty,wordStr)#ersetze leerzeichen in " " durch "_" und loesche "
        
        #deal with wildcards, for our purposes it is enough to delete the wildcard 
        wordStr=wordStr.replace("*","")
        
        words=wordStr.split(" ")
        #if not words is ListType:
        #   words=[words]
        
        
        txtCache = self.en.getHarvestCache();
        txt=  txtCache.get(obj.absolute_url(),None)
       
        if txt==None:
            
            logging.debug("NO CACHE for: "+obj.absolute_url())
            txt=obj.harvest_page(mode="slim")
        
        
        if not txt:
            return ret
        
        soup = BeautifulSoup(txt)
        
        comments = soup.findAll(text=lambda text:isinstance(text, Comment))
        [comment.extract() for comment in comments]
        
        txt = ''.join(soup.findAll(text=True))
        
        
        #txt=re.sub("<.*?>", "", txt) # loesche alle Tags
        for word in words:
            word=re.sub("_"," ",word) # ersetze zurueck "_" durch " "
            pos=0
            
            n=txt.lower().count(word.lower()) # wie oft tritt das Wort auf

            for i in range(n):
                pos=txt.lower().find(word.lower(),pos)

                if pos > 0:
                    x=max(0,pos-length)
                    y=min(len(txt),pos+length)
                  
                    
                    #is word already in one of the results
                    nr=isInRanges(pos,len(word))
                    if nr >=0:# word ist in einer schon gefunden Umgebung, dann vergroessere diese
                        x=min(ranges[nr][0],x)
                        y=max(ranges[nr][1],y)
              
                    str=txt[x:y]
                    if x!=0: #add dots if in the middle of text
                        str="..."+str
                    
                    if y!=len(txt): #add dots if in the middle of text
                        str=str+"..."
                        
                        
                
                    if nr >=0: # word ist in einer schon gefunden Umgebung
                        ranges[nr]=(x,y) # neue Position der Umgebung

                        ret[nr]=str # neue Umgebung
                    else: # andernfalls neue Umgebung hinzufuegen
                        ranges.append((x,y))

                        ret.append(str)
                    
                    pos=pos+len(word)
                else:
                    break;
                
        # now highlight everything        
        if tagging:
            for x in range(len(ret)):
                for word in words:
                    repl=re.compile(word,re.IGNORECASE)
                    ret[x]=repl.sub(""" <span class="found">%s</span>"""%word.upper(),ret[x])

        return ret
    def copyAllImagesToMargin(self):
        """tranformiere alle Bilder in die Margins"""
        projects=self.getTree()
        ret=""
        for project in projects:
            proj=project[3]
            try:
                persons=proj.copyImageToMargin();
            except:
                logging.error("Cannnot do: %s"%repr(project))
                
    def transformProjectsToId(self):
        """trnasformiere zu ID, Hilfsfunktion die die alten Templates analysiert und mit der neuen Liste
        verantwortlicher Personen versieht"""
        projects=self.getTree()
        ret=""
        for project in projects:
            
            proj=project[3]
            persons=proj.identifyNames(proj.getContent('xdata_01'))
            if not hasattr(proj,'responsibleScientistsList'):
                        proj.responsibleScientistsList=[]
                        
            for person in persons.items():
              
                if len(person[1]) >1: #nicht eindeutig
                    ret+="nicht eindeutig ---  %s:  %s\n"%(proj.getId(),person[0])
                    
                elif len(person[1]) ==0: #kein eintrage
                    ret+="kein eintrag---  %s:  %s\n"%(proj.getId(),person[0])
                    proj.responsibleScientistsList.append((person[0],""))
                else:           
                    proj.responsibleScientistsList.append((person[0],person[1][0].getObject().getKey()))
        
        return ret
          
                
    def harvestProjects(self):
        """harvest"""
        folder="/tmp"
        try:
            os.mkdir("/tmp/harvest_MPIWG")
        except:
            pass
        founds=self.ZopeFind(self.aq_parent.projects,obj_metatypes=['MPIWGProject'],search_sub=1)
        for found in founds:
            txt=found[1].harvest_page()
        
            if txt and (txt != ""):
                name=found[0].replace("/","_")
                fh=file("/tmp/harvest_MPIWG/"+name,"w")
                fh.write(txt)
                fh.close()
                
    def decode(self,str):
        """decoder"""

        if not str:
            return ""
        if type(str) is StringType:
            try:            
                return str.decode('utf-8')
            except:
                return str.decode('latin-1')
        else:
            return str


    def getat(self,array,idx=0,default=None):
        """return array element idx or default (but no exception)"""
        if len(array) <= idx:
            return default
        else:
            return array[idx]
        
    def getLang(self):
        """returns the default language"""
        return self.lang

    def browserCheck(self):
        """check the browsers request to find out the browser type"""
        bt = {}
        ua = self.REQUEST.get_header("HTTP_USER_AGENT")
        bt['ua'] = ua
        bt['isIE'] = False
        bt['isN4'] = False
        if string.find(ua, 'MSIE') > -1:
            bt['isIE'] = True
        else:
            bt['isN4'] = (string.find(ua, 'Mozilla/4.') > -1)

        try:
            nav = ua[string.find(ua, '('):]
            ie = string.split(nav, "; ")[1]
            if string.find(ie, "MSIE") > -1:
                bt['versIE'] = string.split(ie, " ")[1]
        except: pass

        bt['isMac'] = string.find(ua, 'Macintosh') > -1
        bt['isWin'] = string.find(ua, 'Windows') > -1
        bt['isIEWin'] = bt['isIE'] and bt['isWin']
        bt['isIEMac'] = bt['isIE'] and bt['isMac']
        bt['staticHTML'] = False

        return bt


    def versionHeaderEN(self):
        """version header text"""
        
        date= self.REQUEST.get('date',None)
        if date:
            txt="""<h2>This pages shows the project which existed at %s</h2>"""%str(date)
            return txt
        return ""

    def versionHeaderDE(self):
        """version header text"""
        date= self.REQUEST.get('date',None)
        if date:
            txt="""<h2>Auf dieser Seite finden Sie die Projekte mit Stand vom %s</h2>"""%str(date)
        return ""
    
        
    def createOrUpdateId_raw(self):
        """create sequence to create ids for bibliography"""
        debug=None
        #suche groesste existierende id
        founds=self.ZSQLQuery("select id from bibliography")
        
        if founds:
            ids=[int(x.id[1:]) for x in founds]
            maximum=max(ids)
            
            id_raw=self.ZSQLQuery("select nextval('id_raw')",debug=debug)
            
            if id_raw:
                self.ZSQLQuery("drop sequence id_raw",debug=debug)
            
            self.ZSQLQuery("create sequence id_raw start %i"%(maximum+1),debug=debug)
        
    
    def queryLink(self,link):
        """append querystring to the link"""
        return "%s?%s"%(link,self.REQUEST.get('QUERY_STRING',''))

    def getKategory(self,url):
        """kategorie"""
        splitted=url.split("/")
        return splitted[4]

    def generateUrlProject(self,url,project=None):
        """erzeuge aus absoluter url, relative des Projektes"""
        if project:
            splitted=url.split("/")
            length=len(splitted)
            short=splitted[length-2:length]
            
            base=self.REQUEST['URL3']+"/"+"/".join(short)

        else:
            findPart=url.find("/projects/")
            base=self.REQUEST['URL1']+"/"+url[findPart:]

                
        return base
    
    def isNewCapital(self,text=None,reset=None):
        
        if text:
            text=text.upper()
        if reset:
            self.REQUEST['capital']="A"
            return True
        else:
            if len(text)>0 and not (text[0]==self.REQUEST['capital']):
                self.REQUEST['capital']=text[0]
                return True
            else:
                return False
    
    def subNavStatic(self,obj):
        """subnav" von self"""
        subs=self.ZopeFind(obj,obj_metatypes=['MPIWGTemplate','MPIWGLink'])
        subret=[]

        for x in subs:
            if not(x[1].title==""):
                subret.append(x)
        subret.sort(sortWeight)
        return subret
    
    def subNav(self,obj):
        """return sub-navigation elements i.e. below sections"""
        # get section -> parent should be MPIWGRoot
        p = obj
        sec = None
        # descend parents to the root (and remember the last id)
        while p is not None and p.meta_type != 'MPIWGRoot':
            sec = p
            p = p.aq_parent
    
        subsecs = sec.objectItems(self.nav_meta_types)
        subsecs = [s for s in subsecs if s[1].title != ""]
        subsecs.sort(sortWeight)
        return subsecs
                
    def isType(self,object,meta_type):
        """teste ob ein object vom meta_type ist."""
        return (object.meta_type==meta_type)
    
    def isActive(self,name):
        """teste ob subnavigation aktiv"""
        for part in self.REQUEST['URL'].split("/"):
            if part==name:
                return True
        return False
        
    
    def getSections(self):
        """returns a list of all sections i.e. top-level MPIWGFolders"""
        secs = self.objectItems(['MPIWGFolder'])
        secs.sort(sortWeight)
        #logging.debug("root: %s secs: %s"%(repr(self.absolute_url()), repr(secs)))
        # return pure list of objects
        return [s[1] for s in secs]

    def getSectionStyle(self, name, style=""):
        """returns a string with the given style + '-sel' if the current section == name"""
        if self.getSection() == name:
            return style + '-sel'
        else:
            return style    

    def getFeatures(self, num=None):
        """returns a list of the last num Features"""
        dir = getattr(self, 'features')
        features = dir.objectItems(['MPIWGFeature'])
        features.sort(sortWeight)
        if num is not None:
            # take only the last num elements
            features = features[-num:]
        # return pure list of objects
        return [f[1] for f in features]


    def getMPIWGRoot(self):
        """returns the MPIWG root"""
        return self

    def MPIWGrootURL(self):
        """returns the URL to the root"""
        return self.absolute_url()
        
    def upDateSQL(self,fileName):
        """updates SQL databases using fm.jar"""
        fmJarPath=os.path.join(package_home(globals()), 'updateSQL/fm.jar')
        xmlPath=os.path.join(package_home(globals()), "updateSQL/%s"%fileName)
        logger("MPIWG Web",logging.INFO,"java -classpath %s -Djava.awt.headless=true Convert %s"%(fmJarPath,xmlPath))
        ret=os.popen("java -classpath %s -Djava.awt.headless=true Convert %s"%(fmJarPath,xmlPath),"r").read()
        logger("MPIWG Web",logging.INFO,"result convert: %s"%ret)
        return 1
    
    def patchProjects(self,RESPONSE):
        """patch"""
        projects=self.ZopeFind(self.projects,obj_metatypes=['MPIWGProject'])
        for project in projects:
                tmp=project[1].WEB_project_description[0].replace("/CD/projects/","")[0:]
                setattr(project[1],'WEB_project_description',[tmp[0:]])
                RESPONSE.write("<p>%s</p>\n"%project[0])
            
    def replaceNotEmpty(self,format,field):
        """replace not empty"""
        if field and (not field.lstrip()==''):
            return self.decode(format%field)
        else:
            return ""
        

    def isActiveMember(self,key):
        """tested ob Mitarbeiter key ist aktiv"""
        key=utf8ify(key)
        ret=self.getat(self.ZSQLInlineSearch(_table='personal_www',
                                            _op_key='eq',key=key,
                                            _op_publish_the_data='eq',
                                            publish_the_data='yes'))
        
        logging.info("ACTIVE_MEMBER  %s"%ret)
        if ret:
            return True
        else:
            return False
        
    def isActual(self,project):
        """checke if project is actual"""
        actualTime=time.localtime()
        
        if hasattr(project,'getObject'): #obj ist aus einer catalogTrefferList
            obj=project.getObject()
        else:
            obj=project
            
        if getattr(obj,'archiveTime',actualTime)< actualTime:
            return False
        else:
            return True
        
    def redirectIndex_html(self,request):
        #return request['URL1']+'/index_html'
        
        return urllib.urlopen(request['URL1']+'/index_html').read()

    
    def formatBibliography(self,here,found):
        """format"""
        return formatBibliography(here,found)
    
    def getValue(self,fieldStr):
        """Inhalt des Feldes"""
        
        if type(fieldStr)==StringType:
            field=fieldStr
        else:
            field=fieldStr[0]
        try:
            if field[len(field)-1]==";":
                field=field[0:len(field)-1]
        except:

            """nothing"""
        field=re.sub(r';([^\s])','; \g<1>',field)
        return field.encode('utf-8')


    
    def sortedNames(self,list):
        """sort names"""

        def sortLastName(x_c,y_c):
            try:
                x=urllib.unquote(x_c).encode('utf-8','ignore')
            except:
                x=urllib.unquote(x_c)

            try:
                y=urllib.unquote(y_c).encode('utf-8','ignore')
            except:
                x=urllib.unquote(y_c)
                

            
            try:
                last_x=x.split()[len(x.split())-1]
                last_y=y.split()[len(y.split())-1]

            except:

                last_x=""
                last_y=""
            
            
            
            if last_x<last_y:
                return 1
            elif last_x>last_y:
                return -1
            else:
                return 0
            
        list.sort(sortLastName)
        list.reverse()
        
        return list
    
    def __init__(self, id, title):
        """init"""
        self.id=id
        self.title=title

    def removeStopWords(self,xo):
        """remove stop words from xo"""
        if not hasattr(self,'_v_stopWords'):
            self._v_stopWords=self.stopwords_en.data.split("\n")
    
        x=str(xo)
    
        strx=x.split(" ")
  
        for tmp in strx:
     
            if tmp.lower() in self._v_stopWords:
                del strx[strx.index(tmp)]

        return " ".join(strx)
    
    def urlQuote(self,str):
        """quote"""
        return urllib.quote(str)

    def urlUnQuote(self,str):
        """quote"""
        return urllib.unquote(str)
    
        

    def getProjectsByFieldContent(self,fieldName,fieldContentsEntry, date=None):
        """gib alle Projekte aus mit Value von field mit fieldName enthaelt ein Element der Liste fieldContents"""
        def sort(x,y):
                return cmp(x.WEB_title[0],y.WEB_title[0])

        if type(fieldContentsEntry) is StringType:
            fieldContentsTmp=[fieldContentsEntry]
        else:
            fieldContentsTmp=fieldContentsEntry

        fieldContents=[]
        for x in fieldContentsTmp:
            fieldContents.append(" AND ".join(x.split()))
        projects=self.ProjectCatalog({fieldName:string.join(fieldContents,' AND')})
        #print projects
        #ret=[x for x in projects]
        ret=[]
        for x in projects:
            obj=x.getObject()
            obj=obj.getActualVersion(date)
            if obj and (not getattr(obj,'invisible',None)):
                #if not (x in ret):
                    ret.append(x)

        ret.sort(sort)
        return ret

    def changeMPIWGRootForm(self):
        """edit"""
        pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','changeMPIWGRootForm')).__of__(self)
        return pt()

    def changeMPIWGRoot(self,title,connection_id,coneServiceURL,harvestPort,harvestServer,lang=None,autocommit=None,RESPONSE=None):
        """change"""
        self.title=title
        self.connection_id=connection_id
        #self.disciplineList=disciplineList
        #self.themesList=themesList
        self.coneServiceURL=coneServiceURL
        self.harvestServer=harvestServer
        try:
            self.harvestPort=int(harvestPort)
        except:
            logging.error("couldn't change port!: no number:"+harvestPort)
        
        if lang is not None:
            self.lang = lang

        self.autocommit = (autocommit == "on")
        if RESPONSE is not None:
            RESPONSE.redirect('manage_main')


    def getContexts(self,childs=None,parents=None,depth=None,date=None,onlyActive=True):
        """childs alle childs, alle parents"""
        ret=[]
    
        if parents:
            pnums=parents.split(".")
            while len(pnums) > 1:
                pnums.pop()
                parentId=string.join(pnums,".")
        
                for project in self.getProjectFields('xdata_05',sort='int',date=date):
                    if project[1]==parentId:
                        ret.append(project)
                
                if (depth is not None) and (len(ret) >= depth):
                    break

        if childs:
            for project in self.getProjectFields('xdata_05',sort='int',date=date):
                searchStr=childs+"(\..*)"
               
                if (onlyActive and project[0].isActiveProject()) or (not onlyActive):
                    if re.match(searchStr,project[1]):
                        
                        if depth:
    
                            if int(depth)>=len(project[1].split("."))-len(childs.split(".")):
                        
                                ret.append(project)
                        else:
                            ret.append(project)
        
        #logging.debug("getContexts: childs=%s parents=%s depth=%s => %s"%(childs,parents,depth,repr(ret)))
     
        return ret


    def getAllProjectsAndTagsAsCSV(self,archived=1,RESPONSE=None):
        """alle projekte auch die nicht getaggten"""
        retList=[]
        headers=['projectId','sortingNumber','projectName','scholars','startedAt','completedAt','lastChangeThesaurusAt','lastChangeProjectAt','projectCreatedAt','persons','places','objects']
        headers.extend(list(self.thesaurus.tags.keys()))
        retList.append("\t".join(headers))
        if not hasattr(self,'thesaurus'):
            return "NON thesaurus (there have to be a MPIWGthesaurus object, with object ID thesaurus)"
        
        projectTags = self.thesaurus.getProjectsAndTags()
        for project in self.getProjectFields('WEB_title_or_short'):
            proj = project[0]
            p_name = project[1]
            retProj=[]
            #if (not proj.isArchivedProject() and archived==1) or (proj.isArchivedProject() and archived==2):
            retProj.append(self.utf8ify(proj.getId()))
            retProj.append(self.utf8ify(proj.getContent('xdata_05')))
            retProj.append(self.utf8ify(p_name))  
            retProj.append(self.utf8ify(proj.getContent('xdata_01')))
            retProj.append(self.utf8ify(proj.getStartedAt()))
            retProj.append(self.utf8ify(proj.getCompletedAt()))
            changeDate=self.thesaurus.lastChangeInThesaurus.get(proj.getId(),'') 
            n = re.sub("[:\- ]","",str(changeDate))
            retProj.append(n)
            retProj.append(self.utf8ify(getattr(proj,'creationTime','20050101000000')))  
            retProj.append("")#TODO: project created at   
            retProj.append(";".join([person[1] for person in self.thesaurus.getPersonsFromProject(proj.getId())]))
            retProj.append(";".join([person[1] for person in self.thesaurus.getHistoricalPlacesFromProject(proj.getId())]))
            retProj.append(";".join([person[1] for person in self.thesaurus.getObjectsFromProject(proj.getId())]))
            retProj+=self.thesaurus.getTags(proj.getId(),projectTags)
            retList.append("\t".join(retProj))
        
        if RESPONSE:
            
            RESPONSE.setHeader('Content-Disposition','attachment; filename="ProjectsAndTags.tsv"')
            RESPONSE.setHeader('Content-Type', "application/octet-stream")
      
        return "\n".join(retList);
    
    
    
  
    def getProjectFields(self,fieldName,date=None,folder=None,sort=None):
        """getListofFieldNames"""
        ret=[]
    
        objects=self.ZopeFind(self.projects,obj_metatypes=['MPIWGProject'],search_sub=0)

                
        for object in objects:
            obj=object[1]
            obj=obj.getActualVersion(date)
            if obj and (not getattr(obj,'invisible',None)):
                if fieldName=="WEB_title_or_short":

                    if len(obj.getContent('xdata_07'))<3: # hack weil z.Z. manchmal noch ein Trennzeichen ; oder , im Feld statt leer
                        fieldNameTmp="WEB_title"
                    else:
                        fieldNameTmp="xdata_07"
                else:
                    fieldNameTmp=fieldName

                ret.append((obj,obj.getContent(fieldNameTmp)))

        
        if sort=="int":
            ret.sort(sortI)
        elif sort=="stopWords":
 
            ret.sort(sortStopWords(self))
            
        else:
            ret.sort(sortF)
        
        return ret

    def showNewProjects(self):
        projects=[]
        for objs in self.getProjectFields('WEB_title_or_short'): # Get all Projets
            if objs[0].xdata_05 and (objs[0].xdata_05[0] == ""):
                
                projects.append(objs)
                
        return projects
    
        
    def updatePublicationDB(self,personId=None):
        """updates the publication db, i.e. copy year and type into the main table"""
        
        if personId:
            founds = self.ZSQLInlineSearch(_table="publications",key_main=personId)
        else:
            founds = self.ZSQLInlineSearch(_table="publications")
            
        for found in founds:
                        
            if found.id_institutsbibliographie and (not found.id_institutsbibliographie =="") and (not found.id_institutsbibliographie =="0"):
                
                entries = self.ZSQLInlineSearch(_table="institutsbiblio",id=found.id_institutsbibliographie)
                for entry in entries:
                    self.ZSQLChange(_table='publications',_identify='oid=%s' % found.oid,year=entry.year,referencetype=entry.reference_type)
                    
            if found.id_gen_bib and (not found.id_gen_bib ==""):
                entries = self.ZSQLInlineSearch(_table="bibliography",id=found.id_gen_bib)
                for entry in entries:
                    self.ZSQLChange(_table='publications',_identify='oid=%s' % found.oid,year=entry.year,referencetype=entry.reference_type)
                    
        return True        
    
    def showNewDBEntries(self):
        """zeige neue Eintraege in der Datenbank ohne e-mail adressen bzw. fuer die noch kein Object angelegt wurde"""
        
        qstr="select * from personal_www where web_object_created='no' and not key=''"
        res=self.ZSQLQuery(qstr)
        
        pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','showNewDBEntries.zpt')).__of__(self)
        return pt(newEntries=res)
   
    def createNewStaffObjects(self,RESPONSE):
        """create new staff object"""
        
        memberFolder=getattr(self,'members')
        args=self.REQUEST.form
        arg_k=args.keys()
        arg_k.remove("submit")
        ret=""
        for key in arg_k:
            k=self.urlUnQuote(key)
          
            qstr="select * from personal_www where key=%s"%self.ZSQLQuote(k)
            res=self.ZSQLQuery(qstr)[0]
            if args[key]!="": #email-adresse wurde eingetragen
                #create the object
                e_mail=args[key]
                try:
                    newObj=MPIWGStaff.MPIWGStaff(e_mail,res.last_name,res.first_name,k)
                    memberFolder._setObject(e_mail,newObj)
                    obj=getattr(memberFolder,e_mail)
                    obj.reindex_object()
                    ret+="Created %s \n"%e_mail
                    created=True
                except:
                    msg="Cannot create new user %s (%s %s)"%(e_mail,sys.exc_info()[0],sys.exc_info()[1])
                    logging.error(msg)
                    ret+=msg+"\n"
                    created=False
                
                if created:
                    qstr="update personal_www set web_object_created='yes',e_mail='%s@mpiwg-berlin.mpg.de' where key=%s"%(e_mail,self.ZSQLQuote(k))
                    self.ZSQLQuery(qstr)
        
        return ret
                   
        
    def generateNewPersonEntry(self,data):
        """generate a new person entry for data, neue personen werden zunaechst nur in der datenbank angelegt """
        
        #memberFolder=getattr(self,'members')
        #create the object
        
#        try:
#            newObj=MPIWGStaff.MPIWGStaff(urllib.quote(data['key']),data['last_name'].encode('utf-8'),data['first_name'].encode('utf-8')) 
#            memberFolder._setObject(urllib.quote(data['key']),newObj)
#        except:
#            return False, "Cannot create new user %s (%s %s)"%(data['key'],sys.exc_info()[0],sys.exc_info()[1])
#        
        
        #create the new entry in the database
        
        
        result,msg=MPIWGStaff.createNewDBEntry(self,data['publish_the_data'],data['key'],data['last_name'],
                                  data['first_name'],data['titles_new'],data['status'],"",
                                  "",data['date_from'],data['date_to'],
                                  data['department'],'',data['funded_by'],
                                  data['e_mail2'],data['current_work'],"yes",data['date_stay_at_mpiwg'],data['group'],"no",data['current_work'])
        
        return result,msg
 
    def updatePersonEntry(self,data,ignoreEntries=None):
        """update an person entry from data. but ignore all fields in ignore Entries"""
        if ignoreEntries is None:
            ignoreEntries = []
            
        #ignoreEntries.append('current_work') # TODO:updatecurrent work
        logging.debug("updatePersonEntry: data=%s ignoreEntries=%s"%(repr(data),repr(ignoreEntries)))
        
        if data['date_to']=="": # wenn date_to leer
             data['date_to']="date_none"
        
        if data['date_from']=="": # wenn date_fromleer
             data['date_from']="date_none"
        msg=""
   
        
        #eintragen
         
        columns=data.keys()
        for x in ignoreEntries:
            logging.debug("updatePersonEntry: ignoring %s"%x)
            try: #falls in ignore entries felder sind, die nicht in columns sind, fange den fehler ab
                columns.remove(x)
            except:
                pass

        
        insert=[]
        for key in columns:
            if data[key]=="date_none": # date_none eintrag wird zu null uebersetzt
                insert.append('%s=null'%key)
            else:
                insert.append(""" "%s"=%s"""%(key,self.ZSQLQuote(data[key])))
            
        insertStr=",".join(insert)
        queryStr="update personal_www SET %s where key='%s'"%(insertStr,data['key'])
        self.ZSQLQuery("SET DATESTYLE TO 'German'")
        self.ZSQLQuery(queryStr)
       
        #currentwork
        #if not (txt==""):
        #    queryStr="INSERT INTO current_work (id_main,current,publish) VALUES ('%s','%s','%s')"%(id,txt,txt_p)
        #
        #    self.ZSQLQuery(queryStr)
        
        return True,msg


    def updatePersonalwww_doIt(self):
        """do the update"""
        args=self.REQUEST.form
        resultSet=self.REQUEST.SESSION['personal_www']['resultSet']
        news=self.REQUEST.SESSION['personal_www']['news']
        conflicts=self.REQUEST.SESSION['personal_www']['conflicts']
        logging.debug("updatePersonalwww_doIt: args=%s\n  resultSet=%s\n  news=%s\n  conflicts=%s"%(args,resultSet,news,conflicts))
        
        ret="<html><body>"
        # generate the new entry
      
        if news and (len(news)>0):
            ret+="<h2>Hinzugef&uuml;gt</h2>"
            ret+="<p>Neueintr&auml;ge erscheinen erst auf der Homepage, wenn ihnen eine e-mail Adresse zugeordnet wurde.</p>"
            ret+="<ul>"
            
        for new in news:
            if args.has_key(self.urlQuote(new.encode('utf-8'))): # entry was selected
                result,msg=self.generateNewPersonEntry(resultSet[new])
                if not result:
                    logging.error("Error (generateNewPersonEntry) %s"%msg)
                    ret+="<li>ERROR: %s %s"%(new.encode('utf-8'),msg)
                else:
                    ret+="<li>OK: %s"%(new.encode('utf-8'))
                    
        if news and (len(news)>0):
            ret+="<p>Neueintr&auml;ge erscheinen erst auf der Homepage, wenn ihnen eine e-mail Adresse zugeordnet wurde.</p>"
            ret+="</ul>"     
        
        # update

        if len(conflicts.keys())>0:
            ret+="<h2>&Auml;nderung des Benutzers &uuml;bernehmen</h2>"
            ret+="<p>Wenn n&ouml;tig in Filemaker-db &auml;ndern:</p>"
            
        # konflicte   
        for conflict in conflicts.keys():
            ignoreEntries=[]
            displayIgnored=[]
            for cf in conflicts[conflict]:
                if args[conflict.encode('utf-8')+'_'+cf[0]]=="stored": #use the stored one
                    ignoreEntries.append(cf[0])  #so ignore field cf[0]       
                    displayIgnored.append(cf)
                    
            if len(displayIgnored)>0:
                ret+="<h3>%s</h3>"%conflict.encode('utf-8')
                ret+="<table border='1'>"
                for iE in displayIgnored:
                    ret+="<tr><td>%s</td><td>%s</td><td>%s</td>"%(iE[0].encode('utf-8'),iE[1].encode('utf-8'),iE[2].encode('utf-8'))
                ret+="</table>"
                
            self.updatePersonEntry(resultSet[conflict],ignoreEntries=ignoreEntries)
         
         # rest
        cl=list(conflicts.keys())
        
        for key in resultSet.keys():
             if key not in cl:
                 self.updatePersonEntry(resultSet[key])
        return ret+"</body></html>"
                     

    def updateInstitutsbiliography(self):
        """update the Institutsbibliogrpahy"""
        self.upDateSQL('personalwww.xml')
        return "<html><body>DONE</body></html>"


    

    def updatePersonalwww_html(self):
        """update form for the homepages web form"""
        pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','updatePersonalwww.zpt')).__of__(self)
        return pt()

    
    def updatePersonalwww(self,uploadfile):
        """update personalwww
        @param uploadfile: file handle auf das file
        """
        dsn=self.getConnectionObj().connection_string
        #dsn="dbname=personalwww"
        resultSet=updatePersonalWWW.importFMPXML(uploadfile)
        news,conflicts=updatePersonalWWW.checkImport(dsn, resultSet)

        self.REQUEST.SESSION['personal_www']={}
        self.REQUEST.SESSION['personal_www']['resultSet']=resultSet
        self.REQUEST.SESSION['personal_www']['news']=news
        self.REQUEST.SESSION['personal_www']['conflicts']=conflicts
        
        pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','updatePersonalwww_check.zpt')).__of__(self)
        return pt()
    

     
    def reindexCatalogs(self,RESPONSE=None):
        """reindex members and project catalog"""
        
        
        try:
            
            self.ProjectCatalog.manage_catalogReindex(self.REQUEST,RESPONSE,self.REQUEST['URL1'])
            logger("MPIWG Root (reindexCatalog: projects)",logging.INFO,"DONE")
        except:
            logger("MPIWG Root (reindexCatalog: projects)",logging.WARNING," %s %s"%sys.exc_info()[:2])
        
        try:
            
            self.MembersCatalog.manage_catalogReindex(self.REQUEST,RESPONSE,self.REQUEST['URL1'])
            logger("MPIWG Root (reindexCatalog: members)",logging.INFO,"DONE")
        except:
            logger("MPIWG Root (reindexCatalog: members)",logging.WARNING," %s %s"%sys.exc_info()[:2])
        
        try:
            
            self.fulltextProjectsMembers.manage_catalogReindex(self.REQUEST,RESPONSE,self.REQUEST['URL1'])
            logger("MPIWG Root (reindexCatalog: fulltextProjectsMembers)",logging.INFO,"DONE")
        except:
            logger("MPIWG Root (reindexCatalog: fulltextProjectsMembers)",logging.WARNING," %s %s"%sys.exc_info()[:2])
            
        
        
        
        
        
        if RESPONSE:
            RESPONSE.redirect('manage_main')

        
        

    def getAllMembers(self):
        #ret=[]
        
        def sorter(x,y):
            return cmp(x[0].lower(),y[0].lower())
                    
        results=self.MembersCatalog({'isPublished':True})
       
        ret=[(unicodify(", ".join([proj.lastName, proj.firstName])), proj.getKey) for proj in results]
        
        ret.sort(sorter)
        return ret
    
    
    def printAllMembers(self):
        """print"""
        members=self.getAllMembers()
        ret=""
        for x in members:
            ret+="<p>%s</p>"%x
        return ret
    
        
    def makeList(self,entry):
        """makes a list out of one entry or repeat a list"""
        if type(entry) is StringType:
            return [entry]
        else:
            return entry

    def getTreeRSS(self,dep=None,date=None,onlyActive=1,onlyArchived=0):
        """generateTree"""
        rss="""<?xml version="1.0" encoding="utf-8"?>
                 <rss version="2.0">
                   <channel>"""
        
        for obj in self.getTree(dep, date, onlyActive, onlyArchived):
            linkStr="""<link>http://www.mpiwg-berlin.mpg.de/en/research/projects/%s</link>"""
            rss+="""<item>"""
            rss+=linkStr%obj[3].getId()
            rss+="""</item>"""
            if hasattr(obj[3],'publicationList'):
	        rss+="""<item>"""
                rss+=linkStr%(obj[3].getId()+"/publicationList");
                rss+="""</item>"""
        rss+="""</channel>
        </rss>"""

        
        return rss

    def getTree(self,dep=None,date=None,onlyActive=0,onlyArchived=0):
        """generate Tree from project list
        als Liste, jeder Eintrag ist ein Tupel ,(Tiefe, ProjektNummer,ProjektObject
        onlyActive = 0  : alle Projekte
        onlyActive = 1 : nur active Projekte
        onlyActive = 2: nur inactive Projekte
        
        onlyArchived=0: alle Projekte
        onlyArchived= 1 : nur aktuelle Projekte
        onlyArchived = 2: nur archivierte Projekte
        
        department fuer das Tree geholt werden soll
        """

        returnListTmp=[]
        returnList=[]
        
        for project in self.getProjectFields('xdata_05',sort="int",date=date): # get Projects sorted by xdata_05

            for idNr in project[1].split(";"): # more than one number
                if not idNr=="":
                    splittedId=idNr.split(".")
                    depth=len(splittedId)
                    nr=idNr
                    #title=project[0].WEB_title
                    title=[project[0].getContent('WEB_title')]
                    #print title
                    
                    if idNr[0]=="x": # kompatibilitaet mit alter Konvention, x vor der Nummer macht project inactive
                        project[0].setActiveFlag(False)
                   
                    if (not dep) or (splittedId[0]==dep): #falls dep gesetzt ist nur dieses hinzufuegen.
                        
                        if (onlyActive==0):
                            returnListTmp.append((depth,nr,title,project[0]))
                        elif (onlyActive==1) and project[0].isActiveProject(): #nur active projekte
                            returnListTmp.append((depth,nr,title,project[0]))
                        elif (onlyActive==2) and (not project[0].isActiveProject()): #nur active projekte
                            returnListTmp.append((depth,nr,title,project[0]))
                   
                   
        #filter jetzt die Liste nach Archived oder nicht
        for entry in returnListTmp:
                    if (onlyArchived==0):
                            returnList.append(entry)
                    elif (onlyArchived==1) and (not entry[3].isArchivedProject()): #nur active projekte
                            returnList.append(entry)
                    elif (onlyArchived==2) and (entry[3].isArchivedProject()): #nur active projekte
                            returnList.append(entry)
                   
        
        return returnList


        
    def changePosition(self,treeId,select,RESPONSE=None):
        """Change Postion Entry"""
        numbers=[]

        # Suche hoechste bisherige nummer
        projects=self.getProjectFields('xdata_05') # get Projects sorted by xdata_05
        #print "pj",projects
        for project in projects: #suche alle subtrees der treeId
            #print treeId
            
            founds=re.match(treeId+"\.(.*)",project[1].split(";")[0])
            if founds:
                #print "x",founds.group(0),len(founds.group(0).split("."))
                if len(founds.group(0).split("."))==len(treeId.split("."))+1: # nur ein punkt mehr, d.h. untere ebene
                    try:
                        numbers.append(int(founds.group(0).split(".")[len(founds.group(0).split("."))-1]))
                    except:
                        numbers.append(int(0))

        try:
            highest=max(numbers)
        except:
            highest=0
        projects=self.showNewProjects()
        for i in self.makeList(select):
            highest+=10
            projects[int(i)][0].xdata_05=treeId+"."+str(highest)


        if RESPONSE is not None:
            RESPONSE.redirect('showTree')
        
    def changeTree(self,RESPONSE=None):
        """change the complete tree"""
        form=self.REQUEST.form
        hashList={}
        onlyArchived=int(form.get("onlyArchived",0))
        onlyActive=int(form.get("onlyActive",0))
        dep=form.get("dep",None)
        
        fields=self.getTree(dep=dep,onlyArchived=onlyArchived,onlyActive=onlyActive)
        
        logging.info("GOT TREE!----------------------------------------------------")
        for field in form.keys():
            
            splitted=field.split('_')
            if (len(splitted)>1) and (splitted[1]=="runningNumber"): #feld hat die Form Nummer_name und runnignNumber
            
                
                nr=int(splitted[0]) # nummer des Datensatzes
                currentEntry = fields[nr]
            
                if form.has_key(str(nr)+'_active'): # active flag is set
                    fields[nr][3].setActiveFlag(True)
                else:
                    fields[nr][3].setActiveFlag(False)
                    
                #nummer hat sich geaendert
                
                entryChanged = False;
                
                if isinstance(fields[nr][3].xdata_05,list): #for some reasons somtimes the content of the field is a list with one entry.
                    fields[nr][3].xdata_05=fields[nr][3].xdata_05[0]
                    
                if not (fields[nr][3].xdata_05==form[str(nr)+'_number']):
                    logging.info("Changed!Number+++++++++++++++++++++++++++++++++")
                    logging.info(repr(fields[nr][3].xdata_05)+" ---> "+ repr(form[str(nr)+'_number']))
                    fields[nr][3].xdata_05=form[str(nr)+'_number']
                    entryChanged = True
                    
                #completed har sich geaendert
                          
                td = fields[nr][3].transformDate # hole die funktion zum transformieren des datums
                  
                if not (td(fields[nr][3].getCompletedAt())==td(form[str(nr)+'_completed'])):
                    fields[nr][3].setCompletedAt(form[str(nr)+'_completed'])
                    logging.info(repr(td(fields[nr][3].getCompletedAt()))+" ---> "+ repr(td(form[str(nr)+'_completed'])))
                    logging.info("Changed!Completed+++++++++++++++++++++++++++++++++")
                    entryChanged = True
                
                if not (td(fields[nr][3].getStartedAt())==td(form[str(nr)+'_started'])):
                    fields[nr][3].setStartedAt(form[str(nr)+'_started'])
            
                    logging.info(repr(td(fields[nr][3].getStartedAt()))+" ---> "+ repr(td(form[str(nr)+'_started'])))
                    logging.info("Changed!Started+++++++++++++++++++++++++++++++++")
                    entryChanged = True
                
                
                if entryChanged:
                    logging.info("Changed!+++++++++++++++++++++++++++++++++")
                    fields[nr][3].copyObjectToArchive()
                
                    
        if RESPONSE is not None:
            RESPONSE.redirect('showTree')

    def getProjectWithId(self,id):
        fields=self.getProjectFields('xdata_05')
        for field in fields:
            if field[1]==id:
                return field[0]

        return None
            
        
            
        
    def getRelativeUrlFromPerson(self,list):
        """get urls to person list"""
        ret=[]
        persons=list.split(";")
        for person in persons:
            
            if len(person)>1: #nicht nur Trennzeichen
                splitted=person.split(",")
                if len(splitted)==1:
                    splitted=person.split(" ")
                splittedNew=[re.sub(r'\s(.*)','$1',split) for split in splitted]
                if splittedNew[0]=='':
                    del splittedNew[0]
                search=string.join(splittedNew,' AND ')
                
                if not search=='':

                    try:
                        proj=self.MembersCatalog({'title':search})
                    except:
                        proj=None

                if proj:
                    #ret.append("<a href=%s >%s</a>"%(proj[0].absolute_url,person.encode('utf-8')))
                    ret.append("<a href=%s >%s</a>"%('members/'+proj[0].id+'/index.html',person))
                else:
                    #ret.append("%s"%person.encode('utf-8'))
                    ret.append("%s"%person)
        return string.join(ret,";")
        
    def getMemberIdFromKey(self,key):
        """gibt die ensprechende id  im members Ordner zum key"""
        
        if key=="":
            return ""
        try:
            key=utf8ify(key)
            catalogged=self.MembersCatalog({'getKey':key})
            if len(catalogged)==0:
                return ""
            else:
                return catalogged[0].getObject().getId()
        
        except:
            return ""

            

    def getProjectsOfMembers(self,date=None):
        """give tuple member /projects"""
        ret=[]
        members=self.getAllMembers()
        logging.debug("X %s"%repr(members))
        #return str(members)
        for x in members:
            #logging.debug("X %s"%repr(x))
            projects=self.getProjectsOfMember(key=x[1],date=date)
            if len(projects)>0:
                ret.append((x[0],projects))
            
        return ret

    def getProjectsOfMember(self,key=None,date=None,onlyArchived=1,onlyActive=1):
        """get projects of a member
    
        @param key: (optional) Key zur Idenfikation des Benutzer
        @param date: (optional) Version die zum Zeitpunkt date gueltig war
        @param onlyArchived: 
        onlyArchived=0: alle Projekte
        onlyArchived= 1 : nur aktuelle Projekte
        onlyArchived = 2: nur archivierte Projekte
        """
        # TODO: Die ganze Loesung
        def sortP(x,y):
            """sort by sorting number"""
            return cmp(x.WEB_title,y.WEB_title)
        
        ret=[]  
        if key:     
            logging.debug("MPIWGROOT (getProjectsOfMember):"+key)
            proj=self.ProjectCatalog({'getPersonKeyList':utf8ify(key)})
        else:
            return ret # key muss definiert sein
        
        #logging.debug("MPIWGROOT (getProjectsOfMember):"+repr(proj))
        if proj:
            proj2=[]
            for x in proj:
                #logging.error("proj:%s"%repr(x.getPath()))
                if (not getattr(x.getObject(),'invisible',None)) and (getattr(x.getObject(),'archiveTime','')==''):   
                      proj2.append(x)

        else:
            proj2=[]
            
       
       
        proj2.sort(sortP)

        projectListe=[]
        #logging.error("getprojectsofmember proj2: %s"%repr(proj2))
        for proj in proj2:   
            obj=proj.getObject()
            add=False
            if onlyArchived==1: #nur aktuell projecte
                if not obj.isArchivedProject():
                    add=True
            elif onlyArchived==2: #nur archivierte
                if obj.isArchivedProject():
                    add=True
            else: #alle
               add=True 
               
            if onlyActive==1: #nur active projecte
                if obj.isActiveProject():
                    add=add & True
                else:
                    add=add & False
                
            elif onlyArchived==2: #nur nicht aktvive
                if not obj.isActiveProject():
                    add=add & True
            else: #alle
               add=add & True
                    
            if add:
                projectListe.append(obj)
                
        #logging.error("getprojectsofmember projectliste: %s"%repr(projectListe))
        return projectListe
     
    def givePersonList(self,name):
        """check if person is in personfolder and return list of person objects"""
        
        splitted=name.split(",")
        if len(splitted)==1:
            splitted=name.lstrip().rstrip().split(" ")
        splittedNew=[split.lstrip() for split in splitted]
        
        if splittedNew[0]=='':
            del splittedNew[0]
        search=string.join(splittedNew,' AND ')
        
        if not search=='':
            proj=self.MembersCatalog({'title':search})

        if proj:
            return [[x.lastName,x.firstName] for x in proj]
        else:
            return []
            
##         splitted=name.split(",") # version nachname, vorname...
##         if len(splitted)>1:
##             lastName=splitted[0] 
##             firstName=splitted[1]
##         else: 
##             splitted=name.split(" ") #version vorname irgenwas nachnamae
        
##             lastName=splitted[len(splitted)-1]
##             firstName=string.join(splitted[0:len(splitted)-1])

##         objs=[]

        #print  self.members 
      ##   for x in self.members.__dict__:
##             obj=getattr(self.members,x)
##             if hasattr(obj,'lastName') and hasattr(obj,'firstName'):
                
##                 if (re.match(".*"+obj.lastName+".*",lastName) or re.match(".*"+lastName+".*",obj.lastName)) and (re.match(".*"+obj.firstName+".*",firstName) or re.match(".*"+firstName+".*",obj.firstName)):
                    
##                     objs.append((obj,lastName+", "+firstName))

        
##        return objs


    def personCheck(self,names):
        """all persons for list"""
        #print "names",names
        splitted=names.split(";")
        ret={}
        for name in splitted:

            if not (name==""):
                try:
                    ret[name]=self.givePersonList(name)
                except:
                    """NOTHIHN"""
        #print "RET",ret
        return ret

    def giveCheckList(self,person,fieldname):
        """return checklist"""
        #print "GCL",fieldname
        if fieldname=='xdata_01':
            x=self.personCheck(person.getContent(fieldname))
            #print "GCLBACKX",x
            return x
        

    def isCheckField(self,fieldname):
        """return chechfield"""
        
        return (fieldname in checkFields)

    
    def generateNameIndex(self):
        """erzeuge einen index verwendeter personen"""
        import psycopg
        o = psycopg.connect('dbname=authorities user=dwinter password=3333',serialize=0) 
        results={}
        print self.fulltext.historicalNames.items()
        for nameItem in self.fulltext.historicalNames.items(): #gehe durch alle namen des lexikons
            
            c = o.cursor() 
            name=nameItem[0]
            print "check",name
            c.execute("select lastname,firstname from persons where lower(lastname) = '%s'"%quote(name))
            tmpres=c.fetchall()
            firstnames=[result[1] for result in tmpres] # find all firstnames
            if tmpres:
                lastname=tmpres[0][0]
                
            for found in self.fulltext({'names':name}):
                if found.getObject().isActual():
                    for nh in found.getObject().getGetNeighbourhood(name, length=50,tagging=False): #hole umgebung
                        #schaue nun ob der vorname hinter oder vor dem name ist
                        position=nh.find(lastname)
                        # vorher
                        #print "NH",nh
                        bevorS=nh[0:position].split()
                        #print "BV",bevorS
                        if len(bevorS)>1:
                            try:
                                bevor=[bevorS[-1],bevorS[-2]]
                            except:
                                bevor=[bevorS[0]]
                        else:
                            bevor=[]
                        #nachher
                        behindS= re.split("[,|;| ]",nh[position:]) 
                        #print "BH",behindS
                        if len(behindS)>2:
                            try:
                                behind=behindS[1:3]
                            except:
                                behind=[bevorS[1]]
                        else:
                            behind=[]
                        for firstname in firstnames:
                            if firstname in bevor+behind: #Namen wie mit Adelspraedikaten werden so erstmal nich gefunden
                                id="%s,%s"%(lastname,firstname)
                                if not results.has_key(id):
                                    results[id]=[]
                                objId=found.getObject().getId()
                                if not (objId in results[id]):
                                    print "d %s for %s"%(id,objId)    
                                    results[id].append(objId)    
            self.nameIndex=results
        return results
                    
    def editNameIndexHTML(self):
        """edit the name index"""
        if not hasattr(self,'nameIndexEdited'): # falls editierter index noch nicht existiert, kopiere automatisch erstellten
            self.nameIndexEdited=copy.copy(self.nameIndex)
            print "huh"
        #self.nameIndexEdited=copy.copy(self.nameIndex)
        #print self.nameIndexEdited
        pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','editHistoricalNames.zpt')).__of__(self)
        return pt()
    
    def getNamesInProject(self,projectId):
        """get all names ofnameIndexEdited which are references in projec with projectId"""
        
        ret=[]
        for name in self.nameIndexEdited.keys():
            if projectId in self.nameIndexEdited[name]:
                ret.append(name)
        
        return ret
    
    def editNameIndex(self,RESPONSE=None,name=None,occurrances=None,submit=None):
        """edit the index"""
        nI=self.nameIndexEdited # mI introduced to make sure that changes to nameIndexEdited are know to ZODB
        if submit=="delete":
           

            dh=getattr(self,'deletedHistoricalNames',{})
            
            if type(dh) is ListType:
                dh={}
            if not dh.has_key(name):
                dh[name]=occurrances.split("\n")
            else:
                dh[name]+=occurrances.split("\n")
            
            self.deletedHistoricalNames=dh
            
            del self.nameIndexEdited[name]
            
        
        elif (submit=="change"):
            
            nI[name]=occurrances.split("\n")[0:]
            
        elif (submit=="add"):
            if not nI.has_key(name):
                nI[name]=occurrances.split("\n")
            else:
                nI[name]+=occurrances.split("\n")
    
        self.nameIndexEdited=nI
   
      
        if RESPONSE is not None:
            RESPONSE.redirect('editNameIndexHTML')
        
    
    
    def restoreIndex(self):
        """restore"""
        self.nameIndexEdited=self.nameIndex
        return "done"
    

    def sortResults(self,results):
        """search the catalog and give results back sorted by meta_type"""
        ret = {}
        logging.debug(results())
        for result in results():
            metaType = result.meta_type
            resultList= ret.get(metaType,[])
            resultList.append(result)
            ret[metaType]=resultList
        
        logging.debug(ret)
        return ret
        
            
def manage_addMPIWGRootForm(self):
    """form for adding the root"""
    pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addMPIWGRootForm.zpt')).__of__(self)
    return pt()

def manage_addMPIWGRoot(self,id,title,connection_id="",RESPONSE=None):
    """add a root folder"""
    newObj=MPIWGRoot(id,title)
    self._setObject(id,newObj)
    ob=getattr(self,id)
    setattr(ob,'connection_id',connection_id)
    if RESPONSE is not None:
        RESPONSE.redirect('manage_main')