"""CDLI extensions of the filearchive"""    
from Products.versionedFile.extVersionedFile import *
from Products.ZCatalog.CatalogPathAwareness import CatalogAware
from tempfile import mkstemp,mkdtemp    
import os.path
import os
from types import *
import urlparse
import urllib
import cgi
from OFS.OrderedFolder import OrderedFolder
from OFS.SimpleItem import SimpleItem
import time
from OFS.Folder import manage_addFolder
import re
from AccessControl import ClassSecurityInfo
from Acquisition import Implicit
from Globals import Persistent
from threading import Thread
from ZPublisher.HTTPRequest import HTTPRequest
from ZPublisher.HTTPResponse import HTTPResponse
from ZPublisher.BaseRequest import RequestContainer
import threading
from BTrees.OOBTree import OOBTree
import logging
import transaction
import copy
import codecs
import sys

def unicodify(s):
    """decode str (utf-8 or latin-1 representation) into unicode object"""
    if not s:
        return u""
    if isinstance(s, str):
        try:
            return s.decode('utf-8')
        except:
            return s.decode('latin-1')
    else:
        return s

def utf8ify(s):
    """encode unicode object or string into byte string in utf-8 representation.
       assumes string objects to be utf-8"""
    if not s:
        return ""
    if isinstance(s, str):
        return s
    else:
        return s.encode('utf-8')


def generateXMLReturn(hash):
    """erzeugt das xml file als returnwert fuer uploadATFRPC"""

    ret="<return>"
    
    ret+="<errors>"
    for error in hash['errors']:
        ret+="""<error atf="%s">%s</error>"""%error
    
    ret+="</errors>"
    
    ret+="<changes>"
    for changed in hash['changed']:
        ret+="""<change atf="%s">%s</change>"""%changed
    ret+="</changes>"
    
    ret+="<newPs>"
    for new in hash['newPs']:
        ret+="""<new atf="%s"/>"""%new
    ret+="</newPs>"
    
    ret+="</return>"
    return ret
    
    
    
    
    
    
    
    
    
    
def unique(s):
    """Return a list of the elements in s, but without duplicates.

    For example, unique([1,2,3,1,2,3]) is some permutation of [1,2,3],
    unique("abcabc") some permutation of ["a", "b", "c"], and
    unique(([1, 2], [2, 3], [1, 2])) some permutation of
    [[2, 3], [1, 2]].

    For best speed, all sequence elements should be hashable.  Then
    unique() will usually work in linear time.

    If not possible, the sequence elements should enjoy a total
    ordering, and if list(s).sort() doesn't raise TypeError it's
    assumed that they do enjoy a total ordering.  Then unique() will
    usually work in O(N*log2(N)) time.

    If that's not possible either, the sequence elements must support
    equality-testing.  Then unique() will usually work in quadratic
    time.
    (from the python cookbook)
    """

    n = len(s)
    if n == 0:
        return []

    # Try using a dict first, as that's the fastest and will usually
    # work.  If it doesn't work, it will usually fail quickly, so it
    # usually doesn't cost much to *try* it.  It requires that all the
    # sequence elements be hashable, and support equality comparison.
    u = {}
    try:
        for x in s:
            u[x] = 1
    except TypeError:
        del u  # move on to the next method
    else:
        return u.keys()

    # We can't hash all the elements.  Second fastest is to sort,
    # which brings the equal elements together; then duplicates are
    # easy to weed out in a single pass.
    # NOTE:  Python's list.sort() was designed to be efficient in the
    # presence of many duplicate elements.  This isn't true of all
    # sort functions in all languages or libraries, so this approach
    # is more effective in Python than it may be elsewhere.
    try:
        t = list(s)
        t.sort()
    except TypeError:
        del t  # move on to the next method
    else:
        assert n > 0
        last = t[0]
        lasti = i = 1
        while i < n:
            if t[i] != last:
                t[lasti] = last = t[i]
                lasti += 1
            i += 1
        return t[:lasti]

    # Brute force is all that's left.
    u = []
    for x in s:
        if x not in u:
            u.append(x)
    return u


class BasketContent(SimpleItem):
    """classe fuer den Inhalt eines Baskets"""
   
    def __init__(self,content=[]):
        """content"""
        self.contentList=content[0:]
    
    def getContent(self):
        """get content"""

        return self.contentList
    
    def setContent(self,content):
        self.contentList=content[0:]
    
    def numberOfItems(self):
        """number"""
        
        return len(self.getContent())
        
    
class uploadATFfinallyThread(Thread):
    """class for adding uploaded filed (temporarily stored in the staging area at /tmp"""
    
    def __init__(self):
        """init for uploadATFfinallyThread"""
        self.continueVar=True
        self.returnValue=None
        self.end=False
        Thread.__init__(self)
           
    def set(self,procedure,comment="",basketname='',unlock=None,SESSION=None,username=None,serverport="8080"):
        """set start values for the thread"""
        self.procedure=procedure
        self.comment=comment
        self.basketname=basketname
        self.unlock=unlock
        self.SESSION=SESSION
        self.username=username
        self.serverport=serverport
       
        
    def __call__(self):
        """call of the thread (equals run)"""
        self.run()
        return True
    
    def getContext(self, app,serverport="8080"):
        """get the context within the ZODB"""
        
        resp = HTTPResponse(stdout=None)
        env = {
            'SERVER_NAME':'localhost',
            'SERVER_PORT':serverport,
            'REQUEST_METHOD':'GET'
            }
        req = HTTPRequest(None, env, resp)
        return app.__of__(RequestContainer(REQUEST = req))
          
        
    def run(self):
        """run"""
        
        self.result=""
        #find context within ZODB
        from Zope import DB
        conn = DB.open()
        root = conn.root()
        app  = root['Application']
        ctx = self.getContext(app,serverport=self.serverport)

        #add the files
        self.uploadATFfinallyThread(ctx,self.procedure,comment=self.comment,basketname=self.basketname,unlock=self.unlock,SESSION=self.SESSION,username=self.username)
        #commit the transactions
        transaction.get().commit()
        conn.close()
        #set flag for end of this method
        self.end=True
        logging.info("ended")
        return True
    
    def __del__(self):
        """delete"""
        
        
    
    def getResult(self):
        """method for accessing result"""
        
        return self.result
     
    def uploadATFfinallyThread(self,ctx,procedure,comment="",basketname='',unlock=None,RESPONSE=None,SESSION=None,username=None):
        """upload the files"""
        #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot
        ctx2=ctx.cdliRoot
   
        self.result+="<h2>Start processing</h2>"
        
        #shall I only upload the changed files?
        logging.info("uploadATFfinally procedure: %s"%procedure)
        if procedure=="uploadchanged":
            changed=[x[0] for x in SESSION.get('changed',[])]
            uploadFns=changed+SESSION.get('newPs',[])
        
        #or all
        elif procedure=="uploadAll":
            uploadFns=[]
            for x in os.listdir(SESSION['tmpdir']):
                if not x in SESSION['lockerrors']:
                    uploadFns.append(x)
                    
        #or maybe nothing
        elif procedure=="noupload":
                        return True
        else:
            uploadFns=[]
            
        #do first the changed files    
        i=0
        for fn in uploadFns:
            i+=1
            founds=ctx2.CDLICatalog.search({'title':fn})
            if len(founds)>0:
                SESSION['author']=str(username)
                self.result="<p>Changing : %s"%fn+self.result
                founds[0].getObject().manage_addCDLIFileObject('',comment,SESSION['author'],file=os.path.join(SESSION['tmpdir'],fn),from_tmp=True)
            if i==200:
                        i=0
                        transaction.get().commit()
                        logging.info("changing: do commit")
        
        transaction.get().commit()
        logging.info("changing: last commit")

        #now add the new files        
        newPs=SESSION['newPs']
        if len(newPs)>0:
            tmpDir=SESSION['tmpdir']
            logging.info("adding start")
            self.result="<p>Adding files</p>"+self.result
            #TODO: make this configurable, at the moment base folder for the files has to be cdli_main
            
            ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self)
            logging.info("adding finished")
        
        
        #unlock locked files?
        if unlock:
            logging.info("unlocking start")
            self.result="<p>Unlock files</p>"+self.result
            unlockFns=[]
            for x in os.listdir(SESSION['tmpdir']):
                    if not x in SESSION['errors']:
                        unlockFns.append(x)
            logging.info("unlocking have now what to unlock")
                        
            for fn in unlockFns:
                #logging.info("will unlock: %s"%fn)
                founds=ctx2.CDLICatalog.search({'title':fn})
                #logging.info("found it: %s"%repr(founds))
                if len(founds)>0:
                    #logging.info("unlock: %s"%founds[0].getObject().getId())
                    SESSION['author']=str(username)
                   
                    founds[0].getObject().lockedBy=""
            logging.info("unlocking done")
                    
        #if a basketname is given, add files to the basket
        if not (basketname ==''):
            logging.info("add to basket %s"%basketname)
            self.result="<p>Add to basket</p>"+self.result
            basketId=ctx2.basketContainer.getBasketIdfromName(basketname)
            
            if not basketId: # create new basket
                logging.info("create basket %s"%basketname)
                self.result="<p>Create a new basket</p>"+self.result
                ob=ctx2.basketContainer.addBasket(basketname)
                basketId=ob.getId()
            basket=getattr(ctx2.basketContainer,str(basketId))
            ids=os.listdir(SESSION['tmpdir'])
            basket.addObjects(ids,deleteOld=True,username=str(username))    
               
        if RESPONSE is not None:
            RESPONSE.redirect(self.aq_parent.absolute_url())
        

        logging.info("uploadfinally done")
        return True

class tmpStore(SimpleItem):
    """simple item"""
    meta_type="cdli_upload"
    
    def __init__(self,id):
        """init tmp"""
        self.id=id
        
class uploadATFThread(Thread):
    """class for checking the files befor uploading"""
    
    def __init__(self):
        """initialise"""
        
        self.continueVar=True
        self.returnValue=None
        
        Thread.__init__(self)
        
        
    def set(self,upload,basketId,username,idTmp,serverport="8080"):
        """set start values for the thread"""
        self.result=""
        self.upload=upload
        self.basketId=basketId
        self.username=username
        self.serverport=serverport
        self.idTmp=idTmp
        
    def __call__(self):
        """call method """
        self.run()
        return True
    
    def getContext(self, app,serverport="8080"):
        """get the context within the ZODB"""
        resp = HTTPResponse(stdout=None)
        env = {
            'SERVER_NAME':'localhost',
            'SERVER_PORT':serverport,
            'REQUEST_METHOD':'GET'
            }
        req = HTTPRequest(None, env, resp)
        return app.__of__(RequestContainer(REQUEST = req))
        
    def run(self):
        idTmp=self.idTmp
        self.result=""
        #find context within ZODB
        from Zope import DB
        conn = DB.open()
        root = conn.root()
        app  = root['Application']
        ctx = self.getContext(app,serverport=self.serverport)
        logging.info("run intern")
        try:
            logging.info("created: %s"%idTmp)
            ctx.temp_folder._setObject(idTmp,tmpStore(idTmp))
        except:
            logging.error("thread upload: %s %s"%sys.exc_info()[0:2])
            
        logging.info("call thread intern")
        self.uploadATFThread(ctx,self.upload,idTmp,self.basketId)
     
        #ctx.cdliRoot.cdli_main.tmpStore2[self.getName()[0:]]=self.returnValue
        
        
        transaction.get().commit()
       
        conn.close()
        
        return getattr(ctx.temp_folder,idTmp)
        
    def getResult(self):
        """method for accessing result"""
        return self.result
    
    def uploadATFThread(self,ctx,upload,idTmp,basketId=0):
        """upload an atf file"""
        #TODO: add comments
        #TODO: finish uploadATF
        
        stObj=getattr(ctx.temp_folder,idTmp)
        logging.info("start, upload thread")
        self.result="<html><body><h2>I got your file, start now to split it into single atf-files!</h2><p>"
    
        #make sure that id is a string and not an integer
        basketId=str(basketId)
        
        #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot
        ctx2=ctx.cdliRoot
        
        #get temporary file for staging the downloaded and splitted files
        dir=mkdtemp()
        
        
        changed=[] # changed files
        errors=[]  # files with errors
        lockerrors=[]  # files with errors

        newPs=[]   # new p filed
        psNotInCatalog=[] # files not in the catalog
        
        #split the uploadedd atf file
        basketNameFromFile, numberOfFiles=splitatf(upload,dir,ext=self)
        
        #find basketId if not set
        
        #get active abaket
        if basketId == '0':
            basketObj=ctx2.basketContainer.getActiveBasket()
            if basketObj:
                basketId=basketObj.getId()
                
        #if there is no active basket and no basketid given, id is empty, else get besketname and length
        if basketId == '0':
            basketNameFromId=""
            basketLen=0
        else:
            basketNameFromId=getattr(ctx2.basketContainer,basketId).title
            basketLen=getattr(ctx2.basketContainer,basketId).getLastVersion().numberOfItems()
            
        logging.info("got the file, upload thread")
        self.result+="""<html><body><h2>I got the files</h2><
                        p>I am computing the differences to the exisiting files</p>"""
                                   
        #start to check the files
        for fn in os.listdir(dir):
            
            self.result="<p>process:%s</p>"%fn+self.result
            
            # check if file is in the catalog
            #TODO: checkCatalog is not implemented yet
            if ctx2.cdli_main.checkCatalog(fn):
                psNotInCatalog.append(fn)
                
            #check if p-file already at the server  
            founds=ctx2.CDLICatalog.search({'title':fn})    
      
            #if not than add filename to the list of newfiles
            
            data=file(os.path.join(dir,fn)).read()
            status,msg=checkFile(fn,data,dir)
            #status=True
            
            
            if not status: # error
                errors.append((fn,msg))
            
            else:
                if len(founds)==0:
                    newPs.append(fn)

                #if p file alread at the server    
                for found in founds:
                    #analyse the differences to the actual file
                    obj=found.getObject()

                    if (not (str(obj.lockedBy))=='') and (not (str(obj.lockedBy)==str(self.username))):
                                lockerrors.append((fn,str(obj.lockedBy)))
                    else:
                
                        diffs=obj.diff(data)
                        if diffs[0]>0:
                            changed.append((obj,diffs)) #hochladen

        #ready, set the returnValues
        self.result+="<h3>Done</h3></body></html>"
        
        stObj.returnValue={}
        
        stObj.returnValue['errors']=errors
        
        stObj.returnValue['newPs']=newPs
        stObj.returnValue['tmpdir']=dir
        stObj.returnValue['basketLen']=basketLen
        stObj.returnValue['numberOfFiles']=numberOfFiles
        stObj.returnValue['basketNameFromId']=basketNameFromId
        stObj.returnValue['basketNameFromFile']=basketNameFromFile
        stObj.returnValue['basketId']=basketId
        stObj.returnValue['dir']=dir
        #stObj.returnValue['changed']=copy.copy(changed)
        stObj.returnValue['changed']=[(x[0].getId(),x[1][0]) for x in changed]
        #stObj.returnValue['lockerrors']=[x[0].getId() for x in lockerrors]
        stObj.returnValue['lockerrors']=[x for x in lockerrors]
        self.returnValue=True
        #ctx2.cdli_main.setTemp('v_uploadATF_returnValue',True)
    
 
class CDLIBasketContainer(OrderedFolder):
    """contains the baskets"""
    

    security=ClassSecurityInfo()
    meta_type="CDLIBasketContainer"
    
    def getPNumbersOfBasket(self,basketName):
        """get all pnumbers of a basket as a list, returns an empty list if basket not found
        @param basketName: name of the basket
        """
        ret=[]
        basketId=self.getBasketIdfromName(basketName)
        if not basketId:
            return []
        
        ob=getattr(self,basketId).getContent()
        
        ret=[x[0].split(".")[0] for x in ob]
        
        return ret
    
    security.declareProtected('manage','getBasketAsOneFile')       
    def getBasketAsOneFile(self,basketName,current="no"):
        """returns all files of the basket combined in one file
        @param basketName: Name of the basket
        @param current: (optional) if current is set to "yes" then the most current version of 
                        all files are downloaded and not the versions of the files as stored in the basket
        """
        ret=""
        basketId=self.getBasketIdfromName(basketName)
        if not basketId:
            return ""
        
        ob=getattr(self,basketId).getLastVersion()
        for object in ob.getContent():
            if current=="no": #version as they are in the basket
                            ret+=str(object[0].getData())+"\n"
            elif current=="yes":
                            #search current object
                            logging.info("crrent: %s"%object[1].getId().split(".")[0])
                            founds=self.CDLICatalog.search({'title':object[1].getId().split(".")[0]})
                            if len(founds)>0:      
                                ret+=str(founds[0].getObject().getLastVersion().getData())+"\n"
        return ret
    
    security.declareProtected('manage','upDateBaskets') 
    def upDateBaskets(self):
        """update content in to objects"""
        
        founds=self.ZopeFind(self,obj_metatypes=['CDLIBasketVersion'],search_sub=1)

        for found in founds:
            found[1].updateBasket()
        
    security.declareProtected('manage','deleteBaskets')        
    def deleteBaskets(self,ids=None):
        """delete baskets, i.e. move them into trash folder"""
        
        
        found=self.ZopeFind(self,obj_ids=['trash'])
        
        if len(found)<1:
            manage_addFolder(self, 'trash')
            trash=self._getOb('trash')
        else:
            trash=found[0][1]
        
        if type(ids) is not ListType:
            ids=[ids]
        cut=self.manage_cutObjects(ids)
        trash.manage_pasteObjects(cut)
        
    security.declareProtected('manage','manageBaskets')       
    def manageBaskets(self,submit,ids=None,basket1="",basket2="",joinBasket="",subtractBasket="",REQUEST=None,RESPONSE=None):
        """manage baskets, delete or copy"""
        if submit=="delete":
            self.deleteBaskets(ids)
        
        elif submit=="join":
            flag,msg=self.joinBasket(joinBasket, ids)
            logging.info("joining %s %s"%(flag,msg))
            
        elif submit=="subtract":
            logging.info("BBBb %s %s"%(basket1,basket2))
            flag,msg=self.subtractBasket(subtractBasket, basket1,basket2)
            logging.info("subtract %s %s"%(flag,msg))
            
        if RESPONSE:
            RESPONSE.redirect(self.absolute_url())
    
    security.declareProtected('View','getBasketIdfromName')       
    def getBasketIdfromName(self,basketname):
        """get id from name"""

        for basket in self.ZopeFind(self,obj_metatypes=["CDLIBasket"]):
            if basket[1].title==basketname:
                return basket[0]
        else:
            None
    
    security.declareProtected('manage','uploadBasket_html')        
            
    def uploadBasket_html(self,basketId='0'):
        """upload an atf file, html form"""
        

        basketId=str(basketId)
        if not basketId=='0':
            basketName=getattr(self.basketContainer,basketId).title
        else:
            basketName=""
            
        pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadBasket_html.zpt')).__of__(self)
        return pt(basketId=basketId,basketName=basketName)
   

    security.declareProtected('manage','index_html')    
    def index_html(self):
        """stanadard ansicht"""
        


        ext=self.ZopeFind(self,obj_ids=["index.html"])
        if ext:
            return ext[0][1]()
        
        pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketContainerMain')).__of__(self)
        return pt()
    
    def getStorageFolderRoot(self):
        """root des storage folders"""
        return self.cdli_main
    
    def __init__(self,id,title):
        """ init basket container"""
        self.id=id
        self.title=title
     
 
    def getBasketsId(self):
        """get all baskets als klartext"""
        
        ret=""
        baskets=self.ZopeFind(self,obj_metatypes=['CDLIBasket'])
        for basket in baskets:
            com,user,time,values = basket[1].getContentIds()
            ret+= "BASKET:"+com+"\t"+user+"\t"+time+"\n"
            for x in values:
                ret+= x[0]+"\t"+x[1]+"\n"
                return ret

    def getBaskets(self,sortField='title'):
        """get all baskets files"""

        def sortName(x,y):
            return cmp(x[1].title.lower(),y[1].title.lower())

        def sortDate(x,y):
            return cmp(y[1].getLastVersion().getTime(),x[1].getLastVersion().getTime())

        
        def sortComment(x,y):

        
            
             try:
                xc=getattr(x[1],'comment','ZZZZZZZZZZZZZ').lower()
             except:
                xc='ZZZZZZZZZZZZZ'.lower()
             try:
                yc=getattr(y[1],'comment','ZZZZZZZZZZZZZ').lower()
             except:
                yc='ZZZZZZZZZZZZZ'.lower()
    
    
             if (xc=='') or (xc=='ZZZZZZZZZZZZZ'.lower()):
                 
                 try:
                     xc=x[1].getLastVersion().getComment().lower()
                 except:
                     xc='ZZZZZZZZZZZZZ'.lower()
                     
             if (yc=='') or (yc=='ZZZZZZZZZZZZZ'.lower()):
                 try:
                     yc=y[1].getLastVersion().getComment().lower()
                 except:
                     yc='ZZZZZZZZZZZZZ'.lower()
    
             
                 return cmp(xc,yc)
        
        def sortAuthor(x,y):
            
            return cmp(x[1].getLastVersion().getUser().lower(),y[1].getLastVersion().getUser().lower())
        
        baskets=self.ZopeFind(self,obj_metatypes=['CDLIBasket'])
        
        
        if sortField=='title':
            baskets.sort(sortName)
        elif sortField=='date':
            baskets.sort(sortDate)
        elif sortField=='author':
            baskets.sort(sortAuthor)
        elif sortField=='comment':
            baskets.sort(sortComment)

        return baskets
    
        
    def subtractBasket(self,newBasket,basket1,basket2):
        """subtract basket2 from basket1 
        (i.e. newbasket will contain alle elements of basket1 which are not in basket2), 
        if basket2 contains files which are not in basket1, then theses files fill be ignored
               
        @param newbasket: name of the new basket
        @param basket1: basket where basket2 will be subtracted from
        @param basket2: see above
      
        """
        logging.info("CCCCC %s %s"%(basket1,basket2))
   
        try:
            newB=self.addBasket(newBasket)
        except:
            return False, "cannot create the new basket"
        
        

       
     
        bas2= getattr(self,basket2)            
        bas2content=bas2.getContent()
        bas2ids=[x[0] for x in bas2content]
        
       
            
        bas1= getattr(self,basket1)   
        bas1content=bas1.getContent()
        
        
        newBasketContent={}
        
        for id,version in bas1content:
            if not (id in bas2ids):
                newBasketContent[id]=version
        
        username=self.getActualUserName()
        
        logging.info("sbc %s"%newBasketContent)
        newB.addObjectsWithVersion(newBasketContent,username=username,catalog=self.CDLICatalog)
        
        return True, ""
    
            
    def joinBasket(self,newBasket,oldBaskets):
        """join two baskets
        @param newbasket: name of the new basket
        @param oldbaskets: list of baskets to be joined
        """
        try:
            newB=self.addBasket(newBasket)
        except:
            return False, "cannot create the new basket"
        
        newBasketContent={}
        for ob in oldBaskets:
            x= getattr(self,ob,None)
            if x is None:
                return False, "cannot find basket: %s"%ob
            
            ids=x.getContent() # hole den Inhalt
            
            for id,version in ids:
                if newBasketContent.has_key(id): # p number gibt's schon
                    newBasketContent[id]=max(newBasketContent[id],version) # speichere die groessere Versionsnumber
                else:
                    newBasketContent[id]=version
        username=self.getActualUserName()
        
        logging.info("nbc %s"%newBasketContent)
        newB.addObjectsWithVersion(newBasketContent,username=username,catalog=self.CDLICatalog)
        
        return True, ""
    
    def getNewId(self):
        """createIds"""
        last=getattr(self,'last',0)
        last +=1
        while len(self.ZopeFind(self,obj_ids=[str(last)]))>0:
            last+=1
    
        self.last=last
        return last
    
    def setActiveBasket(self,basketId,REQUEST=None):
        """store active basketId in a cookie"""
        self.REQUEST.RESPONSE.setCookie("CDLIActiveBasket",basketId,path="/")
        try:
            qs=cgi.parse_qs(REQUEST['QUERY_STRING'])
            del(qs['basketId'])
        except:
            qs={}
        if REQUEST:
            REQUEST.RESPONSE.redirect(REQUEST['URL1']+'?'+urllib.urlencode(qs))
            
    def getActiveBasket(self):
        """get active basket from cookie"""
        
        id= self.REQUEST.cookies.get('CDLIActiveBasket',None)
        if id:
            obj=getattr(self,str(id),None)
        else:
            obj=None
        return obj
    
    def getActualUserName(self):
        """get name of the actualuser"""
        return str(self.REQUEST['AUTHENTICATED_USER'])
    
    security.declareProtected('manage','addBasket') 
    def addBasket(self,newBasketName):
        """add a new basket"""
        
        ob=manage_addCDLIBasket(self,newBasketName)
        return ob

    def storeInBasket(self,submit,ids=None,newBasketName=None,fromFileList=None,RESPONSE=None,REQUEST=None):
        """store it"""
        if not ids:
            ids=self.REQUEST.SESSION['fileIds']
            
        if type(ids) is not ListType:
            ids=[ids]
        
        if (submit.lower()=="store in new basket") or (submit.lower()=="new basket"):
            basketRet=self.addBasket(newBasketName)
            self.setActiveBasket(basketRet.getId())
            basket=getattr(self,basketRet.getId())
        elif (submit.lower()=="store in active basket") or (submit.lower()=="active basket"):
            basket=self.getActiveBasket()
        
        added=basket.addObjects(ids)
        back=self.REQUEST['HTTP_REFERER'].split("?")[0]+"?basketName="+basket.title+"&numberOfObjects="+str(added)
        
        
        if fromFileList:

            return self.cdli_main.findObjectsFromList(list=self.REQUEST.SESSION['fileIds'],basketName=basket.title,numberOfObjects=added)
       
        if RESPONSE:
            
            RESPONSE.redirect(back)
            
        return True
    
def manage_addCDLIBasketContainerForm(self):
    """add the CDLIBasketContainer form"""
    pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIBasketContainer.zpt')).__of__(self)
    return pt()

def manage_addCDLIBasketContainer(self,id,title,RESPONSE=None):
    """add the basket"""
    ob=CDLIBasketContainer(id,title)
    
    self._setObject(id, ob)
    
    if RESPONSE is not None:
        RESPONSE.redirect('manage_main')

class CDLIBasket(Folder,CatalogAware):
    """basket"""
    
    meta_type="CDLIBasket"
    default_catalog="CDLIBasketCatalog"
    
    def searchInBasket(self,indexName,searchStr,regExp=False):
        """searchInBasket"""

        lst=self.searchInLineIndexDocs(indexName,searchStr,uniq=True,regExp=regExp)
        ret={}
        
        lv=self.getLastVersion()


        for obj in lv.content.getContent():
            id=obj[1].getId().split(".")[0]
            if id in lst:
        
                ret[id]=self.showWordInFile(id,searchStr,lineList=self.getLinesFromIndex(indexName,searchStr,id,regExp=regExp),regExp=regExp,indexName=indexName)
        
        
        pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','searchResultsInBasket')).__of__(self)
        return pt(result=ret,indexName=indexName,regExp=regExp,word=searchStr)
        
         
 
        
    def searchInBasket_v1(self,searchStr):
        """search occurences of searchStr in files im basket"""
        ret=[]
        lv=self.getLastVersion()
        logging.info("searching")
        for obj in lv.content.getContent():
            txt=obj[0].getData()
            for x in txt.split("\n"):
                logging.info("search %s"%x)
                if re.match(searchStr,x):
                    ret.append(x)
        
        return "\n".join(ret)
                
            
    def getFile(self,obj):
        return obj[1]
    
    def getFileLastVersion(self,obj):
        return obj[0]
    
    def getFileNamesInLastVersion(self):
        """get content of the last version as list"""
        
        return [x[1].getId() for x in self.getLastVersion().getContent()]
    

    def isActual(self,obj):
        """teste ob im basket die aktuelle version ist"""
        actualNo=obj[1].getLastVersion().getVersionNumber()
        storedNo=obj[0].getVersionNumber()
        
        founds=self.CDLICatalog.search({'title':obj[0].getId()})
        if len(founds)>0:
            actualNo=founds[0].getObject().getLastVersion().getVersionNumber()
            
        if len(founds)>0 and founds[0].getObject().aq_parent.getId()==".trash":
            return False, -1
        
        if actualNo==storedNo:
            return True , 0
        else:
            return False, actualNo
        
    def history(self):
        """history"""  

        ext=self.ZopeFind(self.aq_parent,obj_ids=["history_template.html"])
        if ext:
            return getattr(self,ext[0][1].getId())()
        
        pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketHistory')).__of__(self)
        return pt()
    
    def getStorageFolderRoot(self):
        """root des storage folders"""
        return self.aq_parent.cdli_main
    
    def __init__(self,id,title,shortDescription="",comment=""):
        """init a CDLIBasket"""
        
        self.id=id
        self.title=title
        self.shortDescription=shortDescription
        self.comment=comment
 
    def getActualUserName(self):
        """get name of the actualuser"""
       
        return str(self.REQUEST['AUTHENTICATED_USER'])
  
           
    def getLastVersion(self):
        """hole letzte version"""

        ids=[]
        idsTmp= self.objectIds()
        for x in idsTmp:
            try:
                ids.append(int(x))
            except:
                pass
        ids.sort()
      
        if len(ids)==0:
            return None
        else:    
            ob=getattr(self,str(ids[-1]))

            
            return ob
   
    def getVersions(self):
        """get versions"""
        versions=self.ZopeFind(self,obj_metatypes=["CDLIBasketVersion"])
        return versions

   
    def updateObjects(self,ids,RESPONSE=None,REQUEST=None):
        """update ids, ids not in the basket the add"""
        if type(ids) is not ListType:
            ids=[ids]
       
        lastVersion=self.getLastVersion() 
        oldContent=lastVersion.content.getContent()
        newContent=[]
        
        #first copy the old
        for obj in oldContent:
            if obj[1].getId() not in ids:
                newContent.append(obj)
        #now add the new
               
        for id in ids:
            founds=self.CDLICatalog.search({'title':id})

            for found in founds:
                if found.getObject() not in oldContent:
                    #TODO: was passiert wenn, man eine Object dazufŸgt, das schon da ist aber eine neuere version
                    newContent.append((found.getObject().getLastVersion(),found.getObject()))
        

        content=newContent 
        user=self.getActualUserName()
        
        ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=newContent)
        
        obj=self._getOb(ob.getId())
        if RESPONSE:
           
            RESPONSE.redirect(obj.absolute_url())
        
        return obj
    
    def addObjectsWithVersion(self,ids,deleteOld=None,username=None,catalog=None):
        """generate a new version of the basket with objects added, 
        hier wird jedoch nicht die letzte Version jedes Files hinzugefuegt, s
        ondern ids is ein Tupel mit der Id (d.h. der p-number) und der Versionsnummer.
        """
        logging.info("add to basket (%s)"%(self.getId()))
        lastVersion=self.getLastVersion()
        
        if not catalog:
            catalog=self.CDLICatalog
            
        if lastVersion is None:
            oldContent=[]
        else:
            oldContent=lastVersion.content.getContent()

        if deleteOld:
            oldContent=[]

        newContent=[]
        added=0
       
        for id,version in ids.iteritems():
            logging.info("adding %s %s"%(id,version))
            id=id.split(".")[0] # title nur die pnumber ohne atf
           
            try:
                founds=catalog.search({'title':id})
            except:
                founds=[]
            logging.info(" found %s "%(founds))
            for found in founds:
                if found.getObject() not in oldContent:
                 
                    #TODO: was passiert wenn, man eine Object dazufŸgt, das schon da ist aber eine neuere version
                    newContent.append((found.getObject().getVersions()[version-1][1],found.getObject()))
                    added+=1

        content=oldContent+newContent
        if not username:
            logging.error("XXXXXXXXXXX %s"%repr(self))
            user=self.getActualUserName()
        else:
            user = username
            
        ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=content)
        logging.info("add to basket (%s) done"%(self.getId()))
        return added
    
    
    def addObjects(self,ids,deleteOld=None,username=None):
        """generate a new version of the basket with objects added"""
        logging.info("add to basket (%s)"%(self.getId()))
        lastVersion=self.getLastVersion()
        
        if lastVersion is None:
            oldContent=[]
        else:
            oldContent=lastVersion.content.getContent()

        if deleteOld:
            oldContent=[]

        newContent=[]
        added=0
        for id in ids:
            try:
                founds=self.CDLICatalog.search({'title':id})
            except:
                founds=[]
           
            for found in founds:
                if found.getObject() not in oldContent:
                    #TODO: was passiert wenn, man eine Object dazufŸgt, das schon da ist aber eine neuere version
                    newContent.append((found.getObject().getLastVersion(),found.getObject()))
                    added+=1

        content=oldContent+newContent
        if not username:
            user=self.getActualUserName()
        else:
            user = username
            
        ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=content)
        logging.info("add to basket (%s) done"%(self.getId()))
        return added
    
    
                
    def getContent(self):
        """print content"""
        ret=[]
        
        lv=self.getLastVersion()
        for obj in lv.content.getContent():
            logging.info("XXXXXXXXXX %s"%repr(obj))
            ret.append((obj[1].getId(),obj[0].versionNumber))
            
        return ret
        
    def getContentIds(self):
        """print basket content"""
        ret=[]
        lv=self.getLastVersion()
        for obj in lv.content.getContent():
            ret.append((obj[0].getId(),obj[1].getId()))
        
        
        return lv.getComment(),lv.getUser(),lv.getTime(),ret

    def changeBasket(self,ids,submit,RESPONSE=None,REQUEST=None):
        """change a basket"""
        if submit=="update":
            return self.updateObjects(ids,RESPONSE=RESPONSE,REQUEST=REQUEST)
        elif submit=="delete":
            return self.deleteObjects(ids,RESPONSE=RESPONSE,REQUEST=REQUEST)
            
    def deleteObjects(self,ids,RESPONSE=None,REQUEST=None):
        """delete objects"""
        
        if type(ids) is not ListType:
            ids=[ids]
       
        lastVersion=self.getLastVersion() 
        oldContent=lastVersion.content.getContent()
        newContent=[]
        for obj in oldContent:
            if obj[1].getId() not in ids:
                newContent.append(obj)
        
                
        user=self.getActualUserName()
        
        ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=newContent)
        
        if RESPONSE:
            obj=self._getOb(ob.getId())
            RESPONSE.redirect(obj.absolute_url())
        
def manage_addCDLIBasketForm(self):
    """add the CDLIBasketContainer form"""
    pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIBasket.zpt')).__of__(self)
    return pt()

def manage_addCDLIBasket(self,title,shortDescription="",comment="",RESPONSE=None):
    """add the basket"""
    
    id=str(self.getNewId())
    
    ob=CDLIBasket(id,title,shortDescription,comment)
    
    self._setObject(id, ob)
    
    if RESPONSE is not None:
        RESPONSE.redirect('manage_main')
    else:
        return ob

class CDLIBasketVersion(Implicit,Persistent,Folder):
    """version of a basket"""
    
    meta_type="CDLIBasketVersion"
    security=ClassSecurityInfo()
    
    def updateBasket(self):
        """update"""
        try:
            self._setObject('content',BasketContent(self.basketContent))
        except:
            try:
                if len(self.basketContent)>0:
                    self.content.setContent(self.basketContent)
            except:
                print "error",self.getId(),self.aq_parent.getId()
        self.basketContent=[]

        
    def containsNonActualFiles(self):
        """returns True if basket contains one or more non current files"""
        
        objs=self.getContent()
        for obj in objs:
            if not self.isActual(obj)[0]:
                return True
        return False
    
    def downloadListOfPnumbers(self):
        """download pnumbers of the basket as list"""
        
        basket_name=self.aq_parent.title
        
        ids=self.getContent() # get the list of objects
        logging.error(ids)
        ret="\n".join([x[1].getId().split(".")[0] for x in ids])
        
        self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.txt" """%basket_name)
        self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream")
        length=len(ret)
        self.REQUEST.RESPONSE.setHeader("Content-Length",length)
        self.REQUEST.RESPONSE.write(ret)    
        
    security.declareProtected('manage','downloadObjectsAsOneFile')
    def downloadObjectsAsOneFile(self,lock=None,procedure=None,REQUEST=None,check="yes",current="no"):
        """download all selected files in one file"""
        
        if self.temp_folder.downloadCounterBaskets > 10000:
            return """I am sorry, currently the server has to many requests for downloads, please come back later!"""


        if (check=="yes") and self.containsNonActualFiles():
            pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadObjectAsOneFile_check.zpt')).__of__(self)
            
            return pt(lock=lock)
            
        else:
            
            return self.downloadObjectsAsOneFileFinally(lock=lock,procedure=procedure,REQUEST=REQUEST,current="no")
        
    def downloadObjectsAsOneFileFinally(self,lock=None,procedure=None,REQUEST=None,current="no"):
        """print do the download"""

        ret=""
        lockedObjects={}

        self.temp_folder.downloadCounterBaskets+=1 
        self._p_changed=1
        transaction.get().commit()       
    
        if lock:
            
            if str(self.REQUEST['AUTHENTICATED_USER'])=='Anonymous User':
                self.temp_folder.downloadCounterBaskets-=1 
                self._p_changed=1
                transaction.get().commit()      
                self.temp_folder.downloadCounterBaskets-=1 
                self._p_changed=1
                transaction.get().commit()      
                return "please login first"

            #check if a locked object exist in the basket.
            lockedObjects={}
            for object in self.content.getContent():

                if (not str(object[1].lockedBy)=="") and (not (str(object[1].lockedBy)==str(self.REQUEST['AUTHENTICATED_USER']))):
                    lockedObjects[object[1].title]=repr(object[1].lockedBy)
                   
                    
            keys=lockedObjects.keys()
            
            
            if len(keys)>0 and (not procedure):
                self.REQUEST.SESSION['lockedObjects']=lockedObjects
                pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','lockedObjects.zpt')).__of__(self)
                
                self.temp_folder.downloadCounterBaskets-=1 
                self._p_changed=1
                transaction.get().commit()      

                return pt()
         
            elif not procedure: #keine fails gesperrt dann alle donwloaden
                procedure="downloadAll" 
        
        


        for object in self.content.getContent():
    
                if (procedure=="downloadAll") or (object[1].lockedBy=='') or (object[1].lockedBy==self.REQUEST['AUTHENTICATED_USER']):
                    if current=="no": #version as they are in the basket
                        ret+=str(object[0].getData())+"\n"
                    elif current=="yes":
                        #search current object
                        founds=self.CDLICatalog.search({'title':object[1].getId().split(".")[0]})
                        if len(founds)>0:      
                            ret+=str(founds[0].getObject().getLastVersion().getData())+"\n"
                            
                if lock and object[1].lockedBy=='':
                    object[1].lockedBy=self.REQUEST['AUTHENTICATED_USER']
        basket_name=self.aq_parent.title+"_V"+self.getId()
        
        #write basketname to header of atf file
        ret="#basket: %s\n"%basket_name+ret

        self.temp_folder.downloadCounterBaskets-=1 
        self._p_changed=1
        transaction.get().commit()      
        
        self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.atf" """%basket_name)
        self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream")
        length=len(ret)
        self.REQUEST.RESPONSE.setHeader("Content-Length",length)
        self.REQUEST.RESPONSE.write(ret)    
        return True
        
    def numberOfItems(self):
        """return anzahl der elemente im basket"""
        return self.content.numberOfItems()
    
    def getTime(self):
        """getTime"""
        #return self.bobobase_modification_time().ISO()
      
        if hasattr(self,'time'):
            return time.strftime("%Y-%m-%d %H:%M:%S",self.time)
        elif hasattr(self,'timefixed'):
            return self.timefixed
        else:
            setattr(self,'timefixed',self.bobobase_modification_time().ISO())
            return self.bobobase_modification_time().ISO()
    
    def getContent(self):
        """get Basket Content"""
        return self.content.getContent()

    
    def __init__(self,id,user,comment="",basketContent=[]):
        """ init a basket version"""
        self.id=id
        self.comment=comment
        self._setObject('content',BasketContent(basketContent))
        #self.basketContent=basketContent[0:]a
        self.user=user
        self.time=time.localtime()
        
    def getUser(self):
        """get user"""
        return self.user
    
    def getComment(self):
        """get Comment"""
        return self.comment
 
    security.declareProtected('manage','index_html')
    def index_html(self):
            """view the basket"""

            if self.REQUEST.get('change',False):
                    ob=self.aq_parent.updateObjects(self.REQUEST['change'])
                   
                    self.REQUEST.RESPONSE.redirect(ob.absolute_url())#go to new basket, because changing generates a new basket
                                        
            pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketVersionMain.zpt')).__of__(self)
            return pt()
     
    def getObjUrl(self,result):
        """getUrl of the version of the object"""
        objId=result[1].getTitle()
        founds=self.CDLICatalog.search({'title':objId})
        if len(founds)>0:
             return founds[0].getObject().getLastVersion().absolute_url()
         
        else: #assume version number
            splitted=objId.split("_")
            founds=self.CDLICatalog.search({'title':splitted[1]})        
            return founds[0].getObject().getLastVersion().absolute_url()+'/'+objId
   
def manage_addCDLIBasketVersion(self,user,comment="",basketContent=[],RESPONSE=None):
    """add a version"""
    
    #check for already existing versions
 
    lastVersion=self.getLastVersion()
    if lastVersion is None:
        newId=str(1)
    else:
        newId=str(int(lastVersion.getId())+1)
    
    ob=CDLIBasketVersion(newId,user,comment,basketContent)
    
    self._setObject(newId, ob)
    
    if RESPONSE is not None:
        RESPONSE.redirect('manage_main')
    else:
        return ob
    
class CDLIFileObject(CatalogAware,extVersionedFileObject):
    """CDLI file object"""
    
    meta_type="CDLI File Object"
    default_catalog='CDLIObjectsCatalog'
    
    security=ClassSecurityInfo()
    
 
    security.declarePublic('makeThisVersionCurrent')
    
    security.declareProtected('manage','index_html')
    def PrincipiaSearchSource(self):
           """Return cataloguable key for ourselves."""
           return str(self)
       
    def makeThisVersionCurrent_html(self):
        """form for making this version current"""
        
        pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','makeThisVersionCurrent.zpt')).__of__(self)
        return pt()                 
    def makeThisVersionCurrent(self,comment,author,RESPONSE=None):
        """copy this version to current"""
        parent=self.aq_parent
        
        
        newversion=parent.manage_addCDLIFileObject('',comment,author)
        newversion.manage_upload(self.getData())
                                        
        if RESPONSE is not None:
            RESPONSE.redirect(self.aq_parent.absolute_url()+'/history')


        return True
    
    security.declarePublic('view')
 
    def getFormattedData(self):
        """fromat text"""
        data=self.getData()
#        return re.sub("\s\#lem"," #lem",data) #remove return vor #lem
        return re.sub("#lem","       #lem",data) #remove return vor #lem
        
    def view(self):
        """view file"""
        pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','viewCDLIFile.zpt')).__of__(self)
        return pt()
    
    security.declarePublic('getPNumber')
    def getPNumber(self):
        """get the pnumber"""
        try:
                txt=re.match("&[Pp](\d*)\s*=([^\r\n]*)",self.getData()[0:])
        except:
                txt=self.getData()[0:]
                
                return "ERROR"
        try:
            return "P"+txt.group(1)
        except:
            return "ERROR"

    security.declarePublic('getDesignation')
    def getDesignation(self):
        """get the designation out of the file"""
        try:
                txt=re.match("&[Pp](\d*)\s*=([^\r\n]*)",self.getData()[0:])
        except:
                txt=self.getData()[0:]
                
                return "ERROR"
        try:
            return txt.group(2)
        except:
            return "ERROR"
        
manage_addCDLIFileObjectForm=DTMLFile('dtml/fileAdd', globals(),Kind='CDLIFileObject',kind='CDLIFileObject', version='1')

def manage_addCDLIFileObject(self,id,vC='',author='', file='',title='',precondition='', content_type='',
                             from_tmp=False,REQUEST=None):
    """Add a new File object.

    Creates a new File object 'id' with the contents of 'file'"""
 
    id=str(id)
    title=str(title)
    content_type=str(content_type)
    precondition=str(precondition)
    
    id, title = cookId(id, title, file)

    self=self.this()

    # First, we create the file without data:
    self._setObject(id, CDLIFileObject(id,title,'',content_type, precondition))
    fob = self._getOb(id)
    fob.versionComment=str(vC)
    fob.time=time.localtime()
    setattr(fob,'author',author)
    
    # Now we "upload" the data.  By doing this in two steps, we
    # can use a database trick to make the upload more efficient.

    if file and not from_tmp:
        fob.manage_upload(file)
    elif file and from_tmp:
        fob.manage_upload_from_tmp(file)
    if content_type:
        fob.content_type=content_type

    logging.debug("reindex1: %s"%repr(self))
    self.reindex_object()
    logging.debug("reindex2: %s in %s"%(repr(fob), repr(fob.default_catalog)))
    fob.reindex_object()

    if REQUEST is not None:
        REQUEST['RESPONSE'].redirect(self.absolute_url()+'/manage_main')
    

class CDLIFile(extVersionedFile,CatalogAware):
    """CDLI file"""
    
    security=ClassSecurityInfo()
    meta_type="CDLI file"
    default_catalog='CDLICatalog'
    security.declareProtected('manage','index_html')
    #security.declarePublic('history')
    def getLastVersionData(self):
        """get last version data"""
        return self.getLastVersion().getData()

    def getLastVersionFormattedData(self):
        """get last version data"""
        return self.getLastVersion().getFormattedData()

    #security.declarePublic('history')
    
    
    def history(self):
        """history"""  

        ext=self.ZopeFind(self.aq_parent,obj_ids=["history_template.html"])
        if ext:
            return getattr(self,ext[0][1].getId())()
        
        pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','versionHistory')).__of__(self)
        return pt()


    def getBasketFromId(self,basketid, context=None):
        """get basket from id"""

        if not context:
            context=self
            
        for basket in self.ZopeFind(context,obj_metatypes=["CDLIBasket"]):
            if basket[0]==basketid:
                return basket[1]
        else:
            None

 
    def isContainedInBaskets(self,context=None):
        """check is this file is part of any basket
        @param context: (optional) necessessary if CDLIBasketCatalog is not an (inherited) attribute of self, context.CDLIBasketCatalog
                        has to exist.
        """

        if not context:
            context=self
        
        ret=[]
        for x in context.CDLIBasketCatalog.search({'getFileNamesInLastVersion':self.getId()}):
            #if the basket x is deleted it seemes to be that x is sometimes still in the Catalog, why?
            try:
                ret.append(x.getObject())
            except:
                pass
        return ret
        #return [x.getObject() for x in context.CDLIBasketCatalog.search({'getFileNamesInLastVersion':self.getId()})]
        
        
    def addCDLIFileObjectForm(self):
        """add a new version"""
        
        if str(self.REQUEST['AUTHENTICATED_USER']) in ["Anonymous User"]:
            return "please login first"
        if (self.lockedBy==self.REQUEST['AUTHENTICATED_USER']) or (self.lockedBy==''):
            out=DTMLFile('dtml/fileAdd', globals(),Kind='CDLIFileObject',kind='CDLIFileObject',version=self.getVersion()).__of__(self)
            return out()
        else:
            return "Sorry file is locked by somebody else"
        
    def manage_addCDLIFileObject(self,id,vC,author,
                                 file='',title='',
                                 precondition='', 
                                 content_type='',
                                 changeName='no',newName='', 
                                 come_from=None,
                                 from_tmp=False,RESPONSE=None):
        """add"""
      
        try: #TODO: der ganze vC unsinn muss ueberarbeitet werden
            vC=self.REQUEST['vC']
        except:
            pass
        
        
        if changeName=="yes":
            filename=file.filename
            self.title=filename[max(filename.rfind('/'),
                        filename.rfind('\\'),
                        filename.rfind(':'),
                        )+1:]


        if not newName=='':
            self.title=newName[0:]
        
        

   
        positionVersionNum=getattr(self,'positionVersionNum','front')
        
        if positionVersionNum=='front':
            id="V%i"%self.getVersion()+"_"+self.title
        else:
            tmp=os.path.splitext(self.title)
            if len(tmp)>1:
                id=tmp[0]+"_V%i"%self.getVersion()+tmp[1]
            else:
                id=tmp[0]+"_V%i"%self.getVersion()
            
      
        manage_addCDLIFileObject(self,id,vC,author,file,id,precondition, content_type,from_tmp=from_tmp)
        #objs=self.ZopeFind(self,obj_ids=[id])[0][1].setVersionNumber(int(self.getVersion()))
        objs=getattr(self,id).setVersionNumber(int(self.getVersion()))
        try:
          #FIXME: wozu ist das gut?
          self.REQUEST.SESSION['objID_parent']=self.getId()
        except:
          pass
  
        if RESPONSE:
    
            obj=self.ZopeFind(self,obj_ids=[id])[0][1]
            if obj.getSize()==0:
                self.REQUEST.SESSION['objID']=obj.getId()
                pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','errorUploadFile')).__of__(self)
                return pt()

            else:
                if come_from and (come_from!=""):
                                        RESPONSE.redirect(come_from+"?change="+self.getId())
                else:
                    RESPONSE.redirect(self.REQUEST['URL2']+'?uploaded=%s'%self.title)

        else:
            return self.ZopeFind(self,obj_ids=[id])[0][1]
        
        
def manage_addCDLIFileForm(self):
    """interface for adding the OSAS_root"""
    pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIFile.zpt')).__of__(self)
    return pt()

def manage_addCDLIFile(self,id,title,lockedBy, author=None, RESPONSE=None):
    """add the OSAS_root"""
    newObj=CDLIFile(id,title,lockedBy,author)
                                        
    tryToggle=True
    tryCount=0
    
  

    self._setObject(id,newObj)                  
    getattr(self,id).reindex_object()
        
    if RESPONSE is not None:
        RESPONSE.redirect('manage_main')

def checkUTF8(data):
    """check utf 8"""
    try:
        data.encode('utf-8')
        return True
    except:
        return False
    

def checkFile(filename,data,folder):
    """check the files"""
    # first check the file name
    fn=filename.split(".") # no extension

    if not fn[0][0]=="P":
        return False,"P missing in the filename"
    elif len(fn[0])!=7:
        return False,"P number has not the right length 6"
    elif not checkUTF8(data):
        return False,"not utf-8"
    else:
        return True,""
    
    
def splitatf(fh,dir=None,ext=None):
    """split it"""
    ret=None
    nf=None
    i=0

    if (type(fh) is StringType) or (type(fh) is UnicodeType):
        iter=fh.split("\n")
    else:
        iter=fh.readlines()
        
    for lineTmp in iter:
        lineTmp=lineTmp.replace(codecs.BOM_UTF8,'') # make sure that all BOM are removed..
        for line in lineTmp.split("\r"):
            #logging.log("Deal with: %s"%line)
            if ext:
                i+=1
                if (i%100)==0:
                    ext.result+="."
                if i==10000:
                    i=0
                    ext.result+="<br>"
            #check if basket name is in the first line
            if line.find("#atf basket")>=0: #old convention
                ret=line.replace('#atf basket ','')
                ret=ret.split('_')[0]
            elif line.find("#basket:")>=0: #new convention
                ret=line.replace('#basket: ','')
                ret=ret.split('_')[0]

            else:
                if (len(line.lstrip())>0) and (line.lstrip()[0]=="&"): #newfile
                    if nf:
                        nf.close() #close last file


                    filename=line[1:].split("=")[0].rstrip()+".atf"
                    if dir:
                        filename=os.path.join(dir,filename)
                    nf=file(filename,"w")
                    logging.info("open %s"%filename)
                if nf:    
                    nf.write(line.replace("\n","")+"\n")

    try:        
        nf.close()
    except:
        pass
    
    if not((type(fh) is StringType) or (type(fh) is UnicodeType)):
        fh.close()
    return ret,len(os.listdir(dir))


class CDLIFileFolder(extVersionedFileFolder):
    """CDLI File Folder"""
    
    security=ClassSecurityInfo()
    meta_type="CDLI Folder"
    filesMetaType=['CDLI file']
    folderMetaType=['CDLI Folder']
    default_catalog='CDLICatalog'
    defaultFileCatalog=default_catalog #wenn dieses definiert ist, wird beim hinzufŸgen einer neuen version eines files dieser catalog neuiniziert
    #downloadCounter=0 # counts how many download for all files currently run, be mehr als 5 wird verweigert.
    tmpStore2={}
    def setTemp(self,name,value):
        """set tmp"""

        setattr(self,name,value)
                                        
                                       
    def delete(self,ids):
        """delete this file, i.e. move into a trash folder"""
             
        found=self.ZopeFind(self,obj_ids=['.trash'])
        
        if len(found)<1:
            manage_addCDLIFileFolder(self, '.trash',title="Trash")
            trash=self._getOb('.trash')
        else:
            trash=found[0][1]
        
        if type(ids) is not ListType:
            ids=[ids]
        cut=self.manage_cutObjects(ids)
        trash.manage_pasteObjects(cut)
        
    def getVersionNumbersFromIds(self,ids):
        """get the numbers of the current versions of documents described by their ids"""
        
        ret=[]
        searchStr=" OR ".join(ids)
        
        founds=self.CDLICatalog.search({'title':searchStr})
        
        for found in founds:
            lastVersion=found.getObject().getLastVersion()
            ret.append((found.getId,lastVersion))
        
        return ret
    
    def getFile(self,fn):
        """get the content of the file fn"""
        founds=self.CDLICatalog.search({'title':fn})
        if not founds:
            return "" 
        else:
            obj=founds[0].getObject().getLastVersion()

            return obj.getData()[0:] 
    
    def checkCatalog(self,fn):
        """check if fn is in the catalog"""
        #TODO add checkCatalog
         
        
                                   
    def findObjectsFromListWithVersion(self,list,author=None):
        """find objects from a list with versions
        @param list: list of tuples  (cdliFile,version)
        """
        
       
       
        #self.REQUEST.SESSION['fileIds']=list#store fieldIds in session for further usage
        #self.REQUEST.SESSION['searchList']=self.REQUEST.SESSION['fileIds']
        
        
        pt=getattr(self,'filelistVersioned.html')
            
        return pt(search=list,author=author)
    
    
    def getAllPNumbers(self):
        """get a list of all files (resp their p-numbers) stored"""
        
        ret=[x.getId for x in  self.CDLICatalog()]
     
        return ret
    
    def findObjectsFromList(self,enterList=None,display=False,start=None,upload=None,list=None,basketName=None,numberOfObjects=None,RESPONSE=None):
        """findObjectsFromList (, TAB oder LINE separated)"""
                                       
        
        if upload: # list from file upload
            txt=upload.read()
                                       
        if enterList:
            txt=enterList
            
        if upload or enterList:
            txt=txt.replace(",","\n")
            txt=txt.replace("\t","\n")
            txt=txt.replace("\r","\n")
            idsTmp=txt.split("\n")
            ids=[]
            for id in idsTmp: # make sure that no empty lines
                idTmp=id.lstrip().rstrip()
                if len(idTmp)>0:
                    
                    ids.append(idTmp)
                    
            #self.REQUEST.SESSION['ids']=" OR ".join(ids)

            pt=getattr(self,'filelist.html')
            self.REQUEST.SESSION['searchList']=ids
            return pt(search=ids)
        
        if basketName:
            #TODO: get rid of one of these..
            
            pt=getattr(self,'filelist.html')
            return pt(basketName=basketName,numberOfObjects=numberOfObjects)
        
        if list is not None: # got already a list
            ret=[]
            for fileId in list:
                if fileId.find("*"): #check for wildcards
                        fileId=fileId
                elif len(fileId.split("."))==1:
                        fileId=fileId+".atf"
                
                ret+=self.CDLICatalog({'title':fileId})
            #TODO: get rid of one of these..
            ids=[x.getObject().getId() for x in ret]
            self.REQUEST.SESSION['fileIds']=ids#store fieldIds in session for further usage
            self.REQUEST.SESSION['searchList']=self.REQUEST.SESSION['fileIds']
            
            if display:
                pt=getattr(self,'filelist.html')
                
                return pt(search=ids)
            else:      
                return ret
        
        
        
        if start:
            RESPONSE.redirect("filelist.html?start:int="+str(start))
                                       

    security.declareProtected('Manage','createAllFilesAsSingleFile')
    def createAllFilesAsSingleFile(self,RESPONSE=None):
        """download all files"""
        
        def sortF(x,y):
            return cmp(x[0],y[0])
        
        catalog=getattr(self,self.default_catalog)
        #tf,tfilename=mkstemp()
        if not hasattr(self.temp_folder,'downloadCounter'):
            self.temp_folder.downloadCounter=0

        if getattr(self.temp_folder,'downloadCounter',0) > 5:
            return """I am sorry, currently the server has to many requests for downloads, please come back later!"""

        self.temp_folder.downloadCounter+=1
        self._p_changed=1
        transaction.get().commit()
       
        list=[(x.getId,x) for x in catalog()]
        list.sort(sortF)
        

        
        RESPONSE.setHeader("Content-Disposition","""attachement; filename=%s"""%"all.atf")
        RESPONSE.setHeader("Content-Type","application/octet-stream")
        tmp=""
        for l in list:
            obj=l[1].getObject()
            
            if obj.meta_type=="CDLI file":
                
                #os.write(tf,obj.getLastVersion().data)
                if RESPONSE:
                    RESPONSE.write(obj.getLastVersion().getData()[0:])
                    RESPONSE.write("\n")
                self.temp_folder.downloadCounter-=1 
                self._p_changed=1
        transaction.get().commit()
        #os.close(tf)
        #RESPONSE.redirect(self.absolute_url()+"/downloadFile?fn="%tfilename)
        return True
    
    def downloadFile(self,fn):
        """download fn - not used yet"""
        self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename=%s"""%self.getLastVersion().getId())
        self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream")
        self.REQUEST.RESPONSE.write(file(fn).read())
        
      
                
    def hasParent(self):
        """returns true falls subfolder"""
      
        if self.aq_parent.meta_type in self.folderMetaType:
            return True
        else:
            return False
        
    def getFolders(self):
        """get all subfolders"""
        ret=[]
        folders=self.ZopeFind(self,obj_metatypes=self.folderMetaType)
        for folder in folders:
            ret.append((folder[1],
                        len(self.ZopeFind(folder[1],obj_metatypes=self.folderMetaType)),
                        len(self.ZopeFind(folder[1],obj_metatypes=self.filesMetaType))
                        ))
        return ret
    
            
    security.declareProtected('manage','index_html')
    def index_html(self):
        """main"""
        ext=self.ZopeFind(self,obj_ids=["index.html"])
        if ext:
            return ext[0][1]()
        
        pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','CDLIFileFolderMain')).__of__(self)
        return pt()
    
    
manage_addCDLIFileFolderForm=DTMLFile('dtml/folderAdd', globals())

    
def manage_addCDLIFileFolder(self, id, title='',
                     createPublic=0,
                     createUserF=0,
                     REQUEST=None):
    """Add a new Folder object with id *id*.

    If the 'createPublic' and 'createUserF' parameters are set to any true
    value, an 'index_html' and a 'UserFolder' objects are created respectively
    in the new folder.
    """
    ob=CDLIFileFolder()
    ob.id=str(id)
    ob.title=title
    self._setObject(id, ob)
    ob=self._getOb(id)

    checkPermission=getSecurityManager().checkPermission

    if createUserF:
        if not checkPermission('Add User Folders', ob):
            raise Unauthorized, (
                  'You are not authorized to add User Folders.'
                  )
        ob.manage_addUserFolder()

  
    if REQUEST is not None:
        return self.manage_main(self, REQUEST, update_menu=1)
    
class CDLIRoot(Folder):
    """main folder for cdli"""
    
    meta_type="CDLIRoot"
    downloadCounterBaskets=0# counts the current basket downloads if counter > 10 no downloads are possible
    
    def deleteFiles(self,ids):
        """delete files (resp. move into .trash folder)"""
        # find or generete trash folder
        
        found=self.ZopeFind(self,obj_ids=['.trash'])
        
        if len(found)<1:
            manage_addCDLIFileFolder(self, '.trash',title="Trash")
            trash=self._getOb('.trash')
        else:
            logging.info(found)
            trash=found[0][1]
        
        
        for id in ids:
            founds=self.CDLICatalog.search({'title':id.split(".")[0]})
            if founds:
                logging.info(founds)
                folder=founds[0].getObject().aq_parent #get the parent folder of the object
                logging.info(folder)
                cut=folder.manage_cutObjects([founds[0].getId]) #cut it out
                trash.manage_pasteObjects(cut)  #paste it in the trash


    def findWordRegExp(self,indexName,searchTerm):
        """find all words in index which match regexp in SearchTerm
        @param indexName: name of the index to be searched in
        @param searchTerm: word to be searched"""
        
        ret=[]
        for x in self.lineIndexes[indexName].iterkeys():
            if re.match(searchTerm,x):
                ret.append(x)
        return ret
    
    def searchRegExpInLineIndexDocs(self,indexName,searchTerm):
        """search in inLineIndex with regexp
        @param indexName: name of the index to be searched in
        @param searchTerm: term to be searched
        """
        if not searchTerm:
            return []
        ret=[]
        words=self.findWordRegExp(indexName,searchTerm) # suche nach allen Treffern
        logging.info("wd:%s"%words)
        for word in words:

            ret+=self.searchInLineIndexDocs(indexName,word)
	

        x= unique(ret)
	logging.info("words_done")
        return x

    def showInLineIndex(self):
        """get the index for debug purposes"""
        print "show"
        for key in self.lineIndexes.keys():
            logging.info("index:%s"%key)
            for x in self.lineIndexes[key].iterkeys():
                logging.info("word:%s"%repr(x))
                #for y in self.lineIndex[x].iterkeys():
                #    print "doc",repr(y),repr(self.lineIndex[x][y])
                    
        return self.lineIndexes
        
    def searchInLineIndexDocs(self,indexName,word,uniq=True,regExp=False):
        """search occurences in an index
        @param indexName: name of the index to be searched in
        @param word: word to be searched
        @param unique: (optional) unify the list of results
        @param regExp: (optional) use regular expressions
        """

        if regExp:
            return self.searchRegExpInLineIndexDocs(indexName,word)
        
        try:    
                
                lst=list(self.lineIndexes[indexName].get(word).keys())
        except:
            logging.error("error: searchInLineIndexDocs (%s %s)"%(sys.exc_info()[0:2]))
            lst=[]
        if uniq:
            return unique(lst)
        else:
            return lst
        
    def getLinesFromIndex(self,indexName,word,doc,regExp=False):
        """return all lines from a document where word is found
        @param indexName: Name of the index
        @param word: word to be searched
        @param doc: name of the document (usuallay the p-number)
        @param regExp: (optional) use regExp       
        """
        
        if not regExp:
            return self.lineIndexes[indexName].get(word)[doc]
        else: # wenn regexp, suche welches word
            for w in self.findWordRegExp(indexName,word):
                if self.lineIndexes[indexName].get(w): # ein word in im dex gefunden
                    try:    
                        dc=self.lineIndex[indexName].get(word)[doc]
                        return dc # und ein document dann gib es zurueck
                    except:
                         pass #andernfalls weiter
                     
    def cleanInLineIndex(self,indexName):
        """empty an InlineIndex
        @param indexName: name of the index
        """
        for x in list(self.lineIndexes[indexName].keys()):
            del(self.lineIndexes[indexName][x])
        print [x for x in self.lineIndexes[indexName].keys()]
     
        return "ok"
    
    def storeInLineIndex(self,indexName,key,value):
        """store in index, key is normally a word or grapheme 
        and value is a tuple (documentname, line) where the word can be found
        @param indexName: name of the index
        @param key: key in index
        @param value: value in index, value is a tuple (document name, line)
        """
        logging.error("indexing: %s %s"%(indexName,key))
        if (not hasattr(self,'lineIndexes')):
      
            self.lineIndexes={}
            
        if self.lineIndexes.get(indexName,None) is None:
            #index exisitiert noch nicht dann anlegen
            
            self.lineIndexes[indexName]=OOBTree()
        lis=self.lineIndexes
        li=lis[indexName]
        
        if li.has_key(key):

#            if li[key].has_key(value[0]) and (not (value[1] in li[key][value[0]])):
            if li[key].has_key(value[0]):
                tmp=li[key][value[0]]
                tmp.append(value[1]) # add it if now in the array
                li[key][value[0]]=tmp[0:]
            else:
                li[key][value[0]]=[value[1]] # new array for lines
                
        else:
            
            li[key]=OOBTree()# new btree for lines
            li[key][value[0]]=[value[1]] 
                    
        
        self.lineIndexes=lis
     
        transaction.get().commit()
        

    def showFile(self,fileId,wholePage=False):
        """show a file
        @param fileId: P-Number of the document to be displayed
        """
        f=self.CDLICatalog({'title':fileId})
        if not f:
            return ""
        
        if wholePage:
            logging.info("whole")
            return f[0].getObject().getLastVersion().view()
        else:
            return f[0].getObject().getLastVersionFormattedData()
    

    def showWordInFile(self,fileId,word,lineList=None,regExp=True,indexName=""):
        """get lines with word  fromFileId"""
        
        file=self.showFile(fileId)
	logging.info("regEXP %s"%regExp)
        ret=[]
        if regExp: # wenn regexp dann generiere alle worte aus der list die der regexp entsprechen
            wordlist=self.findWordRegExp(indexName,word)
        else:
            wordlist=[word]
        
        for line in file.split("\n"):
            found=False
            for word in wordlist:
		try: # just a hack because of possible unicode errors in line
                 if line.find(word)>-1:
                        if lineList: #liste of moeglichen Zeilennummern
                                num=line.split(".")[0] #Zeilenummer ist alles vor dem . in der Zeile

                                if num in lineList: 

                                        ret.append(line)
                        else: # nimm alles ohne line check
                                ret.append(line)
                        
                        break;
		except:
			pass
        return ret

    def tagWordInFile(self,fileId,word,lineList=None,regExp=True,indexName=""):
        """get lines with word  fromFileId"""
        
        file=self.showFile(fileId)
        tagStr=u'<span class="found">%s</span>'
        ret=[]
        
        if regExp: # wenn regexp dann generiere alle worte aus der list die der regexp entsprechen
            wordlist=self.findWordRegExp(indexName,word)
        else:
            wordlist=[word]
            
        for line in file.split("\n"):
            line = unicodify(line)
            found=False
            for word in wordlist:
                if line.find(word)>-1: #word ist gefunden dann makiere und breche die Schleife ab
                        if lineList: #liste of moeglichen Zeilennummern
                                num=line.split(".")[0] #Zeilenummer ist alles vor dem . in der Zeile

                                if num in lineList: 

                                        ret.append(line.replace(word,tagStr%word))
                        
                        else: # nimm alles ohne line check
                                ret.append(line.replace(word,tagStr%word))
                        found=True
                        break
            if not found: #word wurde nicht gefunden keine makierung
                        ret.append(line)
                        
        return u'<br>\n'.join(ret)

    def URLquote(self,str):
        """quote url"""
        return urllib.quote(str)
    
    def URLunquote(self,str):
        """unquote url"""
        return urllib.unquote(str)
    
    def URLquote_plus(self,str):
        """quote url"""
        return urllib.quote_plus(str)
    
    def URLunquote_plus(self,str):
        """unquote url"""
        return urllib.unquote_plus(str)
    
    
    def forceunlock(self):
        "break all locks"
        ret=[]
        for f in self.ZopeFind(self,obj_metatypes="CDLI file",search_sub=1):
           un=f[1].forceunlock()

           if un and un !="":
               ret.append((f[0],un))

        return ret
                                        

    def getChangesByAuthor(self,author,n=100):
        """getChangesByAuthor"""
        zcat=self.CDLIObjectsCatalog
        res=zcat({'lastEditor':author,
                     'sort_on':'getTime',
                     'sort_order':'descending',
                     'sort_limit':n})[:n ]
                       
        return res
    
    def getChangesByAuthor_html(self,author,n=100):
        """html output for changes by author"""
        tmp={}
        list=[]                         
        for x in self.getChangesByAuthor(author):
           nr=x.getObject().getVersionNumber()
           id=x.getObject().aq_parent.getId()
           #hinzufuegen, wenn Version neuer als die 
           if tmp.get(id,(0,0))[1] < nr:
                tmp[id]=(x.getObject().aq_parent,nr)

     
        return self.cdli_main.findObjectsFromListWithVersion(list=tmp.values(),author=author)           
        
    def getLastChanges(self,n=100):
        """get the last n changes""" 
        n=int(n)                   
        zcat=self.CDLICatalog
        return zcat({'sort_on':'getLastChangeDate',
                     'sort_order':'descending',
                     'sort_limit':n})[:n ]
     
    
    def getLastChanges_html(self,n=100):
        """get the last n changes"""
        list = [x.getId for x in self.getLastChanges(n)]
        return self.cdli_main.findObjectsFromList(list=list,display=True)
                                       
    def refreshTxt(self,txt="",threadName=None):
        """txt fuer refresh"""
  
        return """ 2;url=%s?repeat=%s """%(self.absolute_url()+txt,threadName)

    
    def getResult(self,threadName=None):
       """result of thread"""
       try:
        return self._v_uploadATF[threadName].getResult()
       except:
        return "One moment, please"
    
        
    def checkThreads(self):
        """check threads"""
        ret="<html><body>"
        for thread in threading.enumerate():
           ret+="<p>%s (%s): %s</p>"%(repr(thread),thread.getName(),thread.isAlive())
       
        return ret
                                       
                                           
    def uploadATFRPC(self,data,username):
        """upload an atffile via xml-rpc"""
        uploader=uploadATFThread()
        
        #generate an random id for the upload object
        from random import randint
        if (not self.REQUEST.SESSION.get('idTmp',None)):

            idTmp=str(randint(0,1000000000))
            self.REQUEST.SESSION['idTmp']=idTmp
        else:
            idTmp=self.REQUEST.SESSION.get('idTmp',None)
            
        
        uploader.set(data,0,username,idTmp)
        
        stObj=uploader.run()
        
        processor=uploadATFfinallyThread()
        
        basketname=stObj.returnValue['basketNameFromFile']
        
        processor.set("uploadchanged",basketname=basketname,SESSION=stObj.returnValue,username=username,serverport=self.REQUEST['SERVER_PORT'])
        
        processor.run()
        
        
        return generateXMLReturn(stObj.returnValue)
        
    def uploadATF(self,repeat=None,upload=None,basketId=0,RESPONSE=None):
        """upload an atf file / basket file"""
        #self._v_uploadATF.returnValue=None
        
        #generate an random id for the upload thread
        from random import randint
        if (not self.REQUEST.SESSION.get('idTmp',None)):

            idTmp=str(randint(0,1000000000))
            self.REQUEST.SESSION['idTmp']=idTmp
        else:
            idTmp=self.REQUEST.SESSION.get('idTmp',None)
            
    
        threadName=repeat
        if not threadName or threadName=="":
            #new thread not called from the waiting page
            tmpVar=False
       
            thread=uploadATFThread()
            threadName=thread.getName()[0:]                                
            if (not hasattr(self,'_v_uploadATF')):
                   self._v_uploadATF={}
                                       
            self._v_uploadATF[threadName]=thread
            #self._xmltrans.start()
            #thread=Thread(target=self._v_uploadATF)
            logging.info("set thread. extern")
            self._v_uploadATF[threadName].set(upload,basketId,self.REQUEST['AUTHENTICATED_USER'],idTmp,serverport=self.REQUEST['SERVER_PORT'])
            #thread.start()
            logging.info("start thread. extern")
            self._v_uploadATF[threadName].start()

            
            self.threadName=self._v_uploadATF[threadName].getName()[0:]
            wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])

            if wait_template:
                return wait_template[0][1]()
            pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self)
            return pt(txt='/uploadATF',threadName=threadName)
            #_v_xmltrans.run()
            
        else:
            #recover thread, if lost
            if (not hasattr(self,'_v_uploadATF')):
               self._v_uploadATF={}
            if not self._v_uploadATF.get(threadName,None):
                 for thread in threading.enumerate():
                         if threadName == thread.getName():
                                       self._v_uploadATF[threadName]=thread
                                       
            if self._v_uploadATF.get(threadName,None) and (not self._v_uploadATF[threadName].returnValue):
        

                wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
                if wait_template:
                        return wait_template[0][1]()
                
                pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self)

                return pt(txt='/uploadATF',threadName=threadName)
                
            else:
                tmp=getattr(self.temp_folder,idTmp).returnValue
 
                pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadCheck.zpt')).__of__(self)

                return pt(changed=tmp['changed'],lockerrors=tmp['lockerrors'],errors=tmp['errors'],dir=tmp['dir'],newPs=tmp['newPs'],basketLen=tmp['basketLen'],numberOfFiles=tmp['numberOfFiles'],
                  basketNameFromId=tmp['basketNameFromId'],basketNameFromFile=tmp['basketNameFromFile'],basketId=tmp['basketId'])
                     
    def redoUpload(self,threadName):
       """redo the upload"""
       tmp=self.cdli_main.tmpStore2[threadName]
       pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadCheck.zpt')).__of__(self)
       return pt(changed=tmp['changed'],lockerrors=tmp['lockerrors'],errors=tmp['errors'],dir=tmp['dir'],newPs=tmp['newPs'],basketLen=tmp['basketLen'],numberOfFiles=tmp['numberOfFiles'],
                  basketNameFromId=tmp['basketNameFromId'],basketNameFromFile=tmp['basketNameFromFile'],basketId=tmp['basketId'])
                 
    def uploadATFfinally(self,procedure='',comment="",basketname='',unlock=None,repeat=None,RESPONSE=None):
        """nowupload the files"""
       
       
       
        threadName=repeat
        if not threadName or threadName=="":
            thread=uploadATFfinallyThread()
            threadName=thread.getName()[0:]

            if (not hasattr(self,'_v_uploadATF')):
                                self._v_uploadATF={}


            self._v_uploadATF[threadName]=thread

            idTmp=self.REQUEST.SESSION['idTmp']
            stObj=getattr(self.temp_folder,idTmp)
            self._v_uploadATF[threadName].set(procedure,comment=comment,basketname=basketname,unlock=unlock,SESSION=stObj.returnValue,username=self.REQUEST['AUTHENTICATED_USER'],serverport=self.REQUEST['SERVER_PORT'])

            self._v_uploadATF[threadName].start()

            
            
            wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])

            if wait_template:
                return wait_template[0][1]()
            pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self)

            return pt(txt='/uploadATFfinally',threadName=threadName)
            #_v_xmltrans.run()
        
        else:
            #recover thread, if lost
            if not hasattr(self,'_v_uploadATF'):
               self._v_uploadATF={}
            if not self._v_uploadATF.get(threadName,None):
                 for thread in threading.enumerate():
                         if threadName == thread.getName():
                                       self._v_uploadATF[threadName]=thread
                                       
            if self._v_uploadATF.get(threadName,None) and (self._v_uploadATF[threadName] is not None) and (not self._v_uploadATF[threadName].end) :

                wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
                if wait_template:
                        return wait_template[0][1]()
                
                pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self)
                return pt(txt='/uploadATFfinally',threadName=threadName)
            else:
              self.REQUEST.SESSION['idTmp']=None
              if RESPONSE is not None:
                  RESPONSE.redirect(self.absolute_url())

    def importFiles(self,comment="",author="" ,folderName="/Users/dwinter/atf", files=None,ext=None):
        """import files"""
        root=self.cdli_main
        count=0
        if not files:
            files=os.listdir(folderName)
            
        for f in files:
            folder=f[0:3]
            f2=f[0:5]
            obj=self.ZopeFind(root,obj_ids=[folder])
            if ext:
  
                ext.result="<p>adding: %s </p>"%f+ext.result
            if not obj:
                manage_addCDLIFileFolder(root,folder,folder)
                fobj=getattr(root,folder)
                #transaction.get().commit()                           
            else:
                fobj=obj[0][1]
            
            obj2=fobj.ZopeFind(fobj,obj_ids=[f2])
        
            if not obj2:
                manage_addCDLIFileFolder(fobj,f2,f2)
                fobj2=getattr(fobj,f2)
        
            else:
                fobj2=obj2[0][1]
              
            file2=os.path.join(folderName,f)  
            id=f
            manage_addCDLIFile(fobj2,f,'','')
            id=f
            ob=fobj2._getOb(f)
            ob.title=id
            
            manage_addCDLIFileObject(ob,id,comment,author,file2,content_type='',from_tmp=True)
            self.CDLICatalog.catalog_object(ob)
            #self.CDLICatalog.manage_catalogFoundItems(obj_ids=[id],search_sub=1)
            #self.CDLICatalog.manage_catalogObject(self.REQUEST, self.REQUEST.RESPONSE, 'CDLICatalog', urlparse.urlparse(ob.absolute_url())[1])
            count+=1

            if count > 1000:
                print "committing"
                transaction.get().commit()
                count=0
            transaction.get().commit()
        return "ok"
         

manage_addCDLIRootForm=DTMLFile('dtml/rootAdd', globals())

    
def manage_addCDLIRoot(self, id, title='',
                     createPublic=0,
                     createUserF=0,
                     REQUEST=None):
    """Add a new Folder object with id *id*.

    If the 'createPublic' and 'createUserF' parameters are set to any true
    value, an 'index_html' and a 'UserFolder' objects are created respectively
    in the new folder.
    """
    ob=CDLIRoot()
    ob.id=str(id)
    ob.title=title
    try:
        self._setObject(id, ob)
    except:
        pass
    ob=self._getOb(id)

    checkPermission=getSecurityManager().checkPermission

    if createUserF:
        if not checkPermission('Add User Folders', ob):
            raise Unauthorized, (
                  'You are not authorized to add User Folders.'
                  )
        ob.manage_addUserFolder()

  
    if REQUEST is not None:
        return self.manage_main(self, REQUEST, update_menu=1)