Return to cdli_files.py CVS log

Up to [Repository] / cdli

minorCVS: ----------------------------------------------------------------------

    1: """CDLI extensions of the filearchive"""    
    2: from Products.versionedFile.extVersionedFile import *
    3: from Products.ZCatalog.CatalogPathAwareness import CatalogAware
    4: from tempfile import mkstemp,mkdtemp    
    5: import os.path
    6: import os
    7: from types import *
    8: import urlparse
    9: import urllib
   10: import cgi
   11: from OFS.OrderedFolder import OrderedFolder
   12: from OFS.SimpleItem import SimpleItem
   13: import time
   14: from OFS.Folder import manage_addFolder
   15: import re
   16: from AccessControl import ClassSecurityInfo
   17: from Acquisition import Implicit
   18: from Globals import Persistent
   19: from threading import Thread
   20: from ZPublisher.HTTPRequest import HTTPRequest
   21: from ZPublisher.HTTPResponse import HTTPResponse
   22: from ZPublisher.BaseRequest import RequestContainer
   23: import threading
   24: from BTrees.OOBTree import OOBTree, OOTreeSet
   25: import logging
   26: import transaction
   27: import copy
   28: import codecs
   29: import sys
   30: from BTrees.IOBTree import IOBTree 
   31: import cdliSplitter
   32: from sets import Set
   33: import md5
   34: from DownloadBasket import DownloadBasketFinallyThread
   35:                                        
   36: def makelist(mySet):
   37:         x = list(mySet)
   38:         x.sort()
   39:         return x
   40:     
   41: def unicodify(s):
   42:     """decode str (utf-8 or latin-1 representation) into unicode object"""
   43:     if not s:
   44:         return u""
   45:     if isinstance(s, str):
   46:         try:
   47:             return s.decode('utf-8')
   48:         except:
   49:             return s.decode('latin-1')
   50:     else:
   51:         return s
   52: 
   53: def utf8ify(s):
   54:     """encode unicode object or string into byte string in utf-8 representation.
   55:        assumes string objects to be utf-8"""
   56:     if not s:
   57:         return ""
   58:     if isinstance(s, str):
   59:         return s
   60:     else:
   61:         return s.encode('utf-8')
   62: 
   63: def formatAtfHtml(l):
   64:     """escape special ATF characters for HTML"""
   65:     if not l:
   66:         return ""
   67: 
   68:     # replace &
   69:     l = l.replace('&','&')
   70:     # replace angular brackets
   71:     l = l.replace('<','&lt;')
   72:     l = l.replace('>','&gt;')
   73:     return l
   74: 
   75: def formatAtfLineHtml(l, nolemma=True):
   76:     """format ATF line for HTML"""
   77:     if not l:
   78:         return ""
   79: 
   80:     if nolemma:
   81:         # ignore lemma lines
   82:         if l.lstrip().startswith('#lem:'):
   83:             return ""
   84:     
   85:     return formatAtfHtml(l)
   86: 
   87: 
   88: 
   89: def formatAtfFullLineNum(txt, nolemma=True):
   90:     """format full line numbers in ATF text"""
   91:     # surface codes
   92:     surfaces = {'@obverse':'obv',
   93:                 '@reverse':'rev',
   94:                 '@surface':'surface',
   95:                 '@edge':'edge',
   96:                 '@left':'left',
   97:                 '@right':'right',
   98:                 '@top':'top',
   99:                 '@bottom':'bottom',
  100:                 '@face':'face',
  101:                 '@seal':'seal'}
  102: 
  103:     if not txt:
  104:         return ""
  105:     
  106:     ret = []
  107:     surf = ""
  108:     col = ""
  109:     for line in txt.splitlines():
  110:         line = unicodify(line)
  111:         if line and line[0] == '@':
  112:             # surface or column
  113:             words = line.split(' ')
  114:             if words[0] in surfaces:
  115:                 surf = line.replace(words[0],surfaces[words[0]]).strip()
  116:             
  117:             elif words[0] == '@column':
  118:                 col = ' '.join(words[1:])
  119:             
  120:         elif line and line[0] in '123456789':
  121:             # ordinary line -> add line number
  122:             line = "%s:%s:%s"%(surf,col,line)
  123:             
  124:         ret.append(line)
  125:     
  126:     return '\n'.join(ret)
  127:             
  128:             
  129: def generateXMLReturn(hash):
  130:     """erzeugt das xml file als returnwert fuer uploadATFRPC"""
  131: 
  132:     ret="<return>"
  133:     
  134:     ret+="<errors>"
  135:     for error in hash['errors']:
  136:         ret+="""<error atf="%s">%s</error>"""%error
  137:     
  138:     ret+="</errors>"
  139:     
  140:     ret+="<changes>"
  141:     for changed in hash['changed']:
  142:         ret+="""<change atf="%s">%s</change>"""%changed
  143:     ret+="</changes>"
  144:     
  145:     ret+="<newPs>"
  146:     for new in hash['newPs']:
  147:         ret+="""<new atf="%s"/>"""%new
  148:     ret+="</newPs>"
  149:     
  150:     ret+="</return>"
  151:     return ret
  152:     
  153:     
  154: def unique(s):
  155:     """Return a list of the elements in s, but without duplicates.
  156: 
  157:     For example, unique([1,2,3,1,2,3]) is some permutation of [1,2,3],
  158:     unique("abcabc") some permutation of ["a", "b", "c"], and
  159:     unique(([1, 2], [2, 3], [1, 2])) some permutation of
  160:     [[2, 3], [1, 2]].
  161: 
  162:     For best speed, all sequence elements should be hashable.  Then
  163:     unique() will usually work in linear time.
  164: 
  165:     If not possible, the sequence elements should enjoy a total
  166:     ordering, and if list(s).sort() doesn't raise TypeError it's
  167:     assumed that they do enjoy a total ordering.  Then unique() will
  168:     usually work in O(N*log2(N)) time.
  169: 
  170:     If that's not possible either, the sequence elements must support
  171:     equality-testing.  Then unique() will usually work in quadratic
  172:     time.
  173:     (from the python cookbook)
  174:     """
  175: 
  176:     n = len(s)
  177:     if n == 0:
  178:         return []
  179: 
  180:     # Try using a dict first, as that's the fastest and will usually
  181:     # work.  If it doesn't work, it will usually fail quickly, so it
  182:     # usually doesn't cost much to *try* it.  It requires that all the
  183:     # sequence elements be hashable, and support equality comparison.
  184:     u = {}
  185:     try:
  186:         for x in s:
  187:             u[x] = 1
  188:     except TypeError:
  189:         del u  # move on to the next method
  190:     else:
  191:         return u.keys()
  192: 
  193:     # We can't hash all the elements.  Second fastest is to sort,
  194:     # which brings the equal elements together; then duplicates are
  195:     # easy to weed out in a single pass.
  196:     # NOTE:  Python's list.sort() was designed to be efficient in the
  197:     # presence of many duplicate elements.  This isn't true of all
  198:     # sort functions in all languages or libraries, so this approach
  199:     # is more effective in Python than it may be elsewhere.
  200:     try:
  201:         t = list(s)
  202:         t.sort()
  203:     except TypeError:
  204:         del t  # move on to the next method
  205:     else:
  206:         assert n > 0
  207:         last = t[0]
  208:         lasti = i = 1
  209:         while i < n:
  210:             if t[i] != last:
  211:                 t[lasti] = last = t[i]
  212:                 lasti += 1
  213:             i += 1
  214:         return t[:lasti]
  215: 
  216:     # Brute force is all that's left.
  217:     u = []
  218:     for x in s:
  219:         if x not in u:
  220:             u.append(x)
  221:     return u
  222: 
  223: 
  224: class BasketContent(SimpleItem):
  225:     """classe fuer den Inhalt eines Baskets"""
  226:    
  227:     def __init__(self,content=[]):
  228:         """content"""
  229:         self.contentList=content[0:]
  230:     
  231:     def getContent(self,filtered=True):
  232:         """get content"""
  233:         ret=[]
  234: 	if filtered:
  235: 	    for x in self.contentList:
  236:                 if not((x[0] is None) or (x[1] is None)):
  237:                         ret.append(x)
  238:             return ret
  239:     	
  240: 	else:
  241: 	    return self.contentList
  242: 
  243:     def allContent(self):
  244:         """get all content"""
  245:         return self.getContent(filtered=False)
  246: 
  247:     def setContent(self,content):
  248:         self.contentList=content[0:]
  249:     
  250:     def numberOfItems(self):
  251:         """number"""
  252:         
  253:         return len(self.getContent())
  254:         
  255:     
  256: class uploadATFfinallyThread(Thread):
  257:     """class for adding uploaded filed (temporarily stored in the staging area at /tmp"""
  258:     
  259:     def __init__(self):
  260:         """init for uploadATFfinallyThread"""
  261:         self.continueVar=True
  262:         self.returnValue=None
  263:         self.end=False
  264:         Thread.__init__(self)
  265:            
  266:     def set(self,procedure,comment="",basketname='',unlock=None,SESSION=None,username=None,serverport="8080"):
  267:         """set start values for the thread"""
  268:         self.procedure=procedure
  269:         self.comment=comment
  270:         self.basketname=basketname
  271:         self.unlock=unlock
  272:         self.SESSION=SESSION
  273:         self.username=username
  274:         self.serverport=serverport
  275:        
  276:         
  277:     def __call__(self):
  278:         """call of the thread (equals run)"""
  279:         self.run()
  280:         return True
  281:     
  282:     def getContext(self, app,serverport="8080"):
  283:         """get the context within the ZODB"""
  284:         
  285:         resp = HTTPResponse(stdout=None)
  286:         env = {
  287:             'SERVER_NAME':'localhost',
  288:             'SERVER_PORT':serverport,
  289:             'REQUEST_METHOD':'GET'
  290:             }
  291:         req = HTTPRequest(None, env, resp)
  292:         return app.__of__(RequestContainer(REQUEST = req))
  293:           
  294:         
  295:     def run(self):
  296:         """run"""
  297:         
  298:         self.result=""
  299:         #find context within ZODB
  300:         from Zope import DB
  301:         conn = DB.open()
  302:         root = conn.root()
  303:         app  = root['Application']
  304:         ctx = self.getContext(app,serverport=self.serverport)
  305: 
  306:         #add the files
  307:         self.uploadATFfinallyThread(ctx,self.procedure,comment=self.comment,basketname=self.basketname,unlock=self.unlock,SESSION=self.SESSION,username=self.username)
  308:         #commit the transactions
  309:         transaction.get().commit()
  310:         conn.close()
  311:         #set flag for end of this method
  312:         self.end=True
  313:         logging.info("ended")
  314:         return True
  315:     
  316:     def __del__(self):
  317:         """delete"""
  318:         
  319:         
  320:     
  321:     def getResult(self):
  322:         """method for accessing result"""
  323:         
  324:         return self.result
  325:      
  326:     def uploadATFfinallyThread(self,ctx,procedure,comment="",basketname='',unlock=None,RESPONSE=None,SESSION=None,username=None):
  327:         """upload the files"""
  328:         #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot
  329:         ctx2=ctx.cdliRoot
  330:    
  331:         self.result+="<h2>Start processing</h2>"
  332:         
  333:         #shall I only upload the changed files?
  334:         logging.debug("uploadATFfinally procedure: %s"%procedure)
  335:         if procedure=="uploadchanged":
  336:             changed=[x[0] for x in SESSION.get('changed',[])]
  337:             uploadFns=changed+SESSION.get('newPs',[])
  338:         
  339:         #or all
  340:         elif procedure=="uploadAll":
  341:             uploadFns=[]
  342:             for x in os.listdir(SESSION['tmpdir']):
  343:                 if not x in SESSION['lockerrors']:
  344:                     uploadFns.append(x)
  345:                     
  346:         #or maybe nothing
  347:         elif procedure=="noupload":
  348:             return True
  349:         else:
  350:             uploadFns=[]
  351:             
  352:         #do first the changed files    
  353:         i=0
  354:         for fn in uploadFns:
  355:             logging.debug("uploadATFfinally uploadFn=%s"%fn)
  356:             i+=1
  357:             founds=ctx2.CDLICatalog.search({'title':fn})
  358:             if len(founds)>0:
  359:                 SESSION['author']=str(username)
  360:                 self.result="<p>Changing : %s"%fn+self.result
  361:                 logging.debug("uploadatffinallythread changing:%s"%fn+self.result)
  362:                 founds[0].getObject().manage_addCDLIFileObject('',comment,SESSION['author'],file=os.path.join(SESSION['tmpdir'],fn),from_tmp=True)
  363:             if i%200==0:
  364:                 transaction.get().commit()
  365:                 logging.debug("uploadatffinallythread changing: do commit")
  366:         
  367:         transaction.get().commit()
  368:         logging.debug("uploadatffinallythread changing: last commit")
  369: 
  370:         #now add the new files        
  371:         newPs=SESSION['newPs']
  372:         if len(newPs)>0:
  373:             tmpDir=SESSION['tmpdir']
  374:             logging.debug("uploadatffinallythread adding start")
  375:             self.result="<p>Adding files</p>"+self.result
  376:             #TODO: make this configurable, at the moment base folder for the files has to be cdli_main
  377:             ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self)
  378:             logging.debug("uploadatffinallythread adding finished")
  379:         
  380:         #unlock locked files?
  381:         if unlock:
  382:             logging.debug("uploadatffinallythread unlocking start")
  383:             self.result="<p>Unlock files</p>"+self.result
  384:             unlockFns=[]
  385:             for x in os.listdir(SESSION['tmpdir']):
  386:                     if not x in SESSION['errors']:
  387:                         unlockFns.append(x)
  388:                         
  389:             logging.debug("unlocking have now what to unlock")
  390:                         
  391:             for fn in unlockFns:
  392:                 #logging.info("will unlock: %s"%fn)
  393:                 founds=ctx2.CDLICatalog.search({'title':fn})
  394:                 #logging.info("found it: %s"%repr(founds))
  395:                 if len(founds)>0:
  396:                     #logging.info("unlock: %s"%founds[0].getObject().getId())
  397:                     SESSION['author']=str(username)
  398:                     founds[0].getObject().lockedBy=""
  399: 
  400:             logging.debug("uploadatffinallythread unlocking done")
  401:                     
  402:         #if a basketname is given, add files to the basket
  403:         if not (basketname ==''):
  404:             logging.debug("uploadatffinallythread add to basket %s"%basketname)
  405:             self.result="<p>Add to basket</p>"+self.result
  406:             basketId=ctx2.basketContainer.getBasketIdfromName(basketname)
  407:             
  408:             if not basketId: # create new basket
  409:                 logging.debug("uploadatffinallythread create basket %s"%basketname)
  410:                 self.result="<p>Create a new basket</p>"+self.result
  411:                 ob=ctx2.basketContainer.addBasket(basketname)
  412:                 basketId=ob.getId()
  413:             basket=getattr(ctx2.basketContainer,str(basketId))
  414:             ids=os.listdir(SESSION['tmpdir'])
  415:             #logging.debug("should add:"+repr(ids))
  416:             basket.addObjects(ids,deleteOld=True,username=str(username))    
  417:                
  418:         logging.debug("uploadatffinallythread uploadfinally done")
  419: 
  420:         if RESPONSE is not None:
  421:             RESPONSE.redirect(self.aq_parent.absolute_url())
  422:         
  423:         return True
  424: 
  425: class tmpStore(SimpleItem):
  426:     """simple item"""
  427:     meta_type="cdli_upload"
  428:     
  429:     def __init__(self,id):
  430:         """init tmp"""
  431:         self.id=id
  432:         
  433: class uploadATFThread(Thread):
  434:     """class for checking the files befor uploading"""
  435:     
  436:     def __init__(self):
  437:         """initialise"""
  438:         
  439:         self.continueVar=True
  440:         self.returnValue=None
  441:         
  442:         Thread.__init__(self)
  443:         
  444:         
  445:     def set(self,upload,basketId,username,idTmp,serverport="8080"):
  446:         """set start values for the thread"""
  447:         self.result=""
  448:         self.upload=upload
  449:         self.basketId=basketId
  450:         self.username=username
  451:         self.serverport=serverport
  452:         self.idTmp=idTmp
  453:         
  454:     def __call__(self):
  455:         """call method """
  456:         self.run()
  457:         return True
  458:     
  459:     def getContext(self, app,serverport="8080"):
  460:         """get the context within the ZODB"""
  461:         resp = HTTPResponse(stdout=None)
  462:         env = {
  463:             'SERVER_NAME':'localhost',
  464:             'SERVER_PORT':serverport,
  465:             'REQUEST_METHOD':'GET'
  466:             }
  467:         req = HTTPRequest(None, env, resp)
  468:         return app.__of__(RequestContainer(REQUEST = req))
  469:         
  470:     def run(self):
  471:         idTmp=self.idTmp
  472:         self.result=""
  473:         #find context within ZODB
  474:         from Zope import DB
  475:         conn = DB.open()
  476:         root = conn.root()
  477:         app  = root['Application']
  478:         ctx = self.getContext(app,serverport=self.serverport)
  479:         logging.info("run intern")
  480:         try:
  481:             logging.info("created: %s"%idTmp)
  482:             ctx.temp_folder._setObject(idTmp,tmpStore(idTmp))
  483:         except:
  484:             logging.error("thread upload: %s %s"%sys.exc_info()[0:2])
  485:             
  486:         logging.info("call thread intern")
  487:         self.uploadATFThread(ctx,self.upload,idTmp,self.basketId)
  488:      
  489:         #ctx.cdliRoot.cdli_main.tmpStore2[self.getName()[0:]]=self.returnValue
  490:         
  491:         
  492:         transaction.get().commit()
  493:        
  494:         conn.close()
  495:         
  496:         return getattr(ctx.temp_folder,idTmp)
  497:         
  498:     def getResult(self):
  499:         """method for accessing result"""
  500:         return self.result
  501:     
  502:     def uploadATFThread(self,ctx,upload,idTmp,basketId=0):
  503:         """upload an atf file"""
  504:         #TODO: add comments
  505:         #TODO: finish uploadATF
  506:         
  507:         stObj=getattr(ctx.temp_folder,idTmp)
  508:         logging.info("start, upload thread")
  509:         self.result="<html><body><h2>I got your file, start now to split it into single atf-files!</h2><p>"
  510:     
  511:         #make sure that id is a string and not an integer
  512:         basketId=str(basketId)
  513:         
  514:         #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot
  515:         ctx2=ctx.cdliRoot
  516:         
  517:         #get temporary file for staging the downloaded and splitted files
  518:         dir=mkdtemp()
  519:         
  520:         
  521:         changed=[] # changed files
  522:         errors=[]  # files with errors
  523:         lockerrors=[]  # files with errors
  524: 
  525:         newPs=[]   # new p filed
  526:         psNotInCatalog=[] # files not in the catalog
  527:         
  528:         #split the uploadedd atf file
  529:         basketNameFromFile, numberOfFiles=splitatf(upload,dir,ext=self)
  530:         
  531:         #find basketId if not set
  532:         
  533:         #get active abaket
  534:         if basketId == '0':
  535:             basketObj=ctx2.basketContainer.getActiveBasket()
  536:             if basketObj:
  537:                 basketId=basketObj.getId()
  538:                 
  539:         #if there is no active basket and no basketid given, id is empty, else get besketname and length
  540:         if basketId == '0':
  541:             basketNameFromId=""
  542:             basketLen=0
  543:         else:
  544:             basketNameFromId=getattr(ctx2.basketContainer,basketId).title
  545:             basketLen=getattr(ctx2.basketContainer,basketId).getLastVersion().numberOfItems()
  546:             
  547:         logging.info("got the file, upload thread")
  548:         self.result+="""<html><body><h2>I got the files</h2><
  549:                         p>I am computing the differences to the exisiting files</p>"""
  550:                                    
  551:         #start to check the files
  552:         for fn in os.listdir(dir):
  553:             
  554:             self.result="<p>process:%s</p>"%fn+self.result
  555:             
  556:             # check if file is in the catalog
  557:             #TODO: checkCatalog is not implemented yet
  558:             if ctx2.cdli_main.checkCatalog(fn):
  559:                 psNotInCatalog.append(fn)
  560:                 
  561:             #check if p-file already at the server  
  562:             founds=ctx2.CDLICatalog.search({'title':fn})    
  563:       
  564:             #if not than add filename to the list of newfiles
  565:             
  566:             data=file(os.path.join(dir,fn)).read()
  567:             status,msg=checkFile(fn,data,dir)
  568:             #status=True
  569:             
  570:             
  571:             if not status: # error
  572:                 errors.append((fn,msg))
  573:             
  574:             else:
  575:                 if len(founds)==0:
  576:                     newPs.append(fn)
  577: 
  578:                 #if p file alread at the server    
  579:                 for found in founds:
  580:                     #analyse the differences to the actual file
  581:                     obj=found.getObject()
  582: 
  583:                     if (not (str(obj.lockedBy))=='') and (not (str(obj.lockedBy)==str(self.username))):
  584:                                 lockerrors.append((fn,str(obj.lockedBy)))
  585:                     else:
  586:                 
  587:                         diffs=obj.diff(data)
  588:                         if diffs[0]>0:
  589:                             changed.append((obj,diffs)) #hochladen
  590: 
  591:         #ready, set the returnValues
  592:         self.result+="<h3>Done</h3></body></html>"
  593:         
  594:         stObj.returnValue={}
  595:         
  596:         stObj.returnValue['errors']=errors
  597:         
  598:         stObj.returnValue['newPs']=newPs
  599:         stObj.returnValue['tmpdir']=dir
  600:         stObj.returnValue['basketLen']=basketLen
  601:         stObj.returnValue['numberOfFiles']=numberOfFiles
  602:         stObj.returnValue['basketNameFromId']=basketNameFromId
  603:         stObj.returnValue['basketNameFromFile']=basketNameFromFile
  604:         stObj.returnValue['basketId']=basketId
  605:         stObj.returnValue['dir']=dir
  606:         #stObj.returnValue['changed']=copy.copy(changed)
  607:         stObj.returnValue['changed']=[(x[0].getId(),x[1][0]) for x in changed]
  608:         #stObj.returnValue['lockerrors']=[x[0].getId() for x in lockerrors]
  609:         stObj.returnValue['lockerrors']=[x for x in lockerrors]
  610:         self.returnValue=True
  611:         #ctx2.cdli_main.setTemp('v_uploadATF_returnValue',True)
  612:     
  613:  
  614: class CDLIBasketContainer(OrderedFolder):
  615:     """contains the baskets"""
  616:     
  617: 
  618:     security=ClassSecurityInfo()
  619:     meta_type="CDLIBasketContainer"
  620:     
  621:     def getPNumbersOfBasket(self,basketName):
  622:         """get all pnumbers of a basket as a list, returns an empty list if basket not found
  623:         @param basketName: name of the basket
  624:         """
  625:         ret=[]
  626:         basketId=self.getBasketIdfromName(basketName)
  627:         if not basketId:
  628:             return []
  629:         
  630:         ob=getattr(self,basketId).getContent()
  631:         
  632:         ret=[x[0].split(".")[0] for x in ob]
  633:         
  634:         return ret
  635:     
  636:     security.declareProtected('manage','getBasketAsOneFile')       
  637:     def getBasketAsOneFile(self,basketName,current="no"):
  638:         """returns all files of the basket combined in one file
  639:         @param basketName: Name of the basket
  640:         @param current: (optional) if current is set to "yes" then the most current version of 
  641:                         all files are downloaded and not the versions of the files as stored in the basket
  642:         """
  643:         ret=""
  644:         basketId=self.getBasketIdfromName(basketName)
  645:         if not basketId:
  646:             return ""
  647:         
  648:         ob=getattr(self,basketId).getLastVersion()
  649:         for object in ob.getContent():
  650:             if current=="no": #version as they are in the basket
  651:                             ret+=str(object[0].getData())+"\n"
  652:             elif current=="yes":
  653:                             #search current object
  654:                             #logging.debug("current: %s"%object[1].getId().split(".")[0])
  655:                             founds=self.CDLICatalog.search({'title':object[1].getId().split(".")[0]})
  656:                             if len(founds)>0:      
  657:                                 ret+=str(founds[0].getObject().getLastVersion().getData())+"\n"
  658:         return ret
  659:     
  660:     security.declareProtected('manage','upDateBaskets') 
  661:     def upDateBaskets(self):
  662:         """update content in to objects"""
  663:         
  664:         founds=self.ZopeFind(self,obj_metatypes=['CDLIBasketVersion'],search_sub=1)
  665: 
  666:         for found in founds:
  667:             found[1].updateBasket()
  668:         
  669:     security.declareProtected('manage','deleteBaskets')        
  670:     def deleteBaskets(self,ids=None):
  671:         """delete baskets, i.e. move them into trash folder"""
  672:         
  673:         if ids is None:
  674:             pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','cdliError_html.zpt')).__of__(self)
  675:             txt="Sorry, no basket selected!"
  676:             return pt(txt=txt)
  677:         
  678:         found=self.ZopeFind(self,obj_ids=['trash'])
  679:         
  680:         if len(found)<1:
  681:             manage_addFolder(self, 'trash')
  682:             trash=self._getOb('trash')
  683:         else:
  684:             trash=found[0][1]
  685:         
  686:         if type(ids) is not ListType:
  687:             ids=[ids]
  688:         logging.error("XERXON:"+repr(ids))
  689:         if len(ids)==0:
  690:             pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','cdliError_html.zpt')).__of__(self)
  691:             txt="Sorry, no basket selected!"
  692:             return pt(txt=txt)
  693:    
  694:         cut=self.manage_cutObjects(ids)
  695:         trash.manage_pasteObjects(cut)
  696:         return None
  697:     security.declareProtected('manage','manageBaskets')       
  698:     def manageBaskets(self,submit,ids=None,basket1="",basket2="",joinBasket="",subtractBasket="",REQUEST=None,RESPONSE=None):
  699:         """manage baskets, delete or copy"""
  700:         if submit=="delete":
  701:             ret= self.deleteBaskets(ids)
  702:             if ret:
  703:                 return ret
  704:         elif submit=="join":
  705:             flag,msg=self.joinBasket(joinBasket, ids)
  706:             logging.info("joining %s %s"%(flag,msg))
  707:             if not flag:
  708:                 pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','cdliError_html.zpt')).__of__(self)
  709:                 
  710:                 return pt(txt=msg)
  711:        
  712:         elif submit=="subtract":
  713:             logging.info("BBBb %s %s"%(basket1,basket2))
  714:             flag,msg=self.subtractBasket(subtractBasket, basket1,basket2)
  715:             logging.info("subtract %s %s"%(flag,msg))
  716:             
  717:         if RESPONSE:
  718:             RESPONSE.redirect(self.absolute_url())
  719:     
  720:     security.declareProtected('View','getBasketIdfromName')       
  721:     def getBasketIdfromName(self,basketname):
  722:         """get id from name"""
  723: 
  724:         for basket in self.ZopeFind(self,obj_metatypes=["CDLIBasket"]):
  725:             if basket[1].title==basketname:
  726:                 return basket[0]
  727:         else:
  728:             None
  729:     
  730:     security.declareProtected('manage','uploadBasket_html')        
  731:             
  732:     def uploadBasket_html(self,basketId='0'):
  733:         """upload an atf file, html form"""
  734:         
  735: 
  736:         basketId=str(basketId)
  737:         if not basketId=='0':
  738:             basketName=getattr(self.basketContainer,basketId).title
  739:         else:
  740:             basketName=""
  741:             
  742:         pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadBasket_html.zpt')).__of__(self)
  743:         return pt(basketId=basketId,basketName=basketName)
  744:    
  745: 
  746:     security.declareProtected('manage','index_html')    
  747:     def index_html(self):
  748:         """stanadard ansicht"""
  749:         
  750: 
  751: 
  752:         ext=self.ZopeFind(self,obj_ids=["index.html"])
  753:         if ext:
  754:             return ext[0][1]()
  755:         
  756:         pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketContainerMain')).__of__(self)
  757:         return pt()
  758:     
  759:     def getStorageFolderRoot(self):
  760:         """root des storage folders"""
  761:         return self.cdli_main
  762:     
  763:     def __init__(self,id,title):
  764:         """ init basket container"""
  765:         self.id=id
  766:         self.title=title
  767:      
  768:  
  769:     def getBasketsId(self):
  770:         """get all baskets als klartext"""
  771:         
  772:         ret=""
  773:         baskets=self.ZopeFind(self,obj_metatypes=['CDLIBasket'])
  774:         for basket in baskets:
  775:             com,user,time,values = basket[1].getContentIds()
  776:             ret+= "BASKET:"+com+"\t"+user+"\t"+time+"\n"
  777:             for x in values:
  778:                 ret+= x[0]+"\t"+x[1]+"\n"
  779:                 return ret
  780: 
  781:     def getBaskets(self,sortField='title'):
  782:         """get all baskets files"""
  783: 
  784:         def sortName(x,y):
  785:             return cmp(x[1].title.lower(),y[1].title.lower())
  786: 
  787:         def sortDate(x,y):
  788:             return cmp(y[1].getLastVersion().getTime(),x[1].getLastVersion().getTime())
  789: 
  790:         
  791:         def sortComment(x,y):
  792: 
  793:         
  794:             
  795:              try:
  796:                 xc=getattr(x[1],'comment','ZZZZZZZZZZZZZ').lower()
  797:              except:
  798:                 xc='ZZZZZZZZZZZZZ'.lower()
  799:              try:
  800:                 yc=getattr(y[1],'comment','ZZZZZZZZZZZZZ').lower()
  801:              except:
  802:                 yc='ZZZZZZZZZZZZZ'.lower()
  803:     
  804:     
  805:              if (xc=='') or (xc=='ZZZZZZZZZZZZZ'.lower()):
  806:                  
  807:                  try:
  808:                      xc=x[1].getLastVersion().getComment().lower()
  809:                  except:
  810:                      xc='ZZZZZZZZZZZZZ'.lower()
  811:                      
  812:              if (yc=='') or (yc=='ZZZZZZZZZZZZZ'.lower()):
  813:                  try:
  814:                      yc=y[1].getLastVersion().getComment().lower()
  815:                  except:
  816:                      yc='ZZZZZZZZZZZZZ'.lower()
  817:     
  818:              
  819:                  return cmp(xc,yc)
  820:         
  821:         def sortAuthor(x,y):
  822:             
  823:             return cmp(x[1].getLastVersion().getUser().lower(),y[1].getLastVersion().getUser().lower())
  824:         
  825:         baskets=self.ZopeFind(self,obj_metatypes=['CDLIBasket'])
  826:         
  827:         
  828:         if sortField=='title':
  829:             baskets.sort(sortName)
  830:         elif sortField=='date':
  831:             baskets.sort(sortDate)
  832:         elif sortField=='author':
  833:             baskets.sort(sortAuthor)
  834:         elif sortField=='comment':
  835:             baskets.sort(sortComment)
  836: 
  837:         return baskets
  838:     
  839:         
  840:     def subtractBasket(self,newBasket,basket1,basket2):
  841:         """subtract basket2 from basket1 
  842:         (i.e. newbasket will contain alle elements of basket1 which are not in basket2), 
  843:         if basket2 contains files which are not in basket1, then theses files fill be ignored
  844:                
  845:         @param newbasket: name of the new basket
  846:         @param basket1: basket where basket2 will be subtracted from
  847:         @param basket2: see above
  848:       
  849:         """
  850:         
  851:         logging.info("CCCCC %s %s"%(basket1,basket2))
  852:    
  853:         try:
  854:             newB=self.addBasket(newBasket)
  855:         except:
  856:             return False, "cannot create the new basket"
  857:         
  858:         
  859: 
  860:        
  861:      
  862:         bas2= getattr(self,basket2)            
  863:         bas2content=bas2.getContent()
  864:         bas2ids=[x[0] for x in bas2content]
  865:         
  866:        
  867:             
  868:         bas1= getattr(self,basket1)   
  869:         bas1content=bas1.getContent()
  870:         
  871:         
  872:         newBasketContent={}
  873:         
  874:         for id,version in bas1content:
  875:             if not (id in bas2ids):
  876:                 newBasketContent[id]=version
  877:         
  878:         username=self.getActualUserName()
  879:         
  880:         logging.info("sbc %s"%newBasketContent)
  881:         newB.addObjectsWithVersion(newBasketContent,username=username,catalog=self.CDLICatalog)
  882:         
  883:         return True, ""
  884:     
  885:             
  886:     def joinBasket(self,newBasket,oldBaskets):
  887:         """join two baskets
  888:         @param newbasket: name of the new basket
  889:         @param oldbaskets: list of baskets to be joined
  890:         """
  891:         if oldBaskets is None:
  892:             return False, "No Baskets selected!"
  893:         
  894:         try:
  895:             newB=self.addBasket(newBasket)
  896:         except:
  897:             return False, "cannot create the new basket"
  898:         
  899:         newBasketContent={}
  900:      
  901:         for ob in oldBaskets:
  902:             x= getattr(self,ob,None)
  903:             if x is None:
  904:                 return False, "cannot find basket: %s"%ob
  905:             
  906:             ids=x.getContent() # hole den Inhalt
  907:             
  908:             for id,version in ids:
  909:                 if newBasketContent.has_key(id): # p number gibt's schon
  910:                     newBasketContent[id]=max(newBasketContent[id],version) # speichere die groessere Versionsnumber
  911:                 else:
  912:                     newBasketContent[id]=version
  913:         username=self.getActualUserName()
  914:         
  915:         logging.info("nbc %s"%newBasketContent)
  916:         newB.addObjectsWithVersion(newBasketContent,username=username,catalog=self.CDLICatalog)
  917:         
  918:         return True, ""
  919:     
  920:     def getNewId(self):
  921:         """createIds"""
  922:         last=getattr(self,'last',0)
  923:         last +=1
  924:         while len(self.ZopeFind(self,obj_ids=[str(last)]))>0:
  925:             last+=1
  926:     
  927:         self.last=last
  928:         return last
  929:     
  930:     def setActiveBasket(self,basketId,REQUEST=None):
  931:         """store active basketId in a cookie"""
  932:         self.REQUEST.RESPONSE.setCookie("CDLIActiveBasket",basketId,path="/")
  933:         try:
  934:             qs=cgi.parse_qs(REQUEST['QUERY_STRING'])
  935:             del(qs['basketId'])
  936:         except:
  937:             qs={}
  938:         if REQUEST:
  939:             REQUEST.RESPONSE.redirect(REQUEST['URL1']+'?'+urllib.urlencode(qs))
  940:             
  941:     def getActiveBasket(self):
  942:         """get active basket from cookie"""
  943:         
  944:         id= self.REQUEST.cookies.get('CDLIActiveBasket',None)
  945:         if id:
  946:             obj=getattr(self,str(id),None)
  947:         else:
  948:             obj=None
  949:         return obj
  950:     
  951:     def getActualUserName(self):
  952:         """get name of the actualuser"""
  953:         return str(self.REQUEST['AUTHENTICATED_USER'])
  954:     
  955:     security.declareProtected('manage','addBasket') 
  956:     def addBasket(self,newBasketName):
  957:         """add a new basket"""
  958:         
  959:         ob=manage_addCDLIBasket(self,newBasketName)
  960:         return ob
  961: 
  962:     def storeInBasket(self,submit,ids=None,newBasketName=None,fromFileList=None,RESPONSE=None,REQUEST=None):
  963:         """store it"""
  964:         if not ids:
  965:             ids=self.REQUEST.SESSION['fileIds']
  966:            
  967:         if (type(ids) is not ListType) and (not isinstance(ids,Set)):
  968:             ids=[ids]
  969:         
  970:         if isinstance(ids,Set):
  971:             ids=list(ids)
  972:             
  973:         if (submit.lower()=="store in new basket") or (submit.lower()=="new basket"):
  974:             basketRet=self.addBasket(newBasketName)
  975:             self.setActiveBasket(basketRet.getId())
  976:             basket=getattr(self,basketRet.getId())
  977:         elif (submit.lower()=="store in active basket") or (submit.lower()=="active basket"):
  978:             basket=self.getActiveBasket()
  979:         
  980:         added=basket.addObjects(ids)
  981:         back=self.REQUEST['HTTP_REFERER'].split("?")[0]+"?basketName="+basket.title+"&numberOfObjects="+str(added)
  982:         
  983:         
  984:         if fromFileList:
  985: 
  986:             return self.cdli_main.findObjectsFromList(list=ids,basketName=basket.title,numberOfObjects=added)
  987:        
  988:         if RESPONSE:
  989:             
  990:             RESPONSE.redirect(back)
  991:             
  992:         return True
  993:     
  994: def manage_addCDLIBasketContainerForm(self):
  995:     """add the CDLIBasketContainer form"""
  996:     pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIBasketContainer.zpt')).__of__(self)
  997:     return pt()
  998: 
  999: def manage_addCDLIBasketContainer(self,id,title,RESPONSE=None):
 1000:     """add the basket"""
 1001:     ob=CDLIBasketContainer(id,title)
 1002:     
 1003:     self._setObject(id, ob)
 1004:     
 1005:     if RESPONSE is not None:
 1006:         RESPONSE.redirect('manage_main')
 1007: 
 1008: class CDLIBasket(Folder,CatalogAware):
 1009:     """basket"""
 1010:     
 1011:     meta_type="CDLIBasket"
 1012:     default_catalog="CDLIBasketCatalog"
 1013:     
 1014:     def searchInBasket(self,indexName,searchStr,regExp=False):
 1015:         """searchInBasket"""
 1016: 
 1017:         lst=self.searchInLineIndexDocs(indexName,searchStr,uniq=True,regExp=regExp) #TODO: fix this
 1018:         ret={}
 1019:         
 1020:         lv=self.getLastVersion()
 1021: 
 1022: 
 1023:         for obj in lv.content.getContent():
 1024:             id=obj[1].getId().split(".")[0]
 1025:             if id in lst:
 1026:         
 1027:                 ret[id]=self.showWordInFile(id,searchStr,lineList=self.getLinesFromIndex(indexName,searchStr,id,regExp=regExp),regExp=regExp,indexName=indexName)
 1028:         
 1029:         
 1030:         pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','searchResultsInBasket')).__of__(self)
 1031:         return pt(result=ret,indexName=indexName,regExp=regExp,word=searchStr)
 1032:         
 1033:          
 1034:  
 1035:         
 1036:     def searchInBasket_v1(self,searchStr):
 1037:         """search occurences of searchStr in files im basket"""
 1038:         ret=[]
 1039:         lv=self.getLastVersion()
 1040:         logging.info("searching")
 1041:         for obj in lv.content.getContent():
 1042:             txt=obj[0].getData()
 1043:             for x in txt.split("\n"):
 1044:                 logging.info("search %s"%x)
 1045:                 if re.match(searchStr,x):
 1046:                     ret.append(x)
 1047:         
 1048:         return "\n".join(ret)
 1049:                 
 1050:             
 1051:     def getFile(self,obj):
 1052:         return obj[1]
 1053:     
 1054:     def getFileLastVersion(self,obj):
 1055:         return obj[0]
 1056:     
 1057:     def getFileNamesInLastVersion(self):
 1058:         """get content of the last version as list"""
 1059:         
 1060:         return [x[1].getId() for x in self.getLastVersion().getContent()]
 1061:     
 1062: 
 1063:     def isActual(self,obj):
 1064:         """teste ob im basket die aktuelle version ist"""
 1065:         try:
 1066:             #logging.debug("isActual:"+repr(obj))
 1067:             actualNo=obj[1].getLastVersion().getVersionNumber()
 1068:             storedNo=obj[0].getVersionNumber()
 1069:             
 1070:            
 1071:             #actualNo=self.getFileObjectLastVersion(obj.getId()).getVersionNumber()
 1072:                 
 1073:             #if len(founds)>0 and founds[0].getObject().aq_parent.getId()==".trash":
 1074:             #    return False, -1
 1075:             
 1076:             if actualNo==storedNo:
 1077:                 return True , 0
 1078:             else:
 1079:                 return False, actualNo
 1080:         except:
 1081:             logging.error( """is actual: %s (%s %s)"""%(repr(obj),sys.exc_info()[0],sys.exc_info()[1]))
 1082:     
 1083:             return False, -1
 1084:             
 1085:     def history(self):
 1086:         """history"""  
 1087: 
 1088:         ext=self.ZopeFind(self.aq_parent,obj_ids=["history_template.html"])
 1089:         if ext:
 1090:             return getattr(self,ext[0][1].getId())()
 1091:         
 1092:         pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketHistory')).__of__(self)
 1093:         return pt()
 1094:     
 1095:     def getStorageFolderRoot(self):
 1096:         """root des storage folders"""
 1097:         return self.aq_parent.cdli_main
 1098:     
 1099:     def __init__(self,id,title,shortDescription="",comment=""):
 1100:         """init a CDLIBasket"""
 1101:         
 1102:         self.id=id
 1103:         self.title=title
 1104:         self.shortDescription=shortDescription
 1105:         self.comment=comment
 1106:  
 1107:     def getActualUserName(self):
 1108:         """get name of the actualuser"""
 1109:        
 1110:         return str(self.REQUEST['AUTHENTICATED_USER'])
 1111:   
 1112:            
 1113:     def getLastVersion(self):
 1114:         """hole letzte version"""
 1115: 
 1116:         ids=[]
 1117:         idsTmp= self.objectIds()
 1118:         for x in idsTmp:
 1119:             try:
 1120:                 ids.append(int(x))
 1121:             except:
 1122:                 pass
 1123:         ids.sort()
 1124:       
 1125:         if len(ids)==0:
 1126:             return None
 1127:         else:    
 1128:             ob=getattr(self,str(ids[-1]))
 1129: 
 1130:             
 1131:             return ob
 1132:    
 1133:     def getVersions(self):
 1134:         """get versions"""
 1135:         versions=self.ZopeFind(self,obj_metatypes=["CDLIBasketVersion"])
 1136:         return versions
 1137: 
 1138:    
 1139:     def updateObjects(self,ids,RESPONSE=None,REQUEST=None):
 1140:         """update ids, ids not in the basket the add"""
 1141:         if type(ids) is not ListType:
 1142:             ids=[ids]
 1143:        
 1144:         lastVersion=self.getLastVersion() 
 1145:         oldContent=lastVersion.content.getContent()
 1146:         newContent=[]
 1147:         
 1148:         #first copy the old
 1149:         for obj in oldContent:
 1150:             if obj[1].getId() not in ids:
 1151:                 newContent.append(obj)
 1152:         #now add the new
 1153:                
 1154:         for id in ids:
 1155:             founds=self.CDLICatalog.search({'title':id})
 1156: 
 1157:             for found in founds:
 1158:                 if found.getObject() not in oldContent:
 1159:                     #TODO: was passiert wenn, man eine Object dazufŸgt, das schon da ist aber eine neuere version
 1160:                     newContent.append((found.getObject().getLastVersion(),found.getObject()))
 1161:         
 1162: 
 1163:         content=newContent 
 1164:         user=self.getActualUserName()
 1165:         
 1166:         ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=newContent)
 1167:         
 1168:         obj=self._getOb(ob.getId())
 1169:         if RESPONSE:
 1170:            
 1171:             RESPONSE.redirect(obj.absolute_url())
 1172:         
 1173:         return obj
 1174:     
 1175:     def addObjectsWithVersion(self,ids,deleteOld=None,username=None,catalog=None):
 1176:         """generate a new version of the basket with objects added, 
 1177:         hier wird jedoch nicht die letzte Version jedes Files hinzugefuegt, s
 1178:         ondern ids is ein Tupel mit der Id (d.h. der p-number) und der Versionsnummer.
 1179:         """
 1180:         logging.info("add to basket (%s)"%(self.getId()))
 1181:         lastVersion=self.getLastVersion()
 1182:         
 1183:         if not catalog:
 1184:             catalog=self.CDLICatalog
 1185:             
 1186:         if lastVersion is None:
 1187:             oldContent=[]
 1188:         else:
 1189:             oldContent=lastVersion.content.getContent()
 1190: 
 1191:         if deleteOld:
 1192:             oldContent=[]
 1193: 
 1194:         newContent=[]
 1195:         added=0
 1196:        
 1197:         for id,version in ids.iteritems():
 1198:             logging.info("adding %s %s"%(id,version))
 1199:             id=id.split(".")[0] # title nur die pnumber ohne atf
 1200:            
 1201:             try:
 1202:                 founds=catalog.search({'title':id})
 1203:             except:
 1204:                 founds=[]
 1205:             logging.info(" found %s "%(founds))
 1206:             for found in founds:
 1207:                 if found.getObject() not in oldContent:
 1208:                  
 1209:                     #TODO: was passiert wenn, man eine Object dazufŸgt, das schon da ist aber eine neuere version
 1210:                     newContent.append((found.getObject().getVersions()[version-1][1],found.getObject()))
 1211:                     added+=1
 1212: 
 1213:         content=oldContent+newContent
 1214:         if not username:
 1215:             logging.error("XXXXXXXXXXX %s"%repr(self))
 1216:             user=self.getActualUserName()
 1217:         else:
 1218:             user = username
 1219:             
 1220:         ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=content)
 1221:         logging.info("add to basket (%s) done"%(self.getId()))
 1222:         return added
 1223:     
 1224:     
 1225:     def addObjects(self,ids,deleteOld=None,username=None):
 1226:         """generate a new version of the basket with objects added"""
 1227:         
 1228:         def swap(x):
 1229:             return (x[1],x[0])
 1230:             
 1231:         logging.info("add to basket (%s)"%(self.getId()))
 1232:         lastVersion=self.getLastVersion()
 1233:         
 1234:         if lastVersion is None:
 1235:             oldContent=[]
 1236:         else:
 1237:             oldContent=lastVersion.content.getContent()
 1238: 
 1239:         if deleteOld:
 1240:             oldContent=[]
 1241: 
 1242:         added=0
 1243: #        for id in ids:
 1244: #            logging.debug("adding:"+id)
 1245: #            try:
 1246: #                founds=self.CDLICatalog.search({'title':id})
 1247: #            except:
 1248: #                founds=[]
 1249: #           
 1250: #            for found in founds:
 1251: #                if found.getObject() not in oldContent:
 1252: #                    #TODO: was passiert wenn, man eine Object dazufŸgt, das schon da ist aber eine neuere version
 1253: #                    newContent.append((found.getObject().getLastVersion(),found.getObject()))
 1254: #                    added+=1
 1255: 
 1256:         hash = md5.new(repr(makelist(ids))).hexdigest() # erzeuge hash als identification
 1257:         #logging.debug("JJJJJJJ:"+repr(self.makelist(ids)))
 1258:        
 1259:                       
 1260:         if hasattr(self.cdliRoot,'v_tmpStore') and self.cdliRoot.v_tmpStore.has_key("hash"):  #TODO: muss eigentlich  self.cdliRoot.v_tmpStore.has_key(hash): heissen (ohne "), erstmal so gesetzt damit der hash hier nie benutzt wird
 1261:             logging.debug("from store!")
 1262:             newContent=Set(map(swap,self.cdliRoot.v_tmpStore[hash]))
 1263:          
 1264:         else:
 1265:             logging.debug("not from store!")
 1266:             newContent=Set([(self.getFileObjectLastVersion(x),self.getFileObject(x)) for x in ids])
 1267:         
 1268:         
 1269:         content=Set(oldContent).union(newContent)
 1270:         added = len(content)-len(oldContent)
 1271:         if not username:
 1272:             user=self.getActualUserName()
 1273:         else:
 1274:             user = username
 1275:         
 1276:         #logging.debug("content:"+repr(list(content)))
 1277:         ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=list(content))
 1278:         logging.info("add to basket (%s) done"%(self.getId()))
 1279:         return added
 1280:     
 1281:     
 1282:                 
 1283:     def getContent(self):
 1284:         """print content"""
 1285:         ret=[]
 1286:         
 1287:         lv=self.getLastVersion()
 1288:         for obj in lv.content.getContent():
 1289:             #logging.info("XXXXXXXXXX %s"%repr(obj))
 1290:             ret.append((obj[1].getId(),obj[0].versionNumber))
 1291:             
 1292:         return ret
 1293:         
 1294:     def getContentIds(self):
 1295:         """print basket content"""
 1296:         ret=[]
 1297:         lv=self.getLastVersion()
 1298:         for obj in lv.content.getContent():
 1299:             ret.append((obj[0].getId(),obj[1].getId()))
 1300:         
 1301:         
 1302:         return lv.getComment(),lv.getUser(),lv.getTime(),ret
 1303: 
 1304:     def changeBasket(self,ids,submit,RESPONSE=None,REQUEST=None):
 1305:         """change a basket"""
 1306:         if submit=="update":
 1307:             return self.updateObjects(ids,RESPONSE=RESPONSE,REQUEST=REQUEST)
 1308:         elif submit=="delete":
 1309:             return self.deleteObjects(ids,RESPONSE=RESPONSE,REQUEST=REQUEST)
 1310:             
 1311:     def deleteObjects(self,ids,RESPONSE=None,REQUEST=None):
 1312:         """delete objects"""
 1313:         
 1314:         if type(ids) is not ListType:
 1315:             ids=[ids]
 1316:        
 1317:         lastVersion=self.getLastVersion() 
 1318:         oldContent=lastVersion.content.getContent()
 1319:         newContent=[]
 1320:         for obj in oldContent:
 1321:             if obj[1].getId() not in ids:
 1322:                 newContent.append(obj)
 1323:         
 1324:                 
 1325:         user=self.getActualUserName()
 1326:         
 1327:         ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=newContent)
 1328:         
 1329:         if RESPONSE:
 1330:             obj=self._getOb(ob.getId())
 1331:             RESPONSE.redirect(obj.absolute_url())
 1332:         
 1333: def manage_addCDLIBasketForm(self):
 1334:     """add the CDLIBasketContainer form"""
 1335:     pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIBasket.zpt')).__of__(self)
 1336:     return pt()
 1337: 
 1338: def manage_addCDLIBasket(self,title,shortDescription="",comment="",RESPONSE=None):
 1339:     """add the basket"""
 1340:     
 1341:     id=str(self.getNewId())
 1342:     
 1343:     ob=CDLIBasket(id,title,shortDescription,comment)
 1344:     
 1345:     self._setObject(id, ob)
 1346:     
 1347:     if RESPONSE is not None:
 1348:         RESPONSE.redirect('manage_main')
 1349:     else:
 1350:         return ob
 1351: 
 1352: class CDLIBasketVersion(Implicit,Persistent,Folder):
 1353:     """version of a basket"""
 1354:     
 1355:     meta_type="CDLIBasketVersion"
 1356:     security=ClassSecurityInfo()
 1357:     
 1358:     def updateBasket(self):
 1359:         """update"""
 1360:         try:
 1361:             self._setObject('content',BasketContent(self.basketContent))
 1362:         except:
 1363:             try:
 1364:                 if len(self.basketContent)>0:
 1365:                     self.content.setContent(self.basketContent)
 1366:             except:
 1367:                 print "error",self.getId(),self.aq_parent.getId()
 1368:         self.basketContent=[]
 1369: 
 1370:         
 1371:     def containsNonActualFiles(self):
 1372:         """returns True if basket contains one or more non current files"""
 1373:         
 1374:         objs=self.getContent()
 1375:         for obj in objs:
 1376:             if not self.isActual(obj)[0]:
 1377:                 return True
 1378:         return False
 1379:     
 1380:     def downloadListOfPnumbers(self):
 1381:         """download pnumbers of the basket as list"""
 1382:         
 1383:         basket_name=self.aq_parent.title
 1384:         
 1385:         ids=self.getContent() # get the list of objects
 1386:         logging.error(ids)
 1387:         ret="\n".join([x[1].getId().split(".")[0] for x in ids])
 1388:         
 1389:         self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.txt" """%basket_name)
 1390:         self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream")
 1391:         length=len(ret)
 1392:         self.REQUEST.RESPONSE.setHeader("Content-Length",length)
 1393:         self.REQUEST.RESPONSE.write(ret)    
 1394:         
 1395:     security.declareProtected('manage','downloadObjectsAsOneFile')
 1396:     def downloadObjectsAsOneFile(self,lock=None,procedure=None,REQUEST=None,check="yes",current="no"):
 1397:         """download all selected files in one file"""
 1398:         
 1399:         if self.temp_folder.downloadCounterBaskets > 10000:
 1400:             return """I am sorry, currently the server has to many requests for downloads, please come back later!"""
 1401: 
 1402: 
 1403:         if (check=="yes") and self.containsNonActualFiles():
 1404:             pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadObjectAsOneFile_check.zpt')).__of__(self)
 1405:             
 1406:             return pt(lock=lock)
 1407:             
 1408:         else:
 1409:             
 1410:             return self.downloadObjectsAsOneFileFinally(lock=lock,procedure=procedure,REQUEST=REQUEST,current="no")
 1411:         
 1412:     def downloadObjectsAsOneFileFinally(self,lock=None,procedure=None,REQUEST=None,current="no",repeat=None):
 1413:         """print do the download"""
 1414:  
 1415:         
 1416:         ret=""
 1417:         lockedObjects={}
 1418: 
 1419:            
 1420:     
 1421:         if lock:
 1422:             logging.debug("------lock:"+repr(lock))
 1423:             if str(self.REQUEST['AUTHENTICATED_USER'])=='Anonymous User':
 1424:                 
 1425:                 return "please login first"
 1426: 
 1427:             #check if a locked object exist in the basket.
 1428:             lockedObjects={}
 1429:             for object in self.content.getContent():
 1430: 
 1431:                 if (not str(object[1].lockedBy)=="") and (not (str(object[1].lockedBy)==str(self.REQUEST['AUTHENTICATED_USER']))):
 1432:                     lockedObjects[object[1].title]=repr(object[1].lockedBy)
 1433:                    
 1434:                     
 1435:             keys=lockedObjects.keys()
 1436:             
 1437:             
 1438:             if len(keys)>0 and (not procedure):
 1439:                 self.REQUEST.SESSION['lockedObjects']=lockedObjects
 1440:                 pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','lockedObjects.zpt')).__of__(self)
 1441:                 
 1442:                
 1443:                 return pt()
 1444:          
 1445:             elif not procedure: #keine fails gesperrt dann alle donwloaden
 1446:                 procedure="downloadAll" 
 1447:         
 1448:         
 1449:        
 1450:        
 1451:         threadName=repeat
 1452:         if not threadName or threadName=="":
 1453:             thread=DownloadBasketFinallyThread()
 1454:             threadName=thread.getName()[0:]
 1455: 
 1456:             if (not hasattr(self,'_v_downloadBasket')):
 1457:                                 self._v_downloadBasket={}
 1458: 
 1459: 
 1460:             self._v_downloadBasket[threadName]=thread
 1461:             logging.debug("dwonloadfinally:"+repr(self))
 1462: 
 1463:             if isinstance(self,CDLIBasketVersion):
 1464:                 obj=self
 1465:             else:
 1466:                 obj=self.aq_parent
 1467:             logging.debug("dwonloadfinally2:"+repr(obj))
 1468:             logging.debug("dwonloadfinally2:"+repr(obj.aq_parent))
 1469: 
 1470:             obj2=obj.aq_parent
 1471:             if not isinstance(obj2,CDLIBasket):
 1472:                 obj2=obj2.aq_parent
 1473: 
 1474:             basketID=obj2.getId()
 1475:             versionNumber=obj.getId()
 1476:             logging.debug("dwonloadfinally2:"+repr(basketID))
 1477:             logging.debug("dwonloadfinally2:"+repr(versionNumber))
 1478: 
 1479: 
 1480:             if lock:
 1481:                 logging.debug("-----start locking")
 1482:                 for object in self.content.getContent():
 1483:                          if object[1].lockedBy =='':
 1484:                              object[1].lockedBy=self.REQUEST['AUTHENTICATED_USER']
 1485:                 logging.debug("-----finished locking")
 1486:                 
 1487:                     #obj.lockedBy=user
 1488:             self._v_downloadBasket[threadName].set(lock,procedure,self.REQUEST['AUTHENTICATED_USER'],current,basketID,versionNumber)
 1489: 
 1490:             self._v_downloadBasket[threadName].start()
 1491: 
 1492:             
 1493:             
 1494:             wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
 1495: 
 1496:             if wait_template:
 1497:                 return wait_template[0][1]()
 1498:             pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadBasketWait.zpt')).__of__(self)
 1499: 
 1500:             return pt(txt=self.absolute_url()+'/downloadObjectsAsOneFileFinally',threadName=threadName,
 1501:                                 counter=self._v_downloadBasket[threadName].getCounter(),
 1502:                                 number=self._v_downloadBasket[threadName].getNumberOfFiles())
 1503:             #_v_xmltrans.run()
 1504:         
 1505:         else:
 1506:             #recover thread, if lost
 1507:             if not hasattr(self,'_v_downloadBasket'):
 1508:                self._v_downloadBasket={}
 1509:             if not self._v_downloadBasket.get(threadName,None):
 1510:                  for thread in threading.enumerate():
 1511:                          if threadName == thread.getName():
 1512:                                        self._v_downloadBasket[threadName]=thread
 1513:                                        
 1514:             if self._v_downloadBasket.get(threadName,None) and (self._v_downloadBasket[threadName] is not None) and (not self._v_downloadBasket[threadName].end) :
 1515: 
 1516:                 wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
 1517:                 if wait_template:
 1518:                         return wait_template[0][1]()
 1519:                 
 1520:                 pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadBasketWait.zpt')).__of__(self)
 1521:                 return pt(txt=self.absolute_url()+'/downloadObjectsAsOneFileFinally',threadName=threadName,
 1522:                           counter=self._v_downloadBasket[threadName].getCounter(),
 1523:                           number=self._v_downloadBasket[threadName].getNumberOfFiles())
 1524:             else:
 1525:               
 1526:              
 1527:               logging.debug("FINISHED")
 1528:               if not self._v_downloadBasket.get(threadName,None):
 1529:                  for thread in threading.enumerate():
 1530:                          if threadName == thread.getName():
 1531:                                        self._v_downloadBasket[threadName]=thread
 1532:                                        
 1533:               #files = self._v_downloadBasket[threadName].result
 1534:               files=self.basketContainer.resultHash[threadName]
 1535:               lockedFiles=self.basketContainer.resultLockedHash[threadName]
 1536:      
 1537:              # fh=file("/var/tmp/test")
 1538:               #ret =fh.read()
 1539:          
 1540:               if (not isinstance(self.aq_parent,CDLIBasket)):
 1541:                   basket_name=self.aq_parent.aq_parent.title+"_V"+self.getId()
 1542:               else:
 1543:                   basket_name=self.aq_parent.title+"_V"+self.getId()
 1544:         
 1545:         
 1546:     
 1547:                   #write basketname to header of atf file
 1548:               
 1549: 
 1550:               self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.atf" """%basket_name)
 1551:               self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream")
 1552:               #length=len(ret)
 1553:               #self.REQUEST.RESPONSE.setHeader("Content-Length",length)
 1554:         
 1555:               ret="#basket: %s\n"%basket_name
 1556:               self.REQUEST.RESPONSE.write(ret) 
 1557:                  
 1558:               for fileName in files:
 1559:                 try:
 1560:                   self.REQUEST.RESPONSE.write(file(fileName).read())
 1561:                 except:
 1562:                   logging.error("downloadasonefile: cannot read %s"%fileName)
 1563:                   
 1564:             
 1565:               self.REQUEST.RESPONSE.write("\n# locked files\n")
 1566:               for fileName in lockedFiles:
 1567:                   self.REQUEST.RESPONSE.write("#  %s by %s\n"%fileName)
 1568:               
 1569:               self.REQUEST.RESPONSE.write("# locked files end\n")
 1570:               
 1571:               del self.basketContainer.resultHash[threadName]
 1572:               del self.basketContainer.resultLockedHash[threadName]
 1573:              
 1574:     def numberOfItems(self):
 1575:         """return anzahl der elemente im basket"""
 1576:         return self.content.numberOfItems()
 1577:     
 1578:     def getTime(self):
 1579:         """getTime"""
 1580:         #return self.bobobase_modification_time().ISO()
 1581:       
 1582:         if hasattr(self,'time'):
 1583:             return time.strftime("%Y-%m-%d %H:%M:%S",self.time)
 1584:         elif hasattr(self,'timefixed'):
 1585:             return self.timefixed
 1586:         else:
 1587:             setattr(self,'timefixed',self.bobobase_modification_time().ISO())
 1588:             return self.bobobase_modification_time().ISO()
 1589:     
 1590:     def getContent(self):
 1591:         """get Basket Content"""
 1592:         return self.content.getContent()
 1593: 
 1594:     
 1595:     def __init__(self,id,user,comment="",basketContent=[]):
 1596:         """ init a basket version"""
 1597:         self.id=id
 1598:         self.comment=comment
 1599:         self._setObject('content',BasketContent(basketContent))
 1600:         #self.basketContent=basketContent[0:]a
 1601:         self.user=user
 1602:         self.time=time.localtime()
 1603:         
 1604:     def getUser(self):
 1605:         """get user"""
 1606:         return self.user
 1607:     
 1608:     def getComment(self):
 1609:         """get Comment"""
 1610:         return self.comment
 1611:  
 1612:     security.declareProtected('manage','index_html')
 1613:     def index_html(self):
 1614:             """view the basket"""
 1615: 
 1616:             if self.REQUEST.get('change',False):
 1617:                     ob=self.aq_parent.updateObjects(self.REQUEST['change'])
 1618:                    
 1619:                     self.REQUEST.RESPONSE.redirect(ob.absolute_url())#go to new basket, because changing generates a new basket
 1620:                                         
 1621:             pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketVersionMain.zpt')).__of__(self)
 1622:             return pt()
 1623:      
 1624:     def getObjUrl(self,result):
 1625:         """getUrl of the version of the object"""
 1626:         objId=result[1].getTitle()
 1627:         founds=self.CDLICatalog.search({'title':objId})
 1628:         if len(founds)>0:
 1629:              return founds[0].getObject().getLastVersion().absolute_url()
 1630:          
 1631:         else: #assume version number
 1632:             splitted=objId.split("_")
 1633:             founds=self.CDLICatalog.search({'title':splitted[1]})        
 1634:             return founds[0].getObject().getLastVersion().absolute_url()+'/'+objId
 1635:    
 1636: def manage_addCDLIBasketVersion(self,user,comment="",basketContent=[],RESPONSE=None):
 1637:     """add a version"""
 1638:     
 1639:     #check for already existing versions
 1640:  
 1641:     lastVersion=self.getLastVersion()
 1642:     if lastVersion is None:
 1643:         newId=str(1)
 1644:     else:
 1645:         newId=str(int(lastVersion.getId())+1)
 1646:     
 1647:     ob=CDLIBasketVersion(newId,user,comment,basketContent)
 1648:     
 1649:     self._setObject(newId, ob)
 1650:     
 1651:     if RESPONSE is not None:
 1652:         RESPONSE.redirect('manage_main')
 1653:     else:
 1654:         return ob
 1655:     
 1656: class CDLIFileObject(CatalogAware,extVersionedFileObject):
 1657:     """CDLI file object"""
 1658:     
 1659:     meta_type="CDLI File Object"
 1660:     default_catalog='CDLIObjectsCatalog'
 1661:     
 1662:     security=ClassSecurityInfo()
 1663:     
 1664:     security.declareProtected('manage','index_html')
 1665: 
 1666:     security.declarePublic('view')
 1667:     view = PageTemplateFile('zpt/viewCDLIFile.zpt', globals())
 1668: 
 1669:     security.declarePublic('editATF')
 1670:     editATF = PageTemplateFile('zpt/editATFFile.zpt', globals())
 1671: 
 1672:     def PrincipiaSearchSource(self):
 1673:            """Return cataloguable key for ourselves."""
 1674:            return str(self)
 1675:        
 1676:     def setAuthor(self, author):
 1677:         """change the author"""
 1678:         self.author = author
 1679:        
 1680:     def makeThisVersionCurrent_html(self):
 1681:         """form for mthis version current"""
 1682:         
 1683:         pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','makeThisVersionCurrent.zpt')).__of__(self)
 1684:         return pt()                 
 1685: 
 1686:     security.declarePublic('makeThisVersionCurrent')
 1687:     def makeThisVersionCurrent(self,comment,author,RESPONSE=None):
 1688:         """copy this version to current"""
 1689:         parent=self.aq_parent
 1690:         parent.manage_addVersionedFileObject(id=None,vC=comment,author=author,file=self.getData(),RESPONSE=RESPONSE)
 1691:         #newversion=parent.manage_addCDLIFileObject('',comment,author)
 1692:         #newversion.manage_upload(self.getData())
 1693:                                         
 1694:         #if RESPONSE is not None:
 1695:         #    RESPONSE.redirect(self.aq_parent.absolute_url()+'/history')
 1696: 
 1697:         return True
 1698:     
 1699:     def getFormattedData(self):
 1700:         """fromat text"""
 1701:         data=self.getData()
 1702: #        return re.sub("\s\#lem"," #lem",data) #remove return vor #lem
 1703:         return re.sub("#lem","       #lem",data) #remove return vor #lem
 1704:         
 1705:     
 1706:     security.declarePublic('getPNumber')
 1707:     def getPNumber(self):
 1708:         """get the pnumber"""
 1709:         try:
 1710:                 txt=re.match("&[Pp](\d*)\s*=([^\r\n]*)",self.getData()[0:])
 1711:         except:
 1712:                 txt=self.getData()[0:]
 1713:                 
 1714:                 return "ERROR"
 1715:         try:
 1716:             return "P"+txt.group(1)
 1717:         except:
 1718:             return "ERROR"
 1719: 
 1720:     security.declarePublic('getDesignation')
 1721:     def getDesignation(self):
 1722:         """get the designation out of the file"""
 1723:         try:
 1724:                 txt=re.match("&[Pp](\d*)\s*=([^\r\n]*)",self.getData()[0:])
 1725:         except:
 1726:                 txt=self.getData()[0:]
 1727:                 
 1728:                 return "ERROR"
 1729:         try:
 1730:             return txt.group(2)
 1731:         except:
 1732:             return "ERROR"
 1733: 
 1734:         
 1735: manage_addCDLIFileObjectForm=DTMLFile('dtml/fileAdd', globals(),Kind='CDLIFileObject',kind='CDLIFileObject', version='1')
 1736: 
 1737: def manage_addCDLIFileObject(self,id,vC='',author='', file='',title='',versionNumber=0,
 1738:                              precondition='', content_type='',
 1739:                              from_tmp=False,REQUEST=None):
 1740:     """Add a new File object.
 1741:     Creates a new File object 'id' with the contents of 'file'"""
 1742:  
 1743:     id=str(id)
 1744:     title=str(title)
 1745:     content_type=str(content_type)
 1746:     precondition=str(precondition)
 1747:     
 1748:     id, title = cookId(id, title, file)
 1749: 
 1750:     self=self.this()
 1751: 
 1752:     # First, we create the file without data:
 1753:     self._setObject(id, CDLIFileObject(id,title,versionNumber=versionNumber,versionComment=vC,time=time.localtime(),author=author))
 1754:     fob = self._getOb(id)
 1755:     
 1756:     # Now we "upload" the data.  By doing this in two steps, we
 1757:     # can use a database trick to make the upload more efficient.
 1758: 
 1759:     if file and not from_tmp:
 1760:         fob.manage_upload(file)
 1761:     elif file and from_tmp:
 1762:         fob.manage_file_upload(file) # manage_upload_from_tmp doesn't exist in ExtFile2
 1763:     #    fob.manage_upload_from_tmp(file) # manage_upload_from_tmp doesn't exist in ExtFile2
 1764:     if content_type:
 1765:         fob.content_type=content_type
 1766: 
 1767:     #logging.debug("manage_add: lastversion=%s"%self.getData())
 1768:     logging.debug("reindex1: %s in %s"%(repr(self),repr(self.default_catalog)))
 1769:     self.reindex_object()
 1770:     #logging.debug("manage_add: fob_data=%s"%fob.getData())
 1771:     logging.debug("reindex2: %s in %s"%(repr(fob), repr(fob.default_catalog)))
 1772:     fob.index_object()
 1773: 
 1774:     self.CDLIRoot.updateOrAddToFileBTree(ob)
 1775:     if REQUEST is not None:
 1776:         REQUEST['RESPONSE'].redirect(self.absolute_url()+'/manage_main')
 1777:     
 1778: 
 1779: class CDLIFile(extVersionedFile,CatalogAware):
 1780:     """CDLI file"""
 1781:     
 1782:     security=ClassSecurityInfo()
 1783:     meta_type="CDLI file"
 1784:     content_meta_type = ["CDLI File Object"]
 1785:     
 1786:     default_catalog='CDLICatalog'
 1787:     
 1788:     security.declareProtected('manage','index_html')
 1789:     
 1790:     def getLastVersionData(self):
 1791:         """get last version data"""
 1792:         return self.getData()
 1793: 
 1794:     def getLastVersionFormattedData(self):
 1795:         """get last version data"""
 1796:         return self.getContentObject().getFormattedData()
 1797: 
 1798:     def getTextId(self):
 1799:         """returns P-number of text"""
 1800:         # assuming that its the beginning of the title
 1801:         return self.title[:7]
 1802: 
 1803:     #security.declarePublic('history')
 1804:     def history(self):
 1805:         """history"""  
 1806: 
 1807:         ext=self.ZopeFind(self.aq_parent,obj_ids=["history_template.html"])
 1808:         if ext:
 1809:             return getattr(self,ext[0][1].getId())()
 1810:         
 1811:         pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','versionHistory')).__of__(self)
 1812:         return pt()
 1813: 
 1814: 
 1815:     def getBasketFromId(self,basketid, context=None):
 1816:         """get basket from id"""
 1817: 
 1818:         if not context:
 1819:             context=self
 1820:             
 1821:         for basket in self.ZopeFind(context,obj_metatypes=["CDLIBasket"]):
 1822:             if basket[0]==basketid:
 1823:                 return basket[1]
 1824:         else:
 1825:             None
 1826: 
 1827:  
 1828:     def isContainedInBaskets(self,context=None):
 1829:         """check is this file is part of any basket
 1830:         @param context: (optional) necessessary if CDLIBasketCatalog is not an (inherited) attribute of self, context.CDLIBasketCatalog
 1831:                         has to exist.
 1832:         """
 1833: 
 1834:         if not context:
 1835:             context=self
 1836:         
 1837:         ret=[]
 1838:         for x in context.CDLIBasketCatalog.search({'getFileNamesInLastVersion':self.getId()}):
 1839:             #if the basket x is deleted it seemes to be that x is sometimes still in the Catalog, why?
 1840:             try:
 1841:                 ret.append(x.getObject())
 1842:             except:
 1843:                 pass
 1844:         return ret
 1845:         #return [x.getObject() for x in context.CDLIBasketCatalog.search({'getFileNamesInLastVersion':self.getId()})]
 1846:         
 1847:         
 1848:     def _newContentObject(self, id, title='', versionNumber=0, versionComment=None, time=None, author=None):
 1849:         """factory for content objects. to be overridden in derived classes."""
 1850:         logging.debug("_newContentObject(CDLI)")
 1851:         return CDLIFileObject(id,title,versionNumber=versionNumber,versionComment=versionComment,time=time,author=author)
 1852: 
 1853: 
 1854:     def addCDLIFileObjectForm(self):
 1855:         """add a new version"""
 1856:         
 1857:         if str(self.REQUEST['AUTHENTICATED_USER']) in ["Anonymous User"]:
 1858:             return "please login first"
 1859:         if (self.lockedBy==self.REQUEST['AUTHENTICATED_USER']) or (self.lockedBy==''):
 1860:             out=DTMLFile('dtml/fileAdd', globals(),Kind='CDLIFileObject',kind='CDLIFileObject',version=self.getVersion()).__of__(self)
 1861:             return out()
 1862:         else:
 1863:             return "Sorry file is locked by somebody else"
 1864:         
 1865:     def manage_addCDLIFileObject(self,id,vC,author,
 1866:                                  file='',title='',
 1867:                                  precondition='', 
 1868:                                  content_type='',
 1869:                                  changeName='no',newName='', 
 1870:                                  come_from=None,
 1871:                                  from_tmp=False,RESPONSE=None):
 1872:         """add"""
 1873:       
 1874:         try: #TODO: der ganze vC unsinn muss ueberarbeitet werden
 1875:             vC=self.REQUEST['vC']
 1876:         except:
 1877:             pass
 1878:         
 1879:         ob = self.addContentObject(id, vC, author, file, title, changeName=changeName, newName=newName, from_tmp=from_tmp,
 1880:                                    precondition=precondition, content_type=content_type)
 1881: 
 1882:         try:
 1883:             #FIXME: wozu ist das gut?
 1884:             self.REQUEST.SESSION['objID_parent']=self.getId()
 1885:         except:
 1886:             pass
 1887:   
 1888:         #self.cdliRoot.updateOrAddToFileBTree(self)# now update the object in the cache
 1889:       
 1890:         
 1891:         if RESPONSE:
 1892:             if ob.getSize()==0:
 1893:                 self.REQUEST.SESSION['objID']=ob.getId()
 1894:                 pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','errorUploadFile')).__of__(self)
 1895:                 return pt()
 1896:             else:
 1897:                 if come_from and (come_from!=""):
 1898:                     RESPONSE.redirect(come_from+"?change="+self.getId())
 1899:                 else:
 1900:                     RESPONSE.redirect(self.REQUEST['URL2']+'?uploaded=%s'%self.title)
 1901:         else:
 1902:             return ob
 1903:         
 1904:         
 1905: def manage_addCDLIFileForm(self):
 1906:     """interface for adding the OSAS_root"""
 1907:     pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIFile.zpt')).__of__(self)
 1908:     return pt()
 1909: 
 1910: def manage_addCDLIFile(self,id,title,lockedBy, author=None, RESPONSE=None):
 1911:     """add the OSAS_root"""
 1912:     newObj=CDLIFile(id,title,lockedBy,author)
 1913:                                         
 1914:     tryToggle=True
 1915:     tryCount=0
 1916: 
 1917:     self._setObject(id,newObj)                  
 1918:     getattr(self,id).reindex_object()
 1919:         
 1920:     if RESPONSE is not None:
 1921:         RESPONSE.redirect('manage_main')
 1922: 
 1923: 
 1924: def checkUTF8(data):
 1925:     """check utf 8"""
 1926:     try:
 1927:         data.encode('utf-8')
 1928:         return True
 1929:     except:
 1930:         return False
 1931:     
 1932: 
 1933: def checkFile(filename,data,folder):
 1934:     """check the files"""
 1935:     # first check the file name
 1936:     fn=filename.split(".") # no extension
 1937: 
 1938:     if not fn[0][0]=="P":
 1939:         return False,"P missing in the filename"
 1940:     elif len(fn[0])!=7:
 1941:         return False,"P number has not the right length 6"
 1942:     elif not checkUTF8(data):
 1943:         return False,"not utf-8"
 1944:     else:
 1945:         return True,""
 1946:     
 1947:     
 1948: def splitatf(fh,dir=None,ext=None):
 1949:     """split it"""
 1950:     ret=None
 1951:     nf=None
 1952:     i=0
 1953: 
 1954:     #ROC: why split \n first and then \r???
 1955:     if (type(fh) is StringType) or (type(fh) is UnicodeType):
 1956:         iter=fh.split("\n")
 1957:     else:
 1958:         iter=fh.readlines()
 1959:         
 1960:     for lineTmp in iter:
 1961:         lineTmp=lineTmp.replace(codecs.BOM_UTF8,'') # make sure that all BOM are removed..
 1962:         for line in lineTmp.split("\r"):
 1963:             #logging.log("Deal with: %s"%line)
 1964:             if ext:
 1965:                 i+=1
 1966:                 if (i%100)==0:
 1967:                     ext.result+="."
 1968:                 if i==10000:
 1969:                     i=0
 1970:                     ext.result+="<br>"
 1971:             #check if basket name is in the first line
 1972:             if line.find("#atf basket")>=0: #old convention
 1973:                 ret=line.replace('#atf basket ','')
 1974:                 ret=ret.split('_')[0]
 1975:             elif line.find("#basket:")>=0: #new convention
 1976:                 ret=line.replace('#basket: ','')
 1977:                 ret=ret.split('_')[0]
 1978: 
 1979:             else:
 1980:                 if (len(line.lstrip())>0) and (line.lstrip()[0]=="&"): #newfile
 1981:                     if nf:
 1982:                         nf.close() #close last file
 1983: 
 1984: 
 1985:                     filename=line[1:].split("=")[0].rstrip()+".atf"
 1986:                     if dir:
 1987:                         filename=os.path.join(dir,filename)
 1988:                     nf=file(filename,"w")
 1989:                     logging.info("open %s"%filename)
 1990:                 if nf:    
 1991:                     nf.write(line.replace("\n","")+"\n")
 1992: 
 1993:     try:        
 1994:         nf.close()
 1995:     except:
 1996:         pass
 1997:     
 1998:     if not((type(fh) is StringType) or (type(fh) is UnicodeType)):
 1999:         fh.close()
 2000:     return ret,len(os.listdir(dir))
 2001: 
 2002: 
 2003: class CDLIFileFolder(extVersionedFileFolder):
 2004:     """CDLI File Folder"""
 2005:     
 2006:     security=ClassSecurityInfo()
 2007:     meta_type="CDLI Folder"
 2008:     file_meta_type=['CDLI file']
 2009:     folder_meta_type=['CDLI Folder']
 2010: 
 2011:     file_catalog='CDLICatalog'
 2012: 
 2013:     #downloadCounter=0 # counts how many download for all files currently run, be mehr als 5 wird verweigert.
 2014:     tmpStore2={}
 2015: 
 2016:     def _newVersionedFile(self, id, title='', lockedBy=None, author=None):
 2017:         """factory for versioned files. to be overridden in derived classes."""
 2018:         logging.debug("_newVersionedFile(CDLI)")
 2019:         return CDLIFile(id, title, lockedBy=lockedBy, author=author)
 2020: 
 2021:     def setTemp(self,name,value):
 2022:         """set tmp"""
 2023: 
 2024:         setattr(self,name,value)
 2025:                                         
 2026:     deleteFileForm = PageTemplateFile("zpt/doDeleteFile", globals())
 2027:                                        
 2028:     def delete(self,ids,REQUEST=None):
 2029:         """delete these files"""
 2030:         if type(ids) is not ListType:
 2031:             ids=[ids]
 2032: 
 2033:         self.manage_delObjects(ids)
 2034:         
 2035:         if REQUEST is not None:
 2036:             return self.index_html()
 2037: 
 2038: 
 2039:     def getVersionNumbersFromIds(self,ids):
 2040:         """get the numbers of the current versions of documents described by their ids"""
 2041:         
 2042:         ret=[]
 2043:         searchStr=" OR ".join(ids)
 2044:         
 2045:         founds=self.CDLICatalog.search({'title':searchStr})
 2046:         
 2047:         for found in founds:
 2048:             lastVersion=found.getObject().getContentObject()
 2049:             ret.append((found.getId,lastVersion))
 2050:         
 2051:         return ret
 2052:     
 2053:     def getFile(self,fn):
 2054:         """get the content of the file fn"""
 2055:         logging.debug("getFile: %s"%repr(fn))
 2056:         if not self.hasObject(fn):
 2057:             # search deeper
 2058:             founds=getattr(self, self.file_catalog).search({'textid':fn})
 2059:             if founds:
 2060:                 obj=founds[0].getObject().getContentObject()
 2061:             else:
 2062:                 return "" 
 2063:         else:
 2064:             obj = self[fn].getContentObject()
 2065: 
 2066:         return obj.getData()[0:] 
 2067:  
 2068:     
 2069:     def checkCatalog(self,fn):
 2070:         """check if fn is in the catalog"""
 2071:         #TODO add checkCatalog
 2072:         
 2073:                                    
 2074:     def findObjectsFromListWithVersion(self,list,author=None):
 2075:         """find objects from a list with versions
 2076:         @param list: list of tuples  (cdliFile,version)
 2077:         """
 2078:         #self.REQUEST.SESSION['fileIds']=list#store fieldIds in session for further usage
 2079:         #self.REQUEST.SESSION['searchList']=self.REQUEST.SESSION['fileIds']
 2080:         
 2081:         pt=getattr(self,'filelistVersioned.html')
 2082:             
 2083:         return pt(search=list,author=author)
 2084:     
 2085:     
 2086:     def getAllPNumbers(self):
 2087:         """get a list of all files (resp their p-numbers) stored"""
 2088:         
 2089:         ret=[x.getId for x in  self.CDLICatalog()]
 2090:      
 2091:         return ret
 2092:     
 2093:     def expandFile(self,fileId,fileTree):
 2094:         """wildcard in fileID suche alle Treffer"""
 2095:         founds=self.CDLICatalog({'title':fileId})
 2096:         for found in founds:
 2097:             fileTree.add(found.getId)
 2098:             logging.debug("ADDD:"+found.getId)
 2099:          
 2100:     def findObjectsFromList(self,enterList=None,display=False,start=None,upload=None,list=None,basketName=None,numberOfObjects=None,RESPONSE=None,REQUEST=None,returnHash=False,hash=None):
 2101:         """findObjectsFromList (, TAB oder LINE separated)"""
 2102:                                        
 2103:         logging.debug("start: findObjectsFromList")
 2104:         #logging.debug("start: findObjectsFromList"+repr(list))
 2105:         
 2106:             
 2107:         if upload: # list from file upload
 2108:             txt=upload.read()
 2109:                                        
 2110:         if enterList:
 2111:             txt=enterList
 2112:             
 2113:         if upload or enterList:
 2114:             txt=txt.replace(",","\n")
 2115:             txt=txt.replace("\t","\n")
 2116:             txt=txt.replace("\r","\n")
 2117:             idsTmp=txt.split("\n")
 2118:             ids=[]
 2119:             for id in idsTmp: # make sure that no empty lines
 2120:                 idTmp=id.lstrip().rstrip()
 2121:                 if len(idTmp)>0:
 2122:                     
 2123:                     ids.append(idTmp)
 2124:                     
 2125:             #self.REQUEST.SESSION['ids']=" OR ".join(ids)
 2126: 
 2127:             pt=getattr(self,'filelist.html')
 2128:             self.REQUEST.SESSION['searchList']=ids
 2129:             return pt(search=ids)
 2130:         
 2131:         if basketName:
 2132:             #TODO: get rid of one of these..
 2133:             
 2134:             pt=getattr(self,'filelist.html')
 2135:             return pt(basketName=basketName,numberOfObjects=numberOfObjects)
 2136:         
 2137:         if hash is not None and hasattr(self.cdliRoot,'v_tmpStore') and self.cdliRoot.v_tmpStore.has_key(hash): 
 2138:                
 2139:                logging.debug("asking for storage2")
 2140:                result =self.cdliRoot.v_tmpStore[hash]
 2141:                if result:
 2142:                    logging.debug("give result from storage2")
 2143:                    return hash,self.cdliRoot.v_tmpStore[hash]
 2144:           
 2145:         if list is not None: # got already a list
 2146:             
 2147:             logging.debug(" ----List version")
 2148:             ret=[]
 2149:             fileTree=Set()
 2150:             
 2151:             for fileId in list:
 2152:                
 2153:                 if fileId.find("*")>-1: #check for wildcards
 2154:                         self.expandFile(fileId,fileTree)
 2155:                         
 2156:                 elif len(fileId.split("."))==1:
 2157:                         fileId=fileId+".atf"
 2158:                         fileTree.add(fileId)
 2159:                 #logging.debug("   -----:"+fileId)
 2160:                 #ret+=self.CDLICatalog({'title':fileId})
 2161:                 #x =self.getFileObject(fileId)
 2162:                 #if x is not None:
 2163:                 #    ret.append(x)
 2164:                 
 2165:             
 2166:             
 2167:             ids = fileTree & self.v_file_ids
 2168:             #self.REQUEST.SESSION['fileIds']=ids#store fieldIds in session for further usage
 2169:             l=makelist(fileTree)[0:]
 2170:             #logging.debug("l-list:"+repr(l))
 2171:             self.REQUEST.SESSION['fileIds']=l#store fieldIds in session for further usage
 2172:             self.REQUEST.SESSION['searchList']=l
 2173:             #self.REQUEST.SESSION['searchList']=['P000001.atf']
 2174:           
 2175:             
 2176:             hash = md5.new(repr(makelist(fileTree))).hexdigest() # erzeuge hash als identification
 2177:             self.REQUEST.SESSION['hash']=hash
 2178:             #TODO: do I need garbage collection for v_tmpStore ?
 2179:             
 2180:             #logging.debug("Hash:"+repr(hash))
 2181: #        
 2182: #            if hasattr(self.cdliRoot,'v_tmpStore') and self.cdliRoot.v_tmpStore.has_key(hash): 
 2183: #               logging.debug("asking for storage")
 2184: #               res=self.cdliRoot.v_tmpStore[hash]
 2185: #               if res:
 2186: #                   if returnHash == True:
 2187: #                       return hash,res
 2188: #                   return res
 2189:           
 2190:             #TODO: get rid of one of these..
 2191:             #ids=[x.getObject().getId() for x in ret]
 2192:             ret=[(self.getFileObject(x),self.getFileObjectLastVersion(x)) for x in ids]
 2193:             
 2194:             #self.REQUEST.SESSION['fileIds']=ids#store fieldIds in session for further usage
 2195:             #self.REQUEST.SESSION['searchList']=self.REQUEST.SESSION['fileIds']
 2196:            
 2197:             if display:
 2198:                 pt=getattr(self,'filelist.html')
 2199:                 
 2200:                 return pt(search=ids)
 2201:             else:     
 2202:                 #self.REQUEST.SESSION['hash'] = ret # store in session 
 2203:                 if not hasattr(self,'v_tmpStore'):
 2204:                     self.cdliRoot.v_tmpStore={}
 2205:                 #logging.debug("HHHHHHNEU:"+repr(self.makelist(ids)))
 2206:                 #logging.debug("HHHHHHNEU:"+repr(hash))
 2207:                 self.cdliRoot.v_tmpStore[hash] = ret # store in session 
 2208:                 if returnHash == True:
 2209:                     return hash,ret
 2210:                 return ret
 2211:         
 2212:         
 2213:         
 2214:         if start:
 2215:             RESPONSE.redirect("filelist.html?start:int="+str(start))
 2216: 
 2217:     security.declareProtected('Manage','createAllFilesAsSingleFile')
 2218:     def createAllFilesAsSingleFile(self,RESPONSE=None):
 2219:         """download all files"""
 2220:         
 2221:         def sortF(x,y):
 2222:             return cmp(x[0],y[0])
 2223:         
 2224:         catalog=getattr(self,self.file_catalog)
 2225:         #tf,tfilename=mkstemp()
 2226:         if not hasattr(self.temp_folder,'downloadCounter'):
 2227:             self.temp_folder.downloadCounter=0
 2228: 
 2229:         if getattr(self.temp_folder,'downloadCounter',0) > 5:
 2230:             return """I am sorry, currently the server has to many requests for downloads, please come back later!"""
 2231: 
 2232:         self.temp_folder.downloadCounter+=1
 2233:         self._p_changed=1
 2234:         transaction.get().commit()
 2235:        
 2236:         list=[(x.getId,x) for x in catalog()]
 2237:         list.sort(sortF)
 2238:         
 2239: 
 2240:         
 2241:         RESPONSE.setHeader("Content-Disposition","""attachement; filename=%s"""%"all.atf")
 2242:         RESPONSE.setHeader("Content-Type","application/octet-stream")
 2243:         tmp=""
 2244:         for l in list:
 2245:             obj=l[1].getObject()
 2246:             
 2247:             if obj.meta_type=="CDLI file":
 2248:                 
 2249:                 #os.write(tf,obj.getLastVersion().data)
 2250:                 if RESPONSE:
 2251:                     RESPONSE.write(obj.getData()[0:])
 2252:                     RESPONSE.write("\n")
 2253:                 self.temp_folder.downloadCounter-=1 
 2254:                 self._p_changed=1
 2255:         transaction.get().commit()
 2256:         #os.close(tf)
 2257:         #RESPONSE.redirect(self.absolute_url()+"/downloadFile?fn="%tfilename)
 2258:         return True
 2259:     
 2260:     def downloadFile(self,fn):
 2261:         """download fn - not used yet"""
 2262:         self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename=%s"""%self.getLastVersion().getId())
 2263:         self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream")
 2264:         self.REQUEST.RESPONSE.write(file(fn).read())
 2265:         
 2266:       
 2267:                 
 2268:     def hasParent(self):
 2269:         """returns true falls subfolder"""
 2270:       
 2271:         if self.aq_parent.meta_type in self.folder_meta_type:
 2272:             return True
 2273:         else:
 2274:             return False
 2275:         
 2276:     def getFolders(self):
 2277:         """get all subfolders"""
 2278:         ret=[]
 2279:         folders=self.ZopeFind(self,obj_metatypes=self.folder_meta_type)
 2280:         for folder in folders:
 2281:             ret.append((folder[1],
 2282:                         len(self.ZopeFind(folder[1],obj_metatypes=self.folder_meta_type)),
 2283:                         len(self.ZopeFind(folder[1],obj_metatypes=self.file_meta_type))
 2284:                         ))
 2285:         return ret
 2286:     
 2287:             
 2288:     security.declareProtected('manage','index_html')
 2289:     def index_html(self):
 2290:         """main"""
 2291:         ext=self.ZopeFind(self,obj_ids=["index.html"])
 2292:         if ext:
 2293:             return ext[0][1]()
 2294:         
 2295:         pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','CDLIFileFolderMain')).__of__(self)
 2296:         return pt()
 2297:     
 2298:     
 2299: manage_addCDLIFileFolderForm=DTMLFile('dtml/folderAdd', globals())
 2300: 
 2301:     
 2302: def manage_addCDLIFileFolder(self, id, title='',
 2303:                      createPublic=0,
 2304:                      createUserF=0,
 2305:                      REQUEST=None):
 2306:     """Add a new Folder object with id *id*.
 2307: 
 2308:     If the 'createPublic' and 'createUserF' parameters are set to any true
 2309:     value, an 'index_html' and a 'UserFolder' objects are created respectively
 2310:     in the new folder.
 2311:     """
 2312:     ob=CDLIFileFolder()
 2313:     ob.id=str(id)
 2314:     ob.title=title
 2315:     self._setObject(id, ob)
 2316:     ob=self._getOb(id)
 2317: 
 2318:     checkPermission=getSecurityManager().checkPermission
 2319: 
 2320:     if createUserF:
 2321:         if not checkPermission('Add User Folders', ob):
 2322:             raise Unauthorized, (
 2323:                   'You are not authorized to add User Folders.'
 2324:                   )
 2325:         ob.manage_addUserFolder()
 2326: 
 2327:   
 2328:     if REQUEST is not None:
 2329:         return self.manage_main(self, REQUEST, update_menu=1)
 2330:     
 2331: class CDLIRoot(Folder):
 2332:     """main folder for cdli"""
 2333:     
 2334:     meta_type="CDLIRoot"
 2335:     downloadCounterBaskets=0 # counts the current basket downloads if counter > 10 no downloads are possible
 2336:     
 2337:     file_catalog = 'CDLICatalog'
 2338:     
 2339:     # word splitter for search
 2340:     splitter = {'words':cdliSplitter.wordSplitter(),
 2341:                 'graphemes':cdliSplitter.graphemeSplitter()}
 2342:     
 2343:     
 2344:     def viewATF(self,id,RESPONSE):
 2345:         """view an Object"""
 2346:         ob = self.CDLICatalog({'title':id})
 2347:         logging.debug(ob[0].getObject().getLastVersion().absolute_url()+"/view")
 2348:         if len(ob)>0:
 2349:             RESPONSE.redirect(ob[0].getObject().getLastVersion().absolute_url()+"/view")
 2350:         return "not found"
 2351:     
 2352:     def history(self,id,RESPONSE):
 2353:         """view an Object"""
 2354:         ob = self.CDLICatalog({'title':id})
 2355:         if len(ob)>0:
 2356:             RESPONSE.redirect(ob[0].absolute_url+"/history")
 2357:         return "not found"
 2358:     
 2359: 
 2360:     def downloadLocked(self,id,RESPONSE):
 2361:         """view an Object"""
 2362:         ob = self.CDLICatalog({'title':id})
 2363:         if len(ob)>0:
 2364:             RESPONSE.redirect(ob[0].absolute_url+"/downloadLocked")
 2365:         return "not found"
 2366:     
 2367:     def download(self,id,RESPONSE):
 2368:         """view an Object"""
 2369:         ob = self.CDLICatalog({'title':id})
 2370:         if len(ob)>0:
 2371:             RESPONSE.redirect(ob[0].getLastVersion().absolute_url())
 2372:         return "not found"
 2373:     def addCDLIFileObjectForm(self,id,RESPONSE):
 2374:         """view an Object"""
 2375:         ob = self.CDLICatalog({'title':id})
 2376:         if len(ob)>0:
 2377:             RESPONSE.redirect(ob[0].absolute_url+"/addCDLIFileObjectForm")
 2378:         return "not found"
 2379:     
 2380:     def addVersionedFileObjectForm(self,id,RESPONSE):
 2381:         """view an Object"""
 2382:         ob = self.CDLICatalog({'title':id})
 2383:         if len(ob)>0:
 2384:             RESPONSE.redirect(ob[0].absolute_url+"/addVersionedFileObjectForm")
 2385:         return "not found"
 2386:     
 2387:     def unlock(self,id,RESPONSE):
 2388:         """view an Object"""
 2389:         ob = self.CDLICatalog({'title':id})
 2390:         if len(ob)>0:
 2391:             RESPONSE.redirect(ob[0].absolute_url+"/unlock")
 2392:         return "not found"
 2393:     
 2394:     def getFileObject(self,fileId):
 2395:         """get an object"""
 2396:         x=self.v_files.get(fileId)
 2397:         #logging.debug(x)
 2398:         return x
 2399:     
 2400:     def getFileObjectLastVersion(self,fileId):
 2401:         """get an object"""
 2402:         x=self.v_files_lastVersion.get(fileId)
 2403:         #logging.debug("lastVersion: "+repr(x))
 2404:         return x
 2405:     
 2406:     def showFileIds(self):
 2407:         """showIds"""
 2408:         return self.v_file_ids
 2409:     
 2410:     def generateFileBTree(self):
 2411:         """erzeuge einen Btree aus allen Files"""
 2412:         self.v_files = OOBTree()
 2413:         self.v_files_lastVersion = OOBTree()
 2414:         self.v_file_ids = Set()
 2415:         
 2416:         for x in self.CDLICatalog.searchResults():
 2417:             
 2418:             self.v_files.update({x.getId:x.getObject()})
 2419:             self.v_files_lastVersion.update({x.getId:x.getObject().getLastVersion()})
 2420:             self.v_file_ids.add(x.getId)
 2421:             logging.debug("add:"+x.getId+"XXX"+repr(x.getObject()))
 2422:         
 2423:         return True
 2424:     
 2425:     
 2426:     def updateOrAddToFileBTree(self,obj):
 2427:         """update a BTree"""
 2428:         self.v_files.update({obj.getId():obj})
 2429:         self.v_files_lastVersion.update({obj.getId():obj.getLastVersion()})
 2430:         
 2431:         self.v_file_ids.add(obj.getId())
 2432:         logging.debug("update:"+obj.getId()+"XXX"+repr(obj))
 2433:         
 2434:     def deleteFromBTree(self,objId):
 2435:         """delete an obj"""
 2436:         self.v_files.pop(objId)
 2437:         self.v_files_lastVersion.pop(objId)
 2438:         self.v_file_ids.remove(objId)
 2439:         
 2440: 
 2441:  
 2442:     def deleteFiles(self,ids):
 2443:         """delete files"""
 2444:         for id in ids:
 2445:             founds=self.CDLICatalog.search({'title':id.split(".")[0]})
 2446:             if founds:
 2447:                 logging.debug("deleting %s"%founds)
 2448:                 folder=founds[0].getObject().aq_parent #get the parent folder of the object
 2449:                 logging.debug("deleting from %s"%folder)
 2450:                 cut=folder.delete([founds[0].getId]) #cut it out
 2451: 
 2452: 
 2453: 
 2454:     def searchText(self, query, index='graphemes'):
 2455:         """searches query in the fulltext index and returns a list of file ids/P-numbers"""
 2456:         # see also: http://www.plope.com/Books/2_7Edition/SearchingZCatalog.stx#2-13
 2457:         logging.debug("searchtext for '%s' in index %s"%(query,index))
 2458:         #import Products.ZCTextIndex.QueryParser
 2459:         #qp = QueryParser.QueryParser()
 2460:         #logging.debug()
 2461:         idxQuery = {index:{'query':query}}
 2462:         idx = getattr(self, self.file_catalog)
 2463:         # do search
 2464:         resultset = idx.search(query_request=idxQuery,sort_index='textid')
 2465:         # put only the P-Number in the result 
 2466:         results = [res.getId[:7] for res in resultset]
 2467:         logging.debug("searchtext: found %d texts"%len(results))
 2468:         return results
 2469: 
 2470: 
 2471:     def getFile(self, pnum):
 2472:         """get the translit file with the given pnum"""
 2473:         f = getattr(self, self.file_catalog).search({'textid':pnum})
 2474:         if not f:
 2475:             return ""
 2476:         
 2477:         return f[0].getObject().getData()
 2478:          
 2479: 
 2480:     def showFile(self,fileId,wholePage=False):
 2481:         """show a file
 2482:         @param fileId: P-Number of the document to be displayed
 2483:         """
 2484:         f=getattr(self, self.file_catalog).search({'textid':fileId})
 2485:         if not f:
 2486:             return ""
 2487:         
 2488:         if wholePage:
 2489:             logging.debug("show whole page")
 2490:             return f[0].getObject().getContentObject().view()
 2491:         else:
 2492:             return f[0].getObject().getLastVersionFormattedData()
 2493:     
 2494: 
 2495:     def showWordInFile(self,fileId,word,indexName='graphemes',regExp=False,):
 2496:         """get lines with word from FileId"""
 2497:         logging.debug("showwordinfile word='%s' index=%s file=%s"%(word,indexName,fileId)) 
 2498:         
 2499:         file = formatAtfFullLineNum(self.getFile(fileId))
 2500:         ret=[]
 2501:         
 2502:         # add whitespace before and whitespace and line-end to splitter bounds expressions
 2503:         bounds = self.splitter[indexName].bounds
 2504:         splitexp = "(%s|\s)(%%s)(%s|\s|\Z)"%(bounds,bounds)
 2505:         # clean word expression 
 2506:         # TODO: this should use QueryParser itself
 2507:         # take out double quotes
 2508:         word = word.replace('"','')
 2509:         # take out ignorable signs
 2510:         ignorable = self.splitter[indexName].ignorex
 2511:         word = ignorable.sub('', word)
 2512:         # compile into regexp objects and escape parens
 2513:         wordlist = [re.compile(splitexp%re.escape(w)) for w in word.split(' ')]
 2514:             
 2515:         for line in file.splitlines():
 2516:             for word in wordlist:
 2517:                 #logging.debug("showwordinfile: searching for %s in %s"%(word.pattern,ignoreable.sub('',line)))
 2518:                 if word.search(ignorable.sub('',line)):
 2519:                     line = formatAtfLineHtml(line)
 2520:                     ret.append(line)
 2521:                     break
 2522:                     
 2523:         return ret
 2524: 
 2525:     
 2526:     def showWordInFiles(self,fileIds,word,indexName='graphemes',regExp=False):
 2527:         """
 2528:         get lines with word from all ids in list FileIds.
 2529:         returns dict with id:lines pairs.
 2530:         """
 2531:         logging.debug("showwordinfiles word='%s' index=%s file=%s"%(word,indexName,fileIds))
 2532:         
 2533:         return dict([(id,self.showWordInFile(id, word, indexName, regExp)) for id in fileIds])
 2534:     
 2535: 
 2536:     def tagWordInFile(self,fileId,word,indexName='graphemes',regExp=False):
 2537:         """get text with word highlighted from FileId"""
 2538:         logging.debug("tagwordinfile word='%s' index=%s file=%s"%(word,indexName,fileId)) 
 2539:         
 2540:         file=self.getFile(fileId)
 2541:         tagStart=u'<span class="found">'
 2542:         tagEnd=u'</span>'
 2543:         tagStr=tagStart + u'%%s' + tagEnd
 2544:         ret=[]
 2545:         
 2546:         # add whitespace to splitter bounds expressions and compile into regexp object
 2547:         bounds = self.splitter[indexName].bounds
 2548:         wordsplit = re.compile("(%s|\s)"%bounds)
 2549:         # clean word expression 
 2550:         # TODO: this should use QueryParser itself
 2551:         word = word.replace('"','') # take out double quotes
 2552:         # take out ignoreable signs
 2553:         ignorable = self.splitter[indexName].ignorex
 2554:         word = ignorable.sub('', word)
 2555:         # split search terms by blanks
 2556:         words = word.split(' ')
 2557:         # split search terms again (for grapheme search with words)
 2558:         splitwords = dict(((w,self.splitter[indexName].process([w])) for w in words))
 2559:             
 2560:         for line in file.splitlines():
 2561:             line = unicodify(line)
 2562:             # ignore lemma and other lines
 2563:             if line.lstrip().startswith('#lem:'):
 2564:                 continue
 2565:             # ignore p-num line
 2566:             if line.startswith('&P'):
 2567:                 continue
 2568:             # ignore version lines
 2569:             if line.startswith('#version'):
 2570:                 continue
 2571:             # ignore atf type lines
 2572:             if line.startswith('#atf:'):
 2573:                 continue
 2574: 
 2575:             # first scan
 2576:             hitwords = []
 2577:             for w in words:
 2578:                 if ignorable.sub('',line).find(w) > -1:
 2579:                     # word is in line
 2580:                     # append split word for grapheme search with words
 2581:                     hitwords.extend(splitwords[w])
 2582:                     #hitwords.extend(wordsplit.split(w))
 2583:                    
 2584:             # examine hits closer
 2585:             if hitwords:
 2586:                 # split line into words
 2587:                 parts = wordsplit.split(line)
 2588:                 line = ""
 2589:                 for p in parts:
 2590:                     #logging.debug("tagwordinfile: searching for %s in %s"%(p,hitwords))
 2591:                     # reassemble line
 2592:                     if ignorable.sub('', p) in hitwords:
 2593:                         #logging.debug("tagwordinfile: found %s in %s"%(p,hitwords))
 2594:                         # this part was found
 2595:                         line += tagStart + formatAtfHtml(p) + tagEnd
 2596:                     else:
 2597:                         line += formatAtfHtml(p)
 2598:                 
 2599:             else:
 2600:                 # no hits
 2601:                 line = formatAtfHtml(line)
 2602:             
 2603:             ret.append(line)
 2604:                         
 2605:         return u'<br>\n'.join(ret)
 2606: 
 2607: 
 2608: 
 2609:     def tagWordInFiles(self,fileIds,word,indexName='graphemes',regExp=False):
 2610:         """
 2611:         get texts with highlighted word from all ids in list FileIds.
 2612:         returns dict with id:text pairs.
 2613:         """
 2614:         logging.debug("tagwordinfiles word='%s' index=%s file=%s"%(word,indexName,fileIds)) 
 2615:         return dict([(id,self.tagWordInFile(id, word, indexName, regExp)) for id in fileIds])
 2616:     
 2617: 
 2618:     def getFileVersionList(self, pnum):
 2619:         """get the version history as a list for the translit file with the given pnum"""
 2620:         f = getattr(self, self.file_catalog).search({'textid':pnum})
 2621:         if not f:
 2622:             return []
 2623:         
 2624:         return f[0].getObject().getVersionList()
 2625:          
 2626: 
 2627:     def URLquote(self,str):
 2628:         """quote url"""
 2629:         return urllib.quote(str)
 2630:     
 2631:     def URLunquote(self,str):
 2632:         """unquote url"""
 2633:         return urllib.unquote(str)
 2634:     
 2635:     def URLquote_plus(self,str):
 2636:         """quote url"""
 2637:         return urllib.quote_plus(str)
 2638:     
 2639:     def URLunquote_plus(self,str):
 2640:         """unquote url"""
 2641:         return urllib.unquote_plus(str)
 2642:     
 2643:     
 2644:     def forceunlock(self):
 2645:         "break all locks"
 2646:         ret=[]
 2647:         for f in self.ZopeFind(self,obj_metatypes="CDLI file",search_sub=1):
 2648:            un=f[1].forceunlock()
 2649: 
 2650:            if un and un !="":
 2651:                ret.append((f[0],un))
 2652: 
 2653:         return ret
 2654:                                         
 2655: 
 2656:     def getChangesByAuthor(self,author,n=100):
 2657:         """getChangesByAuthor"""
 2658:         zcat=self.CDLIObjectsCatalog
 2659:         res=zcat({'lastEditor':author,
 2660:                      'sort_on':'getTime',
 2661:                      'sort_order':'descending',
 2662:                      'sort_limit':n})[:n ]
 2663:                        
 2664:         return res
 2665:     
 2666:     def getChangesByAuthor_html(self,author,n=100):
 2667:         """html output for changes by author"""
 2668:         tmp={}
 2669:         list=[]                         
 2670:         for x in self.getChangesByAuthor(author):
 2671:            nr=x.getObject().getVersionNumber()
 2672:            id=x.getObject().aq_parent.getId()
 2673:            #hinzufuegen, wenn Version neuer als die 
 2674:            if tmp.get(id,(0,0))[1] < nr:
 2675:                 tmp[id]=(x.getObject().aq_parent,nr)
 2676: 
 2677:      
 2678:         return self.cdli_main.findObjectsFromListWithVersion(list=tmp.values(),author=author)           
 2679:         
 2680:     def getLastChanges(self,n=100):
 2681:         """get the last n changes""" 
 2682:         n=int(n)                   
 2683:         zcat=self.CDLICatalog
 2684:         return zcat({'sort_on':'getLastChangeDate',
 2685:                      'sort_order':'descending',
 2686:                      'sort_limit':n})[:n ]
 2687:      
 2688:     
 2689:     def getLastChanges_html(self,n=100):
 2690:         """get the last n changes"""
 2691:         list = [x.getId for x in self.getLastChanges(n)]
 2692:         return self.cdli_main.findObjectsFromList(list=list,display=True)
 2693:                                        
 2694:     def refreshTxt(self,txt="",threadName=None):
 2695:         """txt fuer refresh"""
 2696:   
 2697:         return """ 2;url=%s?repeat=%s """%(self.absolute_url()+txt,threadName)
 2698: 
 2699:     def refreshTxtBasket(self,txt="",threadName=None):
 2700:         """txt fuer refresh"""
 2701:   
 2702:         return """ 2;url=%s?repeat=%s """%(txt,threadName)
 2703: 
 2704:     
 2705:     def getResult(self,threadName=None):
 2706:        """result of thread"""
 2707:        try:
 2708:         return self._v_uploadATF[threadName].getResult()
 2709:        except:
 2710:         return "One moment, please"
 2711:     
 2712:         
 2713:     def checkThreads(self):
 2714:         """check threads"""
 2715:         ret="<html><body>"
 2716:         for thread in threading.enumerate():
 2717:            ret+="<p>%s (%s): %s</p>"%(repr(thread),thread.getName(),thread.isAlive())
 2718:        
 2719:         return ret
 2720:                                        
 2721:                                            
 2722:     def uploadATFRPC(self,data,username):
 2723:         """upload an atffile via xml-rpc"""
 2724:         uploader=uploadATFThread()
 2725:         
 2726:         #generate an random id for the upload object
 2727:         from random import randint
 2728:         if (not self.REQUEST.SESSION.get('idTmp',None)):
 2729: 
 2730:             idTmp=str(randint(0,1000000000))
 2731:             self.REQUEST.SESSION['idTmp']=idTmp
 2732:         else:
 2733:             idTmp=self.REQUEST.SESSION.get('idTmp',None)
 2734:             
 2735:         
 2736:         uploader.set(data,0,username,idTmp)
 2737:         
 2738:         stObj=uploader.run()
 2739:         
 2740:         processor=uploadATFfinallyThread()
 2741:         
 2742:         basketname=stObj.returnValue['basketNameFromFile']
 2743:         
 2744:         processor.set("uploadchanged",basketname=basketname,SESSION=stObj.returnValue,username=username,serverport=self.REQUEST['SERVER_PORT'])
 2745:         
 2746:         processor.run()
 2747:         
 2748:         
 2749:         return generateXMLReturn(stObj.returnValue)
 2750:         
 2751:     def uploadATF(self,repeat=None,upload=None,basketId=0,RESPONSE=None):
 2752:         """upload an atf file / basket file"""
 2753:         #self._v_uploadATF.returnValue=None
 2754:         
 2755:         #generate an random id for the upload thread
 2756:         from random import randint
 2757:         if (not self.REQUEST.SESSION.get('idTmp',None)):
 2758: 
 2759:             idTmp=str(randint(0,1000000000))
 2760:             self.REQUEST.SESSION['idTmp']=idTmp
 2761:         else:
 2762:             idTmp=self.REQUEST.SESSION.get('idTmp',None)
 2763:             
 2764:     
 2765:         threadName=repeat
 2766:         if not threadName or threadName=="":
 2767:             #new thread not called from the waiting page
 2768:             tmpVar=False
 2769:        
 2770:             thread=uploadATFThread()
 2771:             threadName=thread.getName()[0:]                                
 2772:             if (not hasattr(self,'_v_uploadATF')):
 2773:                    self._v_uploadATF={}
 2774:                                        
 2775:             self._v_uploadATF[threadName]=thread
 2776:             #self._xmltrans.start()
 2777:             #thread=Thread(target=self._v_uploadATF)
 2778:             logging.info("set thread. extern")
 2779:             self._v_uploadATF[threadName].set(upload,basketId,self.REQUEST['AUTHENTICATED_USER'],idTmp,serverport=self.REQUEST['SERVER_PORT'])
 2780:             #thread.start()
 2781:             logging.info("start thread. extern")
 2782:             self._v_uploadATF[threadName].start()
 2783: 
 2784:             
 2785:             self.threadName=self._v_uploadATF[threadName].getName()[0:]
 2786:             wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
 2787: 
 2788:             if wait_template:
 2789:                 return wait_template[0][1]()
 2790:             pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self)
 2791:             return pt(txt='/uploadATF',threadName=threadName)
 2792:             #_v_xmltrans.run()
 2793:             
 2794:         else:
 2795:             #recover thread, if lost
 2796:             if (not hasattr(self,'_v_uploadATF')):
 2797:                self._v_uploadATF={}
 2798:             if not self._v_uploadATF.get(threadName,None):
 2799:                  for thread in threading.enumerate():
 2800:                          if threadName == thread.getName():
 2801:                                        self._v_uploadATF[threadName]=thread
 2802:                                        
 2803:             if self._v_uploadATF.get(threadName,None) and (not self._v_uploadATF[threadName].returnValue):
 2804:         
 2805: 
 2806:                 wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
 2807:                 if wait_template:
 2808:                         return wait_template[0][1]()
 2809:                 
 2810:                 pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self)
 2811: 
 2812:                 return pt(txt='/uploadATF',threadName=threadName)
 2813:                 
 2814:             else:
 2815:                 tmp=getattr(self.temp_folder,idTmp).returnValue
 2816:  
 2817:                 pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadCheck.zpt')).__of__(self)
 2818: 
 2819:                 return pt(changed=tmp['changed'],lockerrors=tmp['lockerrors'],errors=tmp['errors'],dir=tmp['dir'],newPs=tmp['newPs'],basketLen=tmp['basketLen'],numberOfFiles=tmp['numberOfFiles'],
 2820:                   basketNameFromId=tmp['basketNameFromId'],basketNameFromFile=tmp['basketNameFromFile'],basketId=tmp['basketId'])
 2821:                      
 2822:     def redoUpload(self,threadName):
 2823:        """redo the upload"""
 2824:        tmp=self.cdli_main.tmpStore2[threadName]
 2825:        pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadCheck.zpt')).__of__(self)
 2826:        return pt(changed=tmp['changed'],lockerrors=tmp['lockerrors'],errors=tmp['errors'],dir=tmp['dir'],newPs=tmp['newPs'],basketLen=tmp['basketLen'],numberOfFiles=tmp['numberOfFiles'],
 2827:                   basketNameFromId=tmp['basketNameFromId'],basketNameFromFile=tmp['basketNameFromFile'],basketId=tmp['basketId'])
 2828:                  
 2829:     def uploadATFfinally(self,procedure='',comment="",basketname='',unlock=None,repeat=None,RESPONSE=None):
 2830:         """nowupload the files"""
 2831:        
 2832:        
 2833:        
 2834:         threadName=repeat
 2835:         if not threadName or threadName=="":
 2836:             thread=uploadATFfinallyThread()
 2837:             threadName=thread.getName()[0:]
 2838: 
 2839:             if (not hasattr(self,'_v_uploadATF')):
 2840:                                 self._v_uploadATF={}
 2841: 
 2842: 
 2843:             self._v_uploadATF[threadName]=thread
 2844: 
 2845:             idTmp=self.REQUEST.SESSION['idTmp']
 2846:             stObj=getattr(self.temp_folder,idTmp)
 2847:             self._v_uploadATF[threadName].set(procedure,comment=comment,basketname=basketname,unlock=unlock,SESSION=stObj.returnValue,username=self.REQUEST['AUTHENTICATED_USER'],serverport=self.REQUEST['SERVER_PORT'])
 2848: 
 2849:             self._v_uploadATF[threadName].start()
 2850: 
 2851:             
 2852:             
 2853:             wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
 2854: 
 2855:             if wait_template:
 2856:                 return wait_template[0][1]()
 2857:             pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self)
 2858: 
 2859:             return pt(txt='/uploadATFfinally',threadName=threadName)
 2860:             #_v_xmltrans.run()
 2861:         
 2862:         else:
 2863:             #recover thread, if lost
 2864:             if not hasattr(self,'_v_uploadATF'):
 2865:                self._v_uploadATF={}
 2866:             if not self._v_uploadATF.get(threadName,None):
 2867:                  for thread in threading.enumerate():
 2868:                          if threadName == thread.getName():
 2869:                                        self._v_uploadATF[threadName]=thread
 2870:                                        
 2871:             if self._v_uploadATF.get(threadName,None) and (self._v_uploadATF[threadName] is not None) and (not self._v_uploadATF[threadName].end) :
 2872: 
 2873:                 wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
 2874:                 if wait_template:
 2875:                         return wait_template[0][1]()
 2876:                 
 2877:                 pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self)
 2878:                 return pt(txt='/uploadATFfinally',threadName=threadName)
 2879:             else:
 2880:               
 2881:              
 2882:               idTmp=self.REQUEST.SESSION['idTmp']
 2883:               stObj=getattr(self.temp_folder,idTmp) 
 2884:               self.REQUEST.SESSION['idTmp']=None
 2885:              
 2886:               #update changed
 2887:               logging.debug("dir:"+repr(stObj.returnValue['changed']))
 2888:               for x in stObj.returnValue['changed']:
 2889:                     ob=self.CDLICatalog.search({'title':x[0]})
 2890:                    
 2891:                     self.cdliRoot.updateOrAddToFileBTree(ob[0].getObject())
 2892:               if RESPONSE is not None:
 2893:                   RESPONSE.redirect(self.absolute_url())
 2894: 
 2895:     def importFiles(self,comment="",author="" ,folderName="/Users/dwinter/atf", files=None,ext=None):
 2896:         """import files"""
 2897:         logging.debug("importFiles folderName=%s files=%s ext=%s"%(folderName,files,ext))
 2898:         root=self.cdli_main
 2899:         count=0
 2900:         if not files:
 2901:             files=os.listdir(folderName)
 2902:             
 2903:         for f in files:
 2904:             folder=f[0:3]
 2905:             f2=f[0:5]
 2906:             
 2907:             #check if main folder PXX already exists
 2908:             obj=self.ZopeFind(root,obj_ids=[folder])
 2909:             logging.debug("importFiles: folder=%s f2=%s obj=%s"%(folder,f2,obj)) 
 2910:             if ext:
 2911:                 ext.result="<p>adding: %s </p>"%f+ext.result
 2912: 
 2913:             
 2914:             if not obj: # if not create it
 2915:                 manage_addCDLIFileFolder(root,folder,folder)
 2916:                 fobj=getattr(root,folder)
 2917:                 #transaction.get().commit()                           
 2918: 
 2919:             else:
 2920:                 fobj=obj[0][1]
 2921:             
 2922:             #check IF PYYYYY already exist
 2923:             obj2=fobj.ZopeFind(fobj,obj_ids=[f2])
 2924:             logging.debug("importFiles: fobj=%s obj2=%s"%(fobj,obj2)) 
 2925:         
 2926:             if not obj2:# if not create it
 2927:                 manage_addCDLIFileFolder(fobj,f2,f2)
 2928:                 fobj2=getattr(fobj,f2)
 2929:         
 2930:             else:
 2931:                 fobj2=obj2[0][1]
 2932:               
 2933:             # not add the file
 2934:             file2=os.path.join(folderName,f)  
 2935:             id=f
 2936:             logging.debug("importFiles: addCDLIFile fobj2=%s, f=%s file2=%s"%(fobj2,repr(f),repr(file2)))
 2937:             fobj2.addFile(vC='',file=file(file2),author=author,newName=f)
 2938:             count+=1
 2939:             
 2940:             #now add the file to the storage
 2941:             ob = getattr(fobj2,f)
 2942:             self.cdliRoot.updateOrAddToFileBTree(ob)
 2943:             
 2944:             if count%100==0:
 2945:                 logging.debug("importfiles: committing")
 2946:                 transaction.get().commit()
 2947: 
 2948:         transaction.get().commit()
 2949:         return "ok"
 2950:          
 2951: 
 2952: manage_addCDLIRootForm=DTMLFile('dtml/rootAdd', globals())
 2953: 
 2954:     
 2955: def manage_addCDLIRoot(self, id, title='',
 2956:                      createPublic=0,
 2957:                      createUserF=0,
 2958:                      REQUEST=None):
 2959:     """Add a new Folder object with id *id*.
 2960: 
 2961:     If the 'createPublic' and 'createUserF' parameters are set to any true
 2962:     value, an 'index_html' and a 'UserFolder' objects are created respectively
 2963:     in the new folder.
 2964:     """
 2965:     ob=CDLIRoot()
 2966:     ob.id=str(id)
 2967:     ob.title=title
 2968:     try:
 2969:         self._setObject(id, ob)
 2970:     except:
 2971:         pass
 2972:     ob=self._getOb(id)
 2973: 
 2974:     checkPermission=getSecurityManager().checkPermission
 2975: 
 2976:     if createUserF:
 2977:         if not checkPermission('Add User Folders', ob):
 2978:             raise Unauthorized, (
 2979:                   'You are not authorized to add User Folders.'
 2980:                   )
 2981:         ob.manage_addUserFolder()
 2982: 
 2983:   
 2984:     if REQUEST is not None:
 2985:         return self.manage_main(self, REQUEST, update_menu=1)    
 2986: