Return to cdli_files.py CVS log

Up to [Repository] / cdli

gro§e basket version 0.9

    1: """CDLI extensions of the filearchive"""    
    2: from Products.versionedFile.extVersionedFile import *
    3: from Products.ZCatalog.CatalogPathAwareness import CatalogAware
    4: from tempfile import mkstemp,mkdtemp    
    5: import os.path
    6: import os
    7: from types import *
    8: import urlparse
    9: import urllib
   10: import cgi
   11: from OFS.OrderedFolder import OrderedFolder
   12: from OFS.SimpleItem import SimpleItem
   13: import time
   14: from OFS.Folder import manage_addFolder
   15: import re
   16: from AccessControl import ClassSecurityInfo
   17: from Acquisition import Implicit
   18: from Globals import Persistent
   19: from threading import Thread
   20: from ZPublisher.HTTPRequest import HTTPRequest
   21: from ZPublisher.HTTPResponse import HTTPResponse
   22: from ZPublisher.BaseRequest import RequestContainer
   23: import threading
   24: from BTrees.OOBTree import OOBTree, OOTreeSet
   25: import logging
   26: import transaction
   27: import copy
   28: import codecs
   29: import sys
   30: from BTrees.IOBTree import IOBTree 
   31: import cdliSplitter
   32: from sets import Set
   33: import md5
   34: 
   35:                                        
   36: def makelist(mySet):
   37:         x = list(mySet)
   38:         x.sort()
   39:         return x
   40:     
   41: def unicodify(s):
   42:     """decode str (utf-8 or latin-1 representation) into unicode object"""
   43:     if not s:
   44:         return u""
   45:     if isinstance(s, str):
   46:         try:
   47:             return s.decode('utf-8')
   48:         except:
   49:             return s.decode('latin-1')
   50:     else:
   51:         return s
   52: 
   53: def utf8ify(s):
   54:     """encode unicode object or string into byte string in utf-8 representation.
   55:        assumes string objects to be utf-8"""
   56:     if not s:
   57:         return ""
   58:     if isinstance(s, str):
   59:         return s
   60:     else:
   61:         return s.encode('utf-8')
   62: 
   63: def formatAtfHtml(l):
   64:     """escape special ATF characters for HTML"""
   65:     if not l:
   66:         return ""
   67: 
   68:     # replace &
   69:     l = l.replace('&','&')
   70:     # replace angular brackets
   71:     l = l.replace('<','&lt;')
   72:     l = l.replace('>','&gt;')
   73:     return l
   74: 
   75: def formatAtfLineHtml(l, nolemma=True):
   76:     """format ATF line for HTML"""
   77:     if not l:
   78:         return ""
   79: 
   80:     if nolemma:
   81:         # ignore lemma lines
   82:         if l.lstrip().startswith('#lem:'):
   83:             return ""
   84:     
   85:     return formatAtfHtml(l)
   86: 
   87: 
   88: 
   89: def formatAtfFullLineNum(txt, nolemma=True):
   90:     """format full line numbers in ATF text"""
   91:     # surface codes
   92:     surfaces = {'@obverse':'obv',
   93:                 '@reverse':'rev',
   94:                 '@surface':'surface',
   95:                 '@edge':'edge',
   96:                 '@left':'left',
   97:                 '@right':'right',
   98:                 '@top':'top',
   99:                 '@bottom':'bottom',
  100:                 '@face':'face',
  101:                 '@seal':'seal'}
  102: 
  103:     if not txt:
  104:         return ""
  105:     
  106:     ret = []
  107:     surf = ""
  108:     col = ""
  109:     for line in txt.splitlines():
  110:         line = unicodify(line)
  111:         if line and line[0] == '@':
  112:             # surface or column
  113:             words = line.split(' ')
  114:             if words[0] in surfaces:
  115:                 surf = line.replace(words[0],surfaces[words[0]]).strip()
  116:             
  117:             elif words[0] == '@column':
  118:                 col = ' '.join(words[1:])
  119:             
  120:         elif line and line[0] in '123456789':
  121:             # ordinary line -> add line number
  122:             line = "%s:%s:%s"%(surf,col,line)
  123:             
  124:         ret.append(line)
  125:     
  126:     return '\n'.join(ret)
  127:             
  128:             
  129: def generateXMLReturn(hash):
  130:     """erzeugt das xml file als returnwert fuer uploadATFRPC"""
  131: 
  132:     ret="<return>"
  133:     
  134:     ret+="<errors>"
  135:     for error in hash['errors']:
  136:         ret+="""<error atf="%s">%s</error>"""%error
  137:     
  138:     ret+="</errors>"
  139:     
  140:     ret+="<changes>"
  141:     for changed in hash['changed']:
  142:         ret+="""<change atf="%s">%s</change>"""%changed
  143:     ret+="</changes>"
  144:     
  145:     ret+="<newPs>"
  146:     for new in hash['newPs']:
  147:         ret+="""<new atf="%s"/>"""%new
  148:     ret+="</newPs>"
  149:     
  150:     ret+="</return>"
  151:     return ret
  152:     
  153:     
  154: def unique(s):
  155:     """Return a list of the elements in s, but without duplicates.
  156: 
  157:     For example, unique([1,2,3,1,2,3]) is some permutation of [1,2,3],
  158:     unique("abcabc") some permutation of ["a", "b", "c"], and
  159:     unique(([1, 2], [2, 3], [1, 2])) some permutation of
  160:     [[2, 3], [1, 2]].
  161: 
  162:     For best speed, all sequence elements should be hashable.  Then
  163:     unique() will usually work in linear time.
  164: 
  165:     If not possible, the sequence elements should enjoy a total
  166:     ordering, and if list(s).sort() doesn't raise TypeError it's
  167:     assumed that they do enjoy a total ordering.  Then unique() will
  168:     usually work in O(N*log2(N)) time.
  169: 
  170:     If that's not possible either, the sequence elements must support
  171:     equality-testing.  Then unique() will usually work in quadratic
  172:     time.
  173:     (from the python cookbook)
  174:     """
  175: 
  176:     n = len(s)
  177:     if n == 0:
  178:         return []
  179: 
  180:     # Try using a dict first, as that's the fastest and will usually
  181:     # work.  If it doesn't work, it will usually fail quickly, so it
  182:     # usually doesn't cost much to *try* it.  It requires that all the
  183:     # sequence elements be hashable, and support equality comparison.
  184:     u = {}
  185:     try:
  186:         for x in s:
  187:             u[x] = 1
  188:     except TypeError:
  189:         del u  # move on to the next method
  190:     else:
  191:         return u.keys()
  192: 
  193:     # We can't hash all the elements.  Second fastest is to sort,
  194:     # which brings the equal elements together; then duplicates are
  195:     # easy to weed out in a single pass.
  196:     # NOTE:  Python's list.sort() was designed to be efficient in the
  197:     # presence of many duplicate elements.  This isn't true of all
  198:     # sort functions in all languages or libraries, so this approach
  199:     # is more effective in Python than it may be elsewhere.
  200:     try:
  201:         t = list(s)
  202:         t.sort()
  203:     except TypeError:
  204:         del t  # move on to the next method
  205:     else:
  206:         assert n > 0
  207:         last = t[0]
  208:         lasti = i = 1
  209:         while i < n:
  210:             if t[i] != last:
  211:                 t[lasti] = last = t[i]
  212:                 lasti += 1
  213:             i += 1
  214:         return t[:lasti]
  215: 
  216:     # Brute force is all that's left.
  217:     u = []
  218:     for x in s:
  219:         if x not in u:
  220:             u.append(x)
  221:     return u
  222: 
  223: 
  224: class BasketContent(SimpleItem):
  225:     """classe fuer den Inhalt eines Baskets"""
  226:    
  227:     def __init__(self,content=[]):
  228:         """content"""
  229:         self.contentList=content[0:]
  230:     
  231:     def getContent(self):
  232:         """get content"""
  233: 
  234:         return self.contentList
  235:     
  236:     def setContent(self,content):
  237:         self.contentList=content[0:]
  238:     
  239:     def numberOfItems(self):
  240:         """number"""
  241:         
  242:         return len(self.getContent())
  243:         
  244:     
  245: class uploadATFfinallyThread(Thread):
  246:     """class for adding uploaded filed (temporarily stored in the staging area at /tmp"""
  247:     
  248:     def __init__(self):
  249:         """init for uploadATFfinallyThread"""
  250:         self.continueVar=True
  251:         self.returnValue=None
  252:         self.end=False
  253:         Thread.__init__(self)
  254:            
  255:     def set(self,procedure,comment="",basketname='',unlock=None,SESSION=None,username=None,serverport="8080"):
  256:         """set start values for the thread"""
  257:         self.procedure=procedure
  258:         self.comment=comment
  259:         self.basketname=basketname
  260:         self.unlock=unlock
  261:         self.SESSION=SESSION
  262:         self.username=username
  263:         self.serverport=serverport
  264:        
  265:         
  266:     def __call__(self):
  267:         """call of the thread (equals run)"""
  268:         self.run()
  269:         return True
  270:     
  271:     def getContext(self, app,serverport="8080"):
  272:         """get the context within the ZODB"""
  273:         
  274:         resp = HTTPResponse(stdout=None)
  275:         env = {
  276:             'SERVER_NAME':'localhost',
  277:             'SERVER_PORT':serverport,
  278:             'REQUEST_METHOD':'GET'
  279:             }
  280:         req = HTTPRequest(None, env, resp)
  281:         return app.__of__(RequestContainer(REQUEST = req))
  282:           
  283:         
  284:     def run(self):
  285:         """run"""
  286:         
  287:         self.result=""
  288:         #find context within ZODB
  289:         from Zope import DB
  290:         conn = DB.open()
  291:         root = conn.root()
  292:         app  = root['Application']
  293:         ctx = self.getContext(app,serverport=self.serverport)
  294: 
  295:         #add the files
  296:         self.uploadATFfinallyThread(ctx,self.procedure,comment=self.comment,basketname=self.basketname,unlock=self.unlock,SESSION=self.SESSION,username=self.username)
  297:         #commit the transactions
  298:         transaction.get().commit()
  299:         conn.close()
  300:         #set flag for end of this method
  301:         self.end=True
  302:         logging.info("ended")
  303:         return True
  304:     
  305:     def __del__(self):
  306:         """delete"""
  307:         
  308:         
  309:     
  310:     def getResult(self):
  311:         """method for accessing result"""
  312:         
  313:         return self.result
  314:      
  315:     def uploadATFfinallyThread(self,ctx,procedure,comment="",basketname='',unlock=None,RESPONSE=None,SESSION=None,username=None):
  316:         """upload the files"""
  317:         #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot
  318:         ctx2=ctx.cdliRoot
  319:    
  320:         self.result+="<h2>Start processing</h2>"
  321:         
  322:         #shall I only upload the changed files?
  323:         logging.debug("uploadATFfinally procedure: %s"%procedure)
  324:         if procedure=="uploadchanged":
  325:             changed=[x[0] for x in SESSION.get('changed',[])]
  326:             uploadFns=changed+SESSION.get('newPs',[])
  327:         
  328:         #or all
  329:         elif procedure=="uploadAll":
  330:             uploadFns=[]
  331:             for x in os.listdir(SESSION['tmpdir']):
  332:                 if not x in SESSION['lockerrors']:
  333:                     uploadFns.append(x)
  334:                     
  335:         #or maybe nothing
  336:         elif procedure=="noupload":
  337:             return True
  338:         else:
  339:             uploadFns=[]
  340:             
  341:         #do first the changed files    
  342:         i=0
  343:         for fn in uploadFns:
  344:             logging.debug("uploadATFfinally uploadFn=%s"%fn)
  345:             i+=1
  346:             founds=ctx2.CDLICatalog.search({'title':fn})
  347:             if len(founds)>0:
  348:                 SESSION['author']=str(username)
  349:                 self.result="<p>Changing : %s"%fn+self.result
  350:                 logging.debug("uploadatffinallythread changing:%s"%fn+self.result)
  351:                 founds[0].getObject().manage_addCDLIFileObject('',comment,SESSION['author'],file=os.path.join(SESSION['tmpdir'],fn),from_tmp=True)
  352:             if i%200==0:
  353:                 transaction.get().commit()
  354:                 logging.debug("uploadatffinallythread changing: do commit")
  355:         
  356:         transaction.get().commit()
  357:         logging.debug("uploadatffinallythread changing: last commit")
  358: 
  359:         #now add the new files        
  360:         newPs=SESSION['newPs']
  361:         if len(newPs)>0:
  362:             tmpDir=SESSION['tmpdir']
  363:             logging.debug("uploadatffinallythread adding start")
  364:             self.result="<p>Adding files</p>"+self.result
  365:             #TODO: make this configurable, at the moment base folder for the files has to be cdli_main
  366:             ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self)
  367:             logging.debug("uploadatffinallythread adding finished")
  368:         
  369:         #unlock locked files?
  370:         if unlock:
  371:             logging.debug("uploadatffinallythread unlocking start")
  372:             self.result="<p>Unlock files</p>"+self.result
  373:             unlockFns=[]
  374:             for x in os.listdir(SESSION['tmpdir']):
  375:                     if not x in SESSION['errors']:
  376:                         unlockFns.append(x)
  377:                         
  378:             logging.debug("unlocking have now what to unlock")
  379:                         
  380:             for fn in unlockFns:
  381:                 #logging.info("will unlock: %s"%fn)
  382:                 founds=ctx2.CDLICatalog.search({'title':fn})
  383:                 #logging.info("found it: %s"%repr(founds))
  384:                 if len(founds)>0:
  385:                     #logging.info("unlock: %s"%founds[0].getObject().getId())
  386:                     SESSION['author']=str(username)
  387:                     founds[0].getObject().lockedBy=""
  388: 
  389:             logging.debug("uploadatffinallythread unlocking done")
  390:                     
  391:         #if a basketname is given, add files to the basket
  392:         if not (basketname ==''):
  393:             logging.debug("uploadatffinallythread add to basket %s"%basketname)
  394:             self.result="<p>Add to basket</p>"+self.result
  395:             basketId=ctx2.basketContainer.getBasketIdfromName(basketname)
  396:             
  397:             if not basketId: # create new basket
  398:                 logging.debug("uploadatffinallythread create basket %s"%basketname)
  399:                 self.result="<p>Create a new basket</p>"+self.result
  400:                 ob=ctx2.basketContainer.addBasket(basketname)
  401:                 basketId=ob.getId()
  402:             basket=getattr(ctx2.basketContainer,str(basketId))
  403:             ids=os.listdir(SESSION['tmpdir'])
  404:             logging.debug("should add:"+repr(ids))
  405:             basket.addObjects(ids,deleteOld=True,username=str(username))    
  406:                
  407:         logging.debug("uploadatffinallythread uploadfinally done")
  408: 
  409:         if RESPONSE is not None:
  410:             RESPONSE.redirect(self.aq_parent.absolute_url())
  411:         
  412:         return True
  413: 
  414: class tmpStore(SimpleItem):
  415:     """simple item"""
  416:     meta_type="cdli_upload"
  417:     
  418:     def __init__(self,id):
  419:         """init tmp"""
  420:         self.id=id
  421:         
  422: class uploadATFThread(Thread):
  423:     """class for checking the files befor uploading"""
  424:     
  425:     def __init__(self):
  426:         """initialise"""
  427:         
  428:         self.continueVar=True
  429:         self.returnValue=None
  430:         
  431:         Thread.__init__(self)
  432:         
  433:         
  434:     def set(self,upload,basketId,username,idTmp,serverport="8080"):
  435:         """set start values for the thread"""
  436:         self.result=""
  437:         self.upload=upload
  438:         self.basketId=basketId
  439:         self.username=username
  440:         self.serverport=serverport
  441:         self.idTmp=idTmp
  442:         
  443:     def __call__(self):
  444:         """call method """
  445:         self.run()
  446:         return True
  447:     
  448:     def getContext(self, app,serverport="8080"):
  449:         """get the context within the ZODB"""
  450:         resp = HTTPResponse(stdout=None)
  451:         env = {
  452:             'SERVER_NAME':'localhost',
  453:             'SERVER_PORT':serverport,
  454:             'REQUEST_METHOD':'GET'
  455:             }
  456:         req = HTTPRequest(None, env, resp)
  457:         return app.__of__(RequestContainer(REQUEST = req))
  458:         
  459:     def run(self):
  460:         idTmp=self.idTmp
  461:         self.result=""
  462:         #find context within ZODB
  463:         from Zope import DB
  464:         conn = DB.open()
  465:         root = conn.root()
  466:         app  = root['Application']
  467:         ctx = self.getContext(app,serverport=self.serverport)
  468:         logging.info("run intern")
  469:         try:
  470:             logging.info("created: %s"%idTmp)
  471:             ctx.temp_folder._setObject(idTmp,tmpStore(idTmp))
  472:         except:
  473:             logging.error("thread upload: %s %s"%sys.exc_info()[0:2])
  474:             
  475:         logging.info("call thread intern")
  476:         self.uploadATFThread(ctx,self.upload,idTmp,self.basketId)
  477:      
  478:         #ctx.cdliRoot.cdli_main.tmpStore2[self.getName()[0:]]=self.returnValue
  479:         
  480:         
  481:         transaction.get().commit()
  482:        
  483:         conn.close()
  484:         
  485:         return getattr(ctx.temp_folder,idTmp)
  486:         
  487:     def getResult(self):
  488:         """method for accessing result"""
  489:         return self.result
  490:     
  491:     def uploadATFThread(self,ctx,upload,idTmp,basketId=0):
  492:         """upload an atf file"""
  493:         #TODO: add comments
  494:         #TODO: finish uploadATF
  495:         
  496:         stObj=getattr(ctx.temp_folder,idTmp)
  497:         logging.info("start, upload thread")
  498:         self.result="<html><body><h2>I got your file, start now to split it into single atf-files!</h2><p>"
  499:     
  500:         #make sure that id is a string and not an integer
  501:         basketId=str(basketId)
  502:         
  503:         #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot
  504:         ctx2=ctx.cdliRoot
  505:         
  506:         #get temporary file for staging the downloaded and splitted files
  507:         dir=mkdtemp()
  508:         
  509:         
  510:         changed=[] # changed files
  511:         errors=[]  # files with errors
  512:         lockerrors=[]  # files with errors
  513: 
  514:         newPs=[]   # new p filed
  515:         psNotInCatalog=[] # files not in the catalog
  516:         
  517:         #split the uploadedd atf file
  518:         basketNameFromFile, numberOfFiles=splitatf(upload,dir,ext=self)
  519:         
  520:         #find basketId if not set
  521:         
  522:         #get active abaket
  523:         if basketId == '0':
  524:             basketObj=ctx2.basketContainer.getActiveBasket()
  525:             if basketObj:
  526:                 basketId=basketObj.getId()
  527:                 
  528:         #if there is no active basket and no basketid given, id is empty, else get besketname and length
  529:         if basketId == '0':
  530:             basketNameFromId=""
  531:             basketLen=0
  532:         else:
  533:             basketNameFromId=getattr(ctx2.basketContainer,basketId).title
  534:             basketLen=getattr(ctx2.basketContainer,basketId).getLastVersion().numberOfItems()
  535:             
  536:         logging.info("got the file, upload thread")
  537:         self.result+="""<html><body><h2>I got the files</h2><
  538:                         p>I am computing the differences to the exisiting files</p>"""
  539:                                    
  540:         #start to check the files
  541:         for fn in os.listdir(dir):
  542:             
  543:             self.result="<p>process:%s</p>"%fn+self.result
  544:             
  545:             # check if file is in the catalog
  546:             #TODO: checkCatalog is not implemented yet
  547:             if ctx2.cdli_main.checkCatalog(fn):
  548:                 psNotInCatalog.append(fn)
  549:                 
  550:             #check if p-file already at the server  
  551:             founds=ctx2.CDLICatalog.search({'title':fn})    
  552:       
  553:             #if not than add filename to the list of newfiles
  554:             
  555:             data=file(os.path.join(dir,fn)).read()
  556:             status,msg=checkFile(fn,data,dir)
  557:             #status=True
  558:             
  559:             
  560:             if not status: # error
  561:                 errors.append((fn,msg))
  562:             
  563:             else:
  564:                 if len(founds)==0:
  565:                     newPs.append(fn)
  566: 
  567:                 #if p file alread at the server    
  568:                 for found in founds:
  569:                     #analyse the differences to the actual file
  570:                     obj=found.getObject()
  571: 
  572:                     if (not (str(obj.lockedBy))=='') and (not (str(obj.lockedBy)==str(self.username))):
  573:                                 lockerrors.append((fn,str(obj.lockedBy)))
  574:                     else:
  575:                 
  576:                         diffs=obj.diff(data)
  577:                         if diffs[0]>0:
  578:                             changed.append((obj,diffs)) #hochladen
  579: 
  580:         #ready, set the returnValues
  581:         self.result+="<h3>Done</h3></body></html>"
  582:         
  583:         stObj.returnValue={}
  584:         
  585:         stObj.returnValue['errors']=errors
  586:         
  587:         stObj.returnValue['newPs']=newPs
  588:         stObj.returnValue['tmpdir']=dir
  589:         stObj.returnValue['basketLen']=basketLen
  590:         stObj.returnValue['numberOfFiles']=numberOfFiles
  591:         stObj.returnValue['basketNameFromId']=basketNameFromId
  592:         stObj.returnValue['basketNameFromFile']=basketNameFromFile
  593:         stObj.returnValue['basketId']=basketId
  594:         stObj.returnValue['dir']=dir
  595:         #stObj.returnValue['changed']=copy.copy(changed)
  596:         stObj.returnValue['changed']=[(x[0].getId(),x[1][0]) for x in changed]
  597:         #stObj.returnValue['lockerrors']=[x[0].getId() for x in lockerrors]
  598:         stObj.returnValue['lockerrors']=[x for x in lockerrors]
  599:         self.returnValue=True
  600:         #ctx2.cdli_main.setTemp('v_uploadATF_returnValue',True)
  601:     
  602:  
  603: class CDLIBasketContainer(OrderedFolder):
  604:     """contains the baskets"""
  605:     
  606: 
  607:     security=ClassSecurityInfo()
  608:     meta_type="CDLIBasketContainer"
  609:     
  610:     def getPNumbersOfBasket(self,basketName):
  611:         """get all pnumbers of a basket as a list, returns an empty list if basket not found
  612:         @param basketName: name of the basket
  613:         """
  614:         ret=[]
  615:         basketId=self.getBasketIdfromName(basketName)
  616:         if not basketId:
  617:             return []
  618:         
  619:         ob=getattr(self,basketId).getContent()
  620:         
  621:         ret=[x[0].split(".")[0] for x in ob]
  622:         
  623:         return ret
  624:     
  625:     security.declareProtected('manage','getBasketAsOneFile')       
  626:     def getBasketAsOneFile(self,basketName,current="no"):
  627:         """returns all files of the basket combined in one file
  628:         @param basketName: Name of the basket
  629:         @param current: (optional) if current is set to "yes" then the most current version of 
  630:                         all files are downloaded and not the versions of the files as stored in the basket
  631:         """
  632:         ret=""
  633:         basketId=self.getBasketIdfromName(basketName)
  634:         if not basketId:
  635:             return ""
  636:         
  637:         ob=getattr(self,basketId).getLastVersion()
  638:         for object in ob.getContent():
  639:             if current=="no": #version as they are in the basket
  640:                             ret+=str(object[0].getData())+"\n"
  641:             elif current=="yes":
  642:                             #search current object
  643:                             logging.debug("current: %s"%object[1].getId().split(".")[0])
  644:                             founds=self.CDLICatalog.search({'title':object[1].getId().split(".")[0]})
  645:                             if len(founds)>0:      
  646:                                 ret+=str(founds[0].getObject().getLastVersion().getData())+"\n"
  647:         return ret
  648:     
  649:     security.declareProtected('manage','upDateBaskets') 
  650:     def upDateBaskets(self):
  651:         """update content in to objects"""
  652:         
  653:         founds=self.ZopeFind(self,obj_metatypes=['CDLIBasketVersion'],search_sub=1)
  654: 
  655:         for found in founds:
  656:             found[1].updateBasket()
  657:         
  658:     security.declareProtected('manage','deleteBaskets')        
  659:     def deleteBaskets(self,ids=None):
  660:         """delete baskets, i.e. move them into trash folder"""
  661:         
  662:         
  663:         found=self.ZopeFind(self,obj_ids=['trash'])
  664:         
  665:         if len(found)<1:
  666:             manage_addFolder(self, 'trash')
  667:             trash=self._getOb('trash')
  668:         else:
  669:             trash=found[0][1]
  670:         
  671:         if type(ids) is not ListType:
  672:             ids=[ids]
  673:         cut=self.manage_cutObjects(ids)
  674:         trash.manage_pasteObjects(cut)
  675:         
  676:     security.declareProtected('manage','manageBaskets')       
  677:     def manageBaskets(self,submit,ids=None,basket1="",basket2="",joinBasket="",subtractBasket="",REQUEST=None,RESPONSE=None):
  678:         """manage baskets, delete or copy"""
  679:         if submit=="delete":
  680:             self.deleteBaskets(ids)
  681:         
  682:         elif submit=="join":
  683:             flag,msg=self.joinBasket(joinBasket, ids)
  684:             logging.info("joining %s %s"%(flag,msg))
  685:             
  686:         elif submit=="subtract":
  687:             logging.info("BBBb %s %s"%(basket1,basket2))
  688:             flag,msg=self.subtractBasket(subtractBasket, basket1,basket2)
  689:             logging.info("subtract %s %s"%(flag,msg))
  690:             
  691:         if RESPONSE:
  692:             RESPONSE.redirect(self.absolute_url())
  693:     
  694:     security.declareProtected('View','getBasketIdfromName')       
  695:     def getBasketIdfromName(self,basketname):
  696:         """get id from name"""
  697: 
  698:         for basket in self.ZopeFind(self,obj_metatypes=["CDLIBasket"]):
  699:             if basket[1].title==basketname:
  700:                 return basket[0]
  701:         else:
  702:             None
  703:     
  704:     security.declareProtected('manage','uploadBasket_html')        
  705:             
  706:     def uploadBasket_html(self,basketId='0'):
  707:         """upload an atf file, html form"""
  708:         
  709: 
  710:         basketId=str(basketId)
  711:         if not basketId=='0':
  712:             basketName=getattr(self.basketContainer,basketId).title
  713:         else:
  714:             basketName=""
  715:             
  716:         pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadBasket_html.zpt')).__of__(self)
  717:         return pt(basketId=basketId,basketName=basketName)
  718:    
  719: 
  720:     security.declareProtected('manage','index_html')    
  721:     def index_html(self):
  722:         """stanadard ansicht"""
  723:         
  724: 
  725: 
  726:         ext=self.ZopeFind(self,obj_ids=["index.html"])
  727:         if ext:
  728:             return ext[0][1]()
  729:         
  730:         pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketContainerMain')).__of__(self)
  731:         return pt()
  732:     
  733:     def getStorageFolderRoot(self):
  734:         """root des storage folders"""
  735:         return self.cdli_main
  736:     
  737:     def __init__(self,id,title):
  738:         """ init basket container"""
  739:         self.id=id
  740:         self.title=title
  741:      
  742:  
  743:     def getBasketsId(self):
  744:         """get all baskets als klartext"""
  745:         
  746:         ret=""
  747:         baskets=self.ZopeFind(self,obj_metatypes=['CDLIBasket'])
  748:         for basket in baskets:
  749:             com,user,time,values = basket[1].getContentIds()
  750:             ret+= "BASKET:"+com+"\t"+user+"\t"+time+"\n"
  751:             for x in values:
  752:                 ret+= x[0]+"\t"+x[1]+"\n"
  753:                 return ret
  754: 
  755:     def getBaskets(self,sortField='title'):
  756:         """get all baskets files"""
  757: 
  758:         def sortName(x,y):
  759:             return cmp(x[1].title.lower(),y[1].title.lower())
  760: 
  761:         def sortDate(x,y):
  762:             return cmp(y[1].getLastVersion().getTime(),x[1].getLastVersion().getTime())
  763: 
  764:         
  765:         def sortComment(x,y):
  766: 
  767:         
  768:             
  769:              try:
  770:                 xc=getattr(x[1],'comment','ZZZZZZZZZZZZZ').lower()
  771:              except:
  772:                 xc='ZZZZZZZZZZZZZ'.lower()
  773:              try:
  774:                 yc=getattr(y[1],'comment','ZZZZZZZZZZZZZ').lower()
  775:              except:
  776:                 yc='ZZZZZZZZZZZZZ'.lower()
  777:     
  778:     
  779:              if (xc=='') or (xc=='ZZZZZZZZZZZZZ'.lower()):
  780:                  
  781:                  try:
  782:                      xc=x[1].getLastVersion().getComment().lower()
  783:                  except:
  784:                      xc='ZZZZZZZZZZZZZ'.lower()
  785:                      
  786:              if (yc=='') or (yc=='ZZZZZZZZZZZZZ'.lower()):
  787:                  try:
  788:                      yc=y[1].getLastVersion().getComment().lower()
  789:                  except:
  790:                      yc='ZZZZZZZZZZZZZ'.lower()
  791:     
  792:              
  793:                  return cmp(xc,yc)
  794:         
  795:         def sortAuthor(x,y):
  796:             
  797:             return cmp(x[1].getLastVersion().getUser().lower(),y[1].getLastVersion().getUser().lower())
  798:         
  799:         baskets=self.ZopeFind(self,obj_metatypes=['CDLIBasket'])
  800:         
  801:         
  802:         if sortField=='title':
  803:             baskets.sort(sortName)
  804:         elif sortField=='date':
  805:             baskets.sort(sortDate)
  806:         elif sortField=='author':
  807:             baskets.sort(sortAuthor)
  808:         elif sortField=='comment':
  809:             baskets.sort(sortComment)
  810: 
  811:         return baskets
  812:     
  813:         
  814:     def subtractBasket(self,newBasket,basket1,basket2):
  815:         """subtract basket2 from basket1 
  816:         (i.e. newbasket will contain alle elements of basket1 which are not in basket2), 
  817:         if basket2 contains files which are not in basket1, then theses files fill be ignored
  818:                
  819:         @param newbasket: name of the new basket
  820:         @param basket1: basket where basket2 will be subtracted from
  821:         @param basket2: see above
  822:       
  823:         """
  824:         logging.info("CCCCC %s %s"%(basket1,basket2))
  825:    
  826:         try:
  827:             newB=self.addBasket(newBasket)
  828:         except:
  829:             return False, "cannot create the new basket"
  830:         
  831:         
  832: 
  833:        
  834:      
  835:         bas2= getattr(self,basket2)            
  836:         bas2content=bas2.getContent()
  837:         bas2ids=[x[0] for x in bas2content]
  838:         
  839:        
  840:             
  841:         bas1= getattr(self,basket1)   
  842:         bas1content=bas1.getContent()
  843:         
  844:         
  845:         newBasketContent={}
  846:         
  847:         for id,version in bas1content:
  848:             if not (id in bas2ids):
  849:                 newBasketContent[id]=version
  850:         
  851:         username=self.getActualUserName()
  852:         
  853:         logging.info("sbc %s"%newBasketContent)
  854:         newB.addObjectsWithVersion(newBasketContent,username=username,catalog=self.CDLICatalog)
  855:         
  856:         return True, ""
  857:     
  858:             
  859:     def joinBasket(self,newBasket,oldBaskets):
  860:         """join two baskets
  861:         @param newbasket: name of the new basket
  862:         @param oldbaskets: list of baskets to be joined
  863:         """
  864:         try:
  865:             newB=self.addBasket(newBasket)
  866:         except:
  867:             return False, "cannot create the new basket"
  868:         
  869:         newBasketContent={}
  870:         for ob in oldBaskets:
  871:             x= getattr(self,ob,None)
  872:             if x is None:
  873:                 return False, "cannot find basket: %s"%ob
  874:             
  875:             ids=x.getContent() # hole den Inhalt
  876:             
  877:             for id,version in ids:
  878:                 if newBasketContent.has_key(id): # p number gibt's schon
  879:                     newBasketContent[id]=max(newBasketContent[id],version) # speichere die groessere Versionsnumber
  880:                 else:
  881:                     newBasketContent[id]=version
  882:         username=self.getActualUserName()
  883:         
  884:         logging.info("nbc %s"%newBasketContent)
  885:         newB.addObjectsWithVersion(newBasketContent,username=username,catalog=self.CDLICatalog)
  886:         
  887:         return True, ""
  888:     
  889:     def getNewId(self):
  890:         """createIds"""
  891:         last=getattr(self,'last',0)
  892:         last +=1
  893:         while len(self.ZopeFind(self,obj_ids=[str(last)]))>0:
  894:             last+=1
  895:     
  896:         self.last=last
  897:         return last
  898:     
  899:     def setActiveBasket(self,basketId,REQUEST=None):
  900:         """store active basketId in a cookie"""
  901:         self.REQUEST.RESPONSE.setCookie("CDLIActiveBasket",basketId,path="/")
  902:         try:
  903:             qs=cgi.parse_qs(REQUEST['QUERY_STRING'])
  904:             del(qs['basketId'])
  905:         except:
  906:             qs={}
  907:         if REQUEST:
  908:             REQUEST.RESPONSE.redirect(REQUEST['URL1']+'?'+urllib.urlencode(qs))
  909:             
  910:     def getActiveBasket(self):
  911:         """get active basket from cookie"""
  912:         
  913:         id= self.REQUEST.cookies.get('CDLIActiveBasket',None)
  914:         if id:
  915:             obj=getattr(self,str(id),None)
  916:         else:
  917:             obj=None
  918:         return obj
  919:     
  920:     def getActualUserName(self):
  921:         """get name of the actualuser"""
  922:         return str(self.REQUEST['AUTHENTICATED_USER'])
  923:     
  924:     security.declareProtected('manage','addBasket') 
  925:     def addBasket(self,newBasketName):
  926:         """add a new basket"""
  927:         
  928:         ob=manage_addCDLIBasket(self,newBasketName)
  929:         return ob
  930: 
  931:     def storeInBasket(self,submit,ids=None,newBasketName=None,fromFileList=None,RESPONSE=None,REQUEST=None):
  932:         """store it"""
  933:         if not ids:
  934:             ids=self.REQUEST.SESSION['fileIds']
  935:            
  936:         if (type(ids) is not ListType) and (not isinstance(ids,Set)):
  937:             ids=[ids]
  938:         
  939:         if isinstance(ids,Set):
  940:             ids=list(ids)
  941:             
  942:         if (submit.lower()=="store in new basket") or (submit.lower()=="new basket"):
  943:             basketRet=self.addBasket(newBasketName)
  944:             self.setActiveBasket(basketRet.getId())
  945:             basket=getattr(self,basketRet.getId())
  946:         elif (submit.lower()=="store in active basket") or (submit.lower()=="active basket"):
  947:             basket=self.getActiveBasket()
  948:         
  949:         added=basket.addObjects(ids)
  950:         back=self.REQUEST['HTTP_REFERER'].split("?")[0]+"?basketName="+basket.title+"&numberOfObjects="+str(added)
  951:         
  952:         
  953:         if fromFileList:
  954: 
  955:             return self.cdli_main.findObjectsFromList(list=ids,basketName=basket.title,numberOfObjects=added)
  956:        
  957:         if RESPONSE:
  958:             
  959:             RESPONSE.redirect(back)
  960:             
  961:         return True
  962:     
  963: def manage_addCDLIBasketContainerForm(self):
  964:     """add the CDLIBasketContainer form"""
  965:     pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIBasketContainer.zpt')).__of__(self)
  966:     return pt()
  967: 
  968: def manage_addCDLIBasketContainer(self,id,title,RESPONSE=None):
  969:     """add the basket"""
  970:     ob=CDLIBasketContainer(id,title)
  971:     
  972:     self._setObject(id, ob)
  973:     
  974:     if RESPONSE is not None:
  975:         RESPONSE.redirect('manage_main')
  976: 
  977: class CDLIBasket(Folder,CatalogAware):
  978:     """basket"""
  979:     
  980:     meta_type="CDLIBasket"
  981:     default_catalog="CDLIBasketCatalog"
  982:     
  983:     def searchInBasket(self,indexName,searchStr,regExp=False):
  984:         """searchInBasket"""
  985: 
  986:         lst=self.searchInLineIndexDocs(indexName,searchStr,uniq=True,regExp=regExp) #TODO: fix this
  987:         ret={}
  988:         
  989:         lv=self.getLastVersion()
  990: 
  991: 
  992:         for obj in lv.content.getContent():
  993:             id=obj[1].getId().split(".")[0]
  994:             if id in lst:
  995:         
  996:                 ret[id]=self.showWordInFile(id,searchStr,lineList=self.getLinesFromIndex(indexName,searchStr,id,regExp=regExp),regExp=regExp,indexName=indexName)
  997:         
  998:         
  999:         pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','searchResultsInBasket')).__of__(self)
 1000:         return pt(result=ret,indexName=indexName,regExp=regExp,word=searchStr)
 1001:         
 1002:          
 1003:  
 1004:         
 1005:     def searchInBasket_v1(self,searchStr):
 1006:         """search occurences of searchStr in files im basket"""
 1007:         ret=[]
 1008:         lv=self.getLastVersion()
 1009:         logging.info("searching")
 1010:         for obj in lv.content.getContent():
 1011:             txt=obj[0].getData()
 1012:             for x in txt.split("\n"):
 1013:                 logging.info("search %s"%x)
 1014:                 if re.match(searchStr,x):
 1015:                     ret.append(x)
 1016:         
 1017:         return "\n".join(ret)
 1018:                 
 1019:             
 1020:     def getFile(self,obj):
 1021:         return obj[1]
 1022:     
 1023:     def getFileLastVersion(self,obj):
 1024:         return obj[0]
 1025:     
 1026:     def getFileNamesInLastVersion(self):
 1027:         """get content of the last version as list"""
 1028:         
 1029:         return [x[1].getId() for x in self.getLastVersion().getContent()]
 1030:     
 1031: 
 1032:     def isActual(self,obj):
 1033:         """teste ob im basket die aktuelle version ist"""
 1034:         try:
 1035:             logging.debug("isActual:"+repr(obj))
 1036:             actualNo=obj[1].getLastVersion().getVersionNumber()
 1037:             storedNo=obj[0].getVersionNumber()
 1038:             
 1039:             founds=self.CDLICatalog.search({'title':obj[0].getId()})
 1040:             if len(founds)>0:
 1041:                 actualNo=founds[0].getObject().getLastVersion().getVersionNumber()
 1042:                 
 1043:             if len(founds)>0 and founds[0].getObject().aq_parent.getId()==".trash":
 1044:                 return False, -1
 1045:             
 1046:             if actualNo==storedNo:
 1047:                 return True , 0
 1048:             else:
 1049:                 return False, actualNo
 1050:         except:
 1051:             return False, -1
 1052:             
 1053:     def history(self):
 1054:         """history"""  
 1055: 
 1056:         ext=self.ZopeFind(self.aq_parent,obj_ids=["history_template.html"])
 1057:         if ext:
 1058:             return getattr(self,ext[0][1].getId())()
 1059:         
 1060:         pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketHistory')).__of__(self)
 1061:         return pt()
 1062:     
 1063:     def getStorageFolderRoot(self):
 1064:         """root des storage folders"""
 1065:         return self.aq_parent.cdli_main
 1066:     
 1067:     def __init__(self,id,title,shortDescription="",comment=""):
 1068:         """init a CDLIBasket"""
 1069:         
 1070:         self.id=id
 1071:         self.title=title
 1072:         self.shortDescription=shortDescription
 1073:         self.comment=comment
 1074:  
 1075:     def getActualUserName(self):
 1076:         """get name of the actualuser"""
 1077:        
 1078:         return str(self.REQUEST['AUTHENTICATED_USER'])
 1079:   
 1080:            
 1081:     def getLastVersion(self):
 1082:         """hole letzte version"""
 1083: 
 1084:         ids=[]
 1085:         idsTmp= self.objectIds()
 1086:         for x in idsTmp:
 1087:             try:
 1088:                 ids.append(int(x))
 1089:             except:
 1090:                 pass
 1091:         ids.sort()
 1092:       
 1093:         if len(ids)==0:
 1094:             return None
 1095:         else:    
 1096:             ob=getattr(self,str(ids[-1]))
 1097: 
 1098:             
 1099:             return ob
 1100:    
 1101:     def getVersions(self):
 1102:         """get versions"""
 1103:         versions=self.ZopeFind(self,obj_metatypes=["CDLIBasketVersion"])
 1104:         return versions
 1105: 
 1106:    
 1107:     def updateObjects(self,ids,RESPONSE=None,REQUEST=None):
 1108:         """update ids, ids not in the basket the add"""
 1109:         if type(ids) is not ListType:
 1110:             ids=[ids]
 1111:        
 1112:         lastVersion=self.getLastVersion() 
 1113:         oldContent=lastVersion.content.getContent()
 1114:         newContent=[]
 1115:         
 1116:         #first copy the old
 1117:         for obj in oldContent:
 1118:             if obj[1].getId() not in ids:
 1119:                 newContent.append(obj)
 1120:         #now add the new
 1121:                
 1122:         for id in ids:
 1123:             founds=self.CDLICatalog.search({'title':id})
 1124: 
 1125:             for found in founds:
 1126:                 if found.getObject() not in oldContent:
 1127:                     #TODO: was passiert wenn, man eine Object dazufŸgt, das schon da ist aber eine neuere version
 1128:                     newContent.append((found.getObject().getLastVersion(),found.getObject()))
 1129:         
 1130: 
 1131:         content=newContent 
 1132:         user=self.getActualUserName()
 1133:         
 1134:         ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=newContent)
 1135:         
 1136:         obj=self._getOb(ob.getId())
 1137:         if RESPONSE:
 1138:            
 1139:             RESPONSE.redirect(obj.absolute_url())
 1140:         
 1141:         return obj
 1142:     
 1143:     def addObjectsWithVersion(self,ids,deleteOld=None,username=None,catalog=None):
 1144:         """generate a new version of the basket with objects added, 
 1145:         hier wird jedoch nicht die letzte Version jedes Files hinzugefuegt, s
 1146:         ondern ids is ein Tupel mit der Id (d.h. der p-number) und der Versionsnummer.
 1147:         """
 1148:         logging.info("add to basket (%s)"%(self.getId()))
 1149:         lastVersion=self.getLastVersion()
 1150:         
 1151:         if not catalog:
 1152:             catalog=self.CDLICatalog
 1153:             
 1154:         if lastVersion is None:
 1155:             oldContent=[]
 1156:         else:
 1157:             oldContent=lastVersion.content.getContent()
 1158: 
 1159:         if deleteOld:
 1160:             oldContent=[]
 1161: 
 1162:         newContent=[]
 1163:         added=0
 1164:        
 1165:         for id,version in ids.iteritems():
 1166:             logging.info("adding %s %s"%(id,version))
 1167:             id=id.split(".")[0] # title nur die pnumber ohne atf
 1168:            
 1169:             try:
 1170:                 founds=catalog.search({'title':id})
 1171:             except:
 1172:                 founds=[]
 1173:             logging.info(" found %s "%(founds))
 1174:             for found in founds:
 1175:                 if found.getObject() not in oldContent:
 1176:                  
 1177:                     #TODO: was passiert wenn, man eine Object dazufŸgt, das schon da ist aber eine neuere version
 1178:                     newContent.append((found.getObject().getVersions()[version-1][1],found.getObject()))
 1179:                     added+=1
 1180: 
 1181:         content=oldContent+newContent
 1182:         if not username:
 1183:             logging.error("XXXXXXXXXXX %s"%repr(self))
 1184:             user=self.getActualUserName()
 1185:         else:
 1186:             user = username
 1187:             
 1188:         ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=content)
 1189:         logging.info("add to basket (%s) done"%(self.getId()))
 1190:         return added
 1191:     
 1192:     
 1193:     def addObjects(self,ids,deleteOld=None,username=None):
 1194:         """generate a new version of the basket with objects added"""
 1195:         
 1196:         def swap(x):
 1197:             return (x[1],x[0])
 1198:             
 1199:         logging.info("add to basket (%s)"%(self.getId()))
 1200:         lastVersion=self.getLastVersion()
 1201:         
 1202:         if lastVersion is None:
 1203:             oldContent=[]
 1204:         else:
 1205:             oldContent=lastVersion.content.getContent()
 1206: 
 1207:         if deleteOld:
 1208:             oldContent=[]
 1209: 
 1210:         added=0
 1211: #        for id in ids:
 1212: #            logging.debug("adding:"+id)
 1213: #            try:
 1214: #                founds=self.CDLICatalog.search({'title':id})
 1215: #            except:
 1216: #                founds=[]
 1217: #           
 1218: #            for found in founds:
 1219: #                if found.getObject() not in oldContent:
 1220: #                    #TODO: was passiert wenn, man eine Object dazufŸgt, das schon da ist aber eine neuere version
 1221: #                    newContent.append((found.getObject().getLastVersion(),found.getObject()))
 1222: #                    added+=1
 1223: 
 1224:         hash = md5.new(repr(makelist(ids))).hexdigest() # erzeuge hash als identification
 1225:         #logging.debug("JJJJJJJ:"+repr(self.makelist(ids)))
 1226:         logging.debug("JJJJJJJ:"+repr(hash))
 1227:                       
 1228:         if hasattr(self.cdliRoot,'v_tmpStore') and self.cdliRoot.v_tmpStore.has_key(hash): 
 1229:             logging.debug("from store!")
 1230:             newContent=Set(map(swap,self.cdliRoot.v_tmpStore[hash]))
 1231:          
 1232:         else:
 1233:             logging.debug("not from store!")
 1234:             newContent=Set([(self.getFileObjectLastVersion(x),self.getFileObject(x)) for x in ids])
 1235:         
 1236:         
 1237:         content=Set(oldContent).union(newContent)
 1238:         added = len(content)-len(oldContent)
 1239:         if not username:
 1240:             user=self.getActualUserName()
 1241:         else:
 1242:             user = username
 1243:         
 1244:         #logging.debug("content:"+repr(list(content)))
 1245:         ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=list(content))
 1246:         logging.info("add to basket (%s) done"%(self.getId()))
 1247:         return added
 1248:     
 1249:     
 1250:                 
 1251:     def getContent(self):
 1252:         """print content"""
 1253:         ret=[]
 1254:         
 1255:         lv=self.getLastVersion()
 1256:         for obj in lv.content.getContent():
 1257:             logging.info("XXXXXXXXXX %s"%repr(obj))
 1258:             ret.append((obj[1].getId(),obj[0].versionNumber))
 1259:             
 1260:         return ret
 1261:         
 1262:     def getContentIds(self):
 1263:         """print basket content"""
 1264:         ret=[]
 1265:         lv=self.getLastVersion()
 1266:         for obj in lv.content.getContent():
 1267:             ret.append((obj[0].getId(),obj[1].getId()))
 1268:         
 1269:         
 1270:         return lv.getComment(),lv.getUser(),lv.getTime(),ret
 1271: 
 1272:     def changeBasket(self,ids,submit,RESPONSE=None,REQUEST=None):
 1273:         """change a basket"""
 1274:         if submit=="update":
 1275:             return self.updateObjects(ids,RESPONSE=RESPONSE,REQUEST=REQUEST)
 1276:         elif submit=="delete":
 1277:             return self.deleteObjects(ids,RESPONSE=RESPONSE,REQUEST=REQUEST)
 1278:             
 1279:     def deleteObjects(self,ids,RESPONSE=None,REQUEST=None):
 1280:         """delete objects"""
 1281:         
 1282:         if type(ids) is not ListType:
 1283:             ids=[ids]
 1284:        
 1285:         lastVersion=self.getLastVersion() 
 1286:         oldContent=lastVersion.content.getContent()
 1287:         newContent=[]
 1288:         for obj in oldContent:
 1289:             if obj[1].getId() not in ids:
 1290:                 newContent.append(obj)
 1291:         
 1292:                 
 1293:         user=self.getActualUserName()
 1294:         
 1295:         ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=newContent)
 1296:         
 1297:         if RESPONSE:
 1298:             obj=self._getOb(ob.getId())
 1299:             RESPONSE.redirect(obj.absolute_url())
 1300:         
 1301: def manage_addCDLIBasketForm(self):
 1302:     """add the CDLIBasketContainer form"""
 1303:     pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIBasket.zpt')).__of__(self)
 1304:     return pt()
 1305: 
 1306: def manage_addCDLIBasket(self,title,shortDescription="",comment="",RESPONSE=None):
 1307:     """add the basket"""
 1308:     
 1309:     id=str(self.getNewId())
 1310:     
 1311:     ob=CDLIBasket(id,title,shortDescription,comment)
 1312:     
 1313:     self._setObject(id, ob)
 1314:     
 1315:     if RESPONSE is not None:
 1316:         RESPONSE.redirect('manage_main')
 1317:     else:
 1318:         return ob
 1319: 
 1320: class CDLIBasketVersion(Implicit,Persistent,Folder):
 1321:     """version of a basket"""
 1322:     
 1323:     meta_type="CDLIBasketVersion"
 1324:     security=ClassSecurityInfo()
 1325:     
 1326:     def updateBasket(self):
 1327:         """update"""
 1328:         try:
 1329:             self._setObject('content',BasketContent(self.basketContent))
 1330:         except:
 1331:             try:
 1332:                 if len(self.basketContent)>0:
 1333:                     self.content.setContent(self.basketContent)
 1334:             except:
 1335:                 print "error",self.getId(),self.aq_parent.getId()
 1336:         self.basketContent=[]
 1337: 
 1338:         
 1339:     def containsNonActualFiles(self):
 1340:         """returns True if basket contains one or more non current files"""
 1341:         
 1342:         objs=self.getContent()
 1343:         for obj in objs:
 1344:             if not self.isActual(obj)[0]:
 1345:                 return True
 1346:         return False
 1347:     
 1348:     def downloadListOfPnumbers(self):
 1349:         """download pnumbers of the basket as list"""
 1350:         
 1351:         basket_name=self.aq_parent.title
 1352:         
 1353:         ids=self.getContent() # get the list of objects
 1354:         logging.error(ids)
 1355:         ret="\n".join([x[1].getId().split(".")[0] for x in ids])
 1356:         
 1357:         self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.txt" """%basket_name)
 1358:         self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream")
 1359:         length=len(ret)
 1360:         self.REQUEST.RESPONSE.setHeader("Content-Length",length)
 1361:         self.REQUEST.RESPONSE.write(ret)    
 1362:         
 1363:     security.declareProtected('manage','downloadObjectsAsOneFile')
 1364:     def downloadObjectsAsOneFile(self,lock=None,procedure=None,REQUEST=None,check="yes",current="no"):
 1365:         """download all selected files in one file"""
 1366:        
 1367:         if self.temp_folder.downloadCounterBaskets > 10000:
 1368:             return """I am sorry, currently the server has to many requests for downloads, please come back later!"""
 1369: 
 1370: 
 1371:         if (check=="yes") and self.containsNonActualFiles():
 1372:             pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadObjectAsOneFile_check.zpt')).__of__(self)
 1373:             
 1374:             return pt(lock=lock)
 1375:             
 1376:         else:
 1377:             
 1378:             return self.downloadObjectsAsOneFileFinally(lock=lock,procedure=procedure,REQUEST=REQUEST,current="no")
 1379:         
 1380:     def downloadObjectsAsOneFileFinally(self,lock=None,procedure=None,REQUEST=None,current="no"):
 1381:         """print do the download"""
 1382:         logging.debug("HIHHHH")
 1383:         ret=""
 1384:         lockedObjects={}
 1385: 
 1386:         self.temp_folder.downloadCounterBaskets+=1 
 1387:         self._p_changed=1
 1388:         transaction.get().commit()       
 1389:     
 1390:         if lock:
 1391:             
 1392:             if str(self.REQUEST['AUTHENTICATED_USER'])=='Anonymous User':
 1393:                 self.temp_folder.downloadCounterBaskets-=1 
 1394:                 self._p_changed=1
 1395:                 transaction.get().commit()      
 1396:                 self.temp_folder.downloadCounterBaskets-=1 
 1397:                 self._p_changed=1
 1398:                 transaction.get().commit()      
 1399:                 return "please login first"
 1400: 
 1401:             #check if a locked object exist in the basket.
 1402:             lockedObjects={}
 1403:             for object in self.content.getContent():
 1404: 
 1405:                 if (not str(object[1].lockedBy)=="") and (not (str(object[1].lockedBy)==str(self.REQUEST['AUTHENTICATED_USER']))):
 1406:                     lockedObjects[object[1].title]=repr(object[1].lockedBy)
 1407:                    
 1408:                     
 1409:             keys=lockedObjects.keys()
 1410:             
 1411:             
 1412:             if len(keys)>0 and (not procedure):
 1413:                 self.REQUEST.SESSION['lockedObjects']=lockedObjects
 1414:                 pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','lockedObjects.zpt')).__of__(self)
 1415:                 
 1416:                 self.temp_folder.downloadCounterBaskets-=1 
 1417:                 self._p_changed=1
 1418:                 transaction.get().commit()      
 1419: 
 1420:                 return pt()
 1421:          
 1422:             elif not procedure: #keine fails gesperrt dann alle donwloaden
 1423:                 procedure="downloadAll" 
 1424:         
 1425:         
 1426: 
 1427: 
 1428:         for object in self.content.getContent():
 1429:                 logging.error("ret:"+repr(object[0]))
 1430:                 logging.error("    -"+repr(procedure))
 1431:                 logging.error("    -"+repr(object[1].lockedBy))
 1432:                 
 1433:                 if (procedure=="downloadAll") or (object[1].lockedBy=='') or (object[1].lockedBy==self.REQUEST['AUTHENTICATED_USER']):
 1434:                     logging.error("ret1")
 1435:                     if current=="no": #version as they are in the basket
 1436:                         logging.error("ret2")
 1437:                         ret+=str(object[0].getData())+"\n"
 1438:                     elif current=="yes":
 1439:                         logging.error("ret3")
 1440:                         #search current object
 1441:                         founds=self.CDLICatalog.search({'title':object[1].getId().split(".")[0]})
 1442:                         if len(founds)>0:      
 1443:                             ret+=str(founds[0].getObject().getLastVersion().getData())+"\n"
 1444:                             
 1445:                 if lock and object[1].lockedBy=='':
 1446:                     object[1].lockedBy=self.REQUEST['AUTHENTICATED_USER']
 1447:         
 1448:         if (not isinstance(self.aq_parent,CDLIBasket)):
 1449:             basket_name=self.aq_parent.aq_parent.title+"_V"+self.getId()
 1450:         else:
 1451:             basket_name=self.aq_parent.title+"_V"+self.getId()
 1452:         
 1453:         
 1454:     
 1455:         #write basketname to header of atf file
 1456:         ret="#basket: %s\n"%basket_name+ret
 1457: 
 1458:         self.temp_folder.downloadCounterBaskets-=1 
 1459:         self._p_changed=1
 1460:         transaction.get().commit()      
 1461:         
 1462:         self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.atf" """%basket_name)
 1463:         self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream")
 1464:         length=len(ret)
 1465:         self.REQUEST.RESPONSE.setHeader("Content-Length",length)
 1466:         self.REQUEST.RESPONSE.write(ret)    
 1467:         return True
 1468:         
 1469:     def numberOfItems(self):
 1470:         """return anzahl der elemente im basket"""
 1471:         return self.content.numberOfItems()
 1472:     
 1473:     def getTime(self):
 1474:         """getTime"""
 1475:         #return self.bobobase_modification_time().ISO()
 1476:       
 1477:         if hasattr(self,'time'):
 1478:             return time.strftime("%Y-%m-%d %H:%M:%S",self.time)
 1479:         elif hasattr(self,'timefixed'):
 1480:             return self.timefixed
 1481:         else:
 1482:             setattr(self,'timefixed',self.bobobase_modification_time().ISO())
 1483:             return self.bobobase_modification_time().ISO()
 1484:     
 1485:     def getContent(self):
 1486:         """get Basket Content"""
 1487:         return self.content.getContent()
 1488: 
 1489:     
 1490:     def __init__(self,id,user,comment="",basketContent=[]):
 1491:         """ init a basket version"""
 1492:         self.id=id
 1493:         self.comment=comment
 1494:         self._setObject('content',BasketContent(basketContent))
 1495:         #self.basketContent=basketContent[0:]a
 1496:         self.user=user
 1497:         self.time=time.localtime()
 1498:         
 1499:     def getUser(self):
 1500:         """get user"""
 1501:         return self.user
 1502:     
 1503:     def getComment(self):
 1504:         """get Comment"""
 1505:         return self.comment
 1506:  
 1507:     security.declareProtected('manage','index_html')
 1508:     def index_html(self):
 1509:             """view the basket"""
 1510: 
 1511:             if self.REQUEST.get('change',False):
 1512:                     ob=self.aq_parent.updateObjects(self.REQUEST['change'])
 1513:                    
 1514:                     self.REQUEST.RESPONSE.redirect(ob.absolute_url())#go to new basket, because changing generates a new basket
 1515:                                         
 1516:             pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketVersionMain.zpt')).__of__(self)
 1517:             return pt()
 1518:      
 1519:     def getObjUrl(self,result):
 1520:         """getUrl of the version of the object"""
 1521:         objId=result[1].getTitle()
 1522:         founds=self.CDLICatalog.search({'title':objId})
 1523:         if len(founds)>0:
 1524:              return founds[0].getObject().getLastVersion().absolute_url()
 1525:          
 1526:         else: #assume version number
 1527:             splitted=objId.split("_")
 1528:             founds=self.CDLICatalog.search({'title':splitted[1]})        
 1529:             return founds[0].getObject().getLastVersion().absolute_url()+'/'+objId
 1530:    
 1531: def manage_addCDLIBasketVersion(self,user,comment="",basketContent=[],RESPONSE=None):
 1532:     """add a version"""
 1533:     
 1534:     #check for already existing versions
 1535:  
 1536:     lastVersion=self.getLastVersion()
 1537:     if lastVersion is None:
 1538:         newId=str(1)
 1539:     else:
 1540:         newId=str(int(lastVersion.getId())+1)
 1541:     
 1542:     ob=CDLIBasketVersion(newId,user,comment,basketContent)
 1543:     
 1544:     self._setObject(newId, ob)
 1545:     
 1546:     if RESPONSE is not None:
 1547:         RESPONSE.redirect('manage_main')
 1548:     else:
 1549:         return ob
 1550:     
 1551: class CDLIFileObject(CatalogAware,extVersionedFileObject):
 1552:     """CDLI file object"""
 1553:     
 1554:     meta_type="CDLI File Object"
 1555:     default_catalog='CDLIObjectsCatalog'
 1556:     
 1557:     security=ClassSecurityInfo()
 1558:     
 1559:     security.declareProtected('manage','index_html')
 1560: 
 1561:     security.declarePublic('view')
 1562:     view = PageTemplateFile('zpt/viewCDLIFile.zpt', globals())
 1563: 
 1564:     security.declarePublic('editATF')
 1565:     editATF = PageTemplateFile('zpt/editATFFile.zpt', globals())
 1566: 
 1567:     def PrincipiaSearchSource(self):
 1568:            """Return cataloguable key for ourselves."""
 1569:            return str(self)
 1570:        
 1571:     def makeThisVersionCurrent_html(self):
 1572:         """form for mthis version current"""
 1573:         
 1574:         pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','makeThisVersionCurrent.zpt')).__of__(self)
 1575:         return pt()                 
 1576: 
 1577:     security.declarePublic('makeThisVersionCurrent')
 1578:     def makeThisVersionCurrent(self,comment,author,RESPONSE=None):
 1579:         """copy this version to current"""
 1580:         parent=self.aq_parent
 1581:         parent.manage_addVersionedFileObject(id=None,vC=comment,author=author,file=self.getData(),RESPONSE=RESPONSE)
 1582:         #newversion=parent.manage_addCDLIFileObject('',comment,author)
 1583:         #newversion.manage_upload(self.getData())
 1584:                                         
 1585:         #if RESPONSE is not None:
 1586:         #    RESPONSE.redirect(self.aq_parent.absolute_url()+'/history')
 1587: 
 1588:         return True
 1589:     
 1590:     def getFormattedData(self):
 1591:         """fromat text"""
 1592:         data=self.getData()
 1593: #        return re.sub("\s\#lem"," #lem",data) #remove return vor #lem
 1594:         return re.sub("#lem","       #lem",data) #remove return vor #lem
 1595:         
 1596:     
 1597:     security.declarePublic('getPNumber')
 1598:     def getPNumber(self):
 1599:         """get the pnumber"""
 1600:         try:
 1601:                 txt=re.match("&[Pp](\d*)\s*=([^\r\n]*)",self.getData()[0:])
 1602:         except:
 1603:                 txt=self.getData()[0:]
 1604:                 
 1605:                 return "ERROR"
 1606:         try:
 1607:             return "P"+txt.group(1)
 1608:         except:
 1609:             return "ERROR"
 1610: 
 1611:     security.declarePublic('getDesignation')
 1612:     def getDesignation(self):
 1613:         """get the designation out of the file"""
 1614:         try:
 1615:                 txt=re.match("&[Pp](\d*)\s*=([^\r\n]*)",self.getData()[0:])
 1616:         except:
 1617:                 txt=self.getData()[0:]
 1618:                 
 1619:                 return "ERROR"
 1620:         try:
 1621:             return txt.group(2)
 1622:         except:
 1623:             return "ERROR"
 1624: 
 1625:         
 1626: manage_addCDLIFileObjectForm=DTMLFile('dtml/fileAdd', globals(),Kind='CDLIFileObject',kind='CDLIFileObject', version='1')
 1627: 
 1628: def manage_addCDLIFileObject(self,id,vC='',author='', file='',title='',versionNumber=0,
 1629:                              precondition='', content_type='',
 1630:                              from_tmp=False,REQUEST=None):
 1631:     """Add a new File object.
 1632:     Creates a new File object 'id' with the contents of 'file'"""
 1633:  
 1634:     id=str(id)
 1635:     title=str(title)
 1636:     content_type=str(content_type)
 1637:     precondition=str(precondition)
 1638:     
 1639:     id, title = cookId(id, title, file)
 1640: 
 1641:     self=self.this()
 1642: 
 1643:     # First, we create the file without data:
 1644:     self._setObject(id, CDLIFileObject(id,title,versionNumber=versionNumber,versionComment=vC,time=time.localtime(),author=author))
 1645:     fob = self._getOb(id)
 1646:     
 1647:     # Now we "upload" the data.  By doing this in two steps, we
 1648:     # can use a database trick to make the upload more efficient.
 1649: 
 1650:     if file and not from_tmp:
 1651:         fob.manage_upload(file)
 1652:     elif file and from_tmp:
 1653:         fob.manage_file_upload(file) # manage_upload_from_tmp doesn't exist in ExtFile2
 1654:     #    fob.manage_upload_from_tmp(file) # manage_upload_from_tmp doesn't exist in ExtFile2
 1655:     if content_type:
 1656:         fob.content_type=content_type
 1657: 
 1658:     #logging.debug("manage_add: lastversion=%s"%self.getData())
 1659:     logging.debug("reindex1: %s in %s"%(repr(self),repr(self.default_catalog)))
 1660:     self.reindex_object()
 1661:     #logging.debug("manage_add: fob_data=%s"%fob.getData())
 1662:     logging.debug("reindex2: %s in %s"%(repr(fob), repr(fob.default_catalog)))
 1663:     fob.index_object()
 1664: 
 1665:     self.CDLIRoot.updateOrAddToFileBTree(ob)
 1666:     if REQUEST is not None:
 1667:         REQUEST['RESPONSE'].redirect(self.absolute_url()+'/manage_main')
 1668:     
 1669: 
 1670: class CDLIFile(extVersionedFile,CatalogAware):
 1671:     """CDLI file"""
 1672:     
 1673:     security=ClassSecurityInfo()
 1674:     meta_type="CDLI file"
 1675:     content_meta_type = ["CDLI File Object"]
 1676:     
 1677:     default_catalog='CDLICatalog'
 1678:     
 1679:     security.declareProtected('manage','index_html')
 1680:     
 1681:     def getLastVersionData(self):
 1682:         """get last version data"""
 1683:         return self.getData()
 1684: 
 1685:     def getLastVersionFormattedData(self):
 1686:         """get last version data"""
 1687:         return self.getContentObject().getFormattedData()
 1688: 
 1689:     def getTextId(self):
 1690:         """returns P-number of text"""
 1691:         # assuming that its the beginning of the title
 1692:         return self.title[:7]
 1693: 
 1694:     #security.declarePublic('history')
 1695:     def history(self):
 1696:         """history"""  
 1697: 
 1698:         ext=self.ZopeFind(self.aq_parent,obj_ids=["history_template.html"])
 1699:         if ext:
 1700:             return getattr(self,ext[0][1].getId())()
 1701:         
 1702:         pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','versionHistory')).__of__(self)
 1703:         return pt()
 1704: 
 1705: 
 1706:     def getBasketFromId(self,basketid, context=None):
 1707:         """get basket from id"""
 1708: 
 1709:         if not context:
 1710:             context=self
 1711:             
 1712:         for basket in self.ZopeFind(context,obj_metatypes=["CDLIBasket"]):
 1713:             if basket[0]==basketid:
 1714:                 return basket[1]
 1715:         else:
 1716:             None
 1717: 
 1718:  
 1719:     def isContainedInBaskets(self,context=None):
 1720:         """check is this file is part of any basket
 1721:         @param context: (optional) necessessary if CDLIBasketCatalog is not an (inherited) attribute of self, context.CDLIBasketCatalog
 1722:                         has to exist.
 1723:         """
 1724: 
 1725:         if not context:
 1726:             context=self
 1727:         
 1728:         ret=[]
 1729:         for x in context.CDLIBasketCatalog.search({'getFileNamesInLastVersion':self.getId()}):
 1730:             #if the basket x is deleted it seemes to be that x is sometimes still in the Catalog, why?
 1731:             try:
 1732:                 ret.append(x.getObject())
 1733:             except:
 1734:                 pass
 1735:         return ret
 1736:         #return [x.getObject() for x in context.CDLIBasketCatalog.search({'getFileNamesInLastVersion':self.getId()})]
 1737:         
 1738:         
 1739:     def _newContentObject(self, id, title='', versionNumber=0, versionComment=None, time=None, author=None):
 1740:         """factory for content objects. to be overridden in derived classes."""
 1741:         logging.debug("_newContentObject(CDLI)")
 1742:         return CDLIFileObject(id,title,versionNumber=versionNumber,versionComment=versionComment,time=time,author=author)
 1743: 
 1744: 
 1745:     def addCDLIFileObjectForm(self):
 1746:         """add a new version"""
 1747:         
 1748:         if str(self.REQUEST['AUTHENTICATED_USER']) in ["Anonymous User"]:
 1749:             return "please login first"
 1750:         if (self.lockedBy==self.REQUEST['AUTHENTICATED_USER']) or (self.lockedBy==''):
 1751:             out=DTMLFile('dtml/fileAdd', globals(),Kind='CDLIFileObject',kind='CDLIFileObject',version=self.getVersion()).__of__(self)
 1752:             return out()
 1753:         else:
 1754:             return "Sorry file is locked by somebody else"
 1755:         
 1756:     def manage_addCDLIFileObject(self,id,vC,author,
 1757:                                  file='',title='',
 1758:                                  precondition='', 
 1759:                                  content_type='',
 1760:                                  changeName='no',newName='', 
 1761:                                  come_from=None,
 1762:                                  from_tmp=False,RESPONSE=None):
 1763:         """add"""
 1764:       
 1765:         try: #TODO: der ganze vC unsinn muss ueberarbeitet werden
 1766:             vC=self.REQUEST['vC']
 1767:         except:
 1768:             pass
 1769:         
 1770:         ob = self.addContentObject(id, vC, author, file, title, changeName=changeName, newName=newName, from_tmp=from_tmp,
 1771:                                    precondition=precondition, content_type=content_type)
 1772: 
 1773:         try:
 1774:             #FIXME: wozu ist das gut?
 1775:             self.REQUEST.SESSION['objID_parent']=self.getId()
 1776:         except:
 1777:             pass
 1778:   
 1779:         #self.cdliRoot.updateOrAddToFileBTree(self)# now update the object in the cache
 1780:       
 1781:         
 1782:         if RESPONSE:
 1783:             if ob.getSize()==0:
 1784:                 self.REQUEST.SESSION['objID']=ob.getId()
 1785:                 pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','errorUploadFile')).__of__(self)
 1786:                 return pt()
 1787:             else:
 1788:                 if come_from and (come_from!=""):
 1789:                     RESPONSE.redirect(come_from+"?change="+self.getId())
 1790:                 else:
 1791:                     RESPONSE.redirect(self.REQUEST['URL2']+'?uploaded=%s'%self.title)
 1792:         else:
 1793:             return ob
 1794:         
 1795:         
 1796: def manage_addCDLIFileForm(self):
 1797:     """interface for adding the OSAS_root"""
 1798:     pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIFile.zpt')).__of__(self)
 1799:     return pt()
 1800: 
 1801: def manage_addCDLIFile(self,id,title,lockedBy, author=None, RESPONSE=None):
 1802:     """add the OSAS_root"""
 1803:     newObj=CDLIFile(id,title,lockedBy,author)
 1804:                                         
 1805:     tryToggle=True
 1806:     tryCount=0
 1807: 
 1808:     self._setObject(id,newObj)                  
 1809:     getattr(self,id).reindex_object()
 1810:         
 1811:     if RESPONSE is not None:
 1812:         RESPONSE.redirect('manage_main')
 1813: 
 1814: 
 1815: def checkUTF8(data):
 1816:     """check utf 8"""
 1817:     try:
 1818:         data.encode('utf-8')
 1819:         return True
 1820:     except:
 1821:         return False
 1822:     
 1823: 
 1824: def checkFile(filename,data,folder):
 1825:     """check the files"""
 1826:     # first check the file name
 1827:     fn=filename.split(".") # no extension
 1828: 
 1829:     if not fn[0][0]=="P":
 1830:         return False,"P missing in the filename"
 1831:     elif len(fn[0])!=7:
 1832:         return False,"P number has not the right length 6"
 1833:     elif not checkUTF8(data):
 1834:         return False,"not utf-8"
 1835:     else:
 1836:         return True,""
 1837:     
 1838:     
 1839: def splitatf(fh,dir=None,ext=None):
 1840:     """split it"""
 1841:     ret=None
 1842:     nf=None
 1843:     i=0
 1844: 
 1845:     #ROC: why split \n first and then \r???
 1846:     if (type(fh) is StringType) or (type(fh) is UnicodeType):
 1847:         iter=fh.split("\n")
 1848:     else:
 1849:         iter=fh.readlines()
 1850:         
 1851:     for lineTmp in iter:
 1852:         lineTmp=lineTmp.replace(codecs.BOM_UTF8,'') # make sure that all BOM are removed..
 1853:         for line in lineTmp.split("\r"):
 1854:             #logging.log("Deal with: %s"%line)
 1855:             if ext:
 1856:                 i+=1
 1857:                 if (i%100)==0:
 1858:                     ext.result+="."
 1859:                 if i==10000:
 1860:                     i=0
 1861:                     ext.result+="<br>"
 1862:             #check if basket name is in the first line
 1863:             if line.find("#atf basket")>=0: #old convention
 1864:                 ret=line.replace('#atf basket ','')
 1865:                 ret=ret.split('_')[0]
 1866:             elif line.find("#basket:")>=0: #new convention
 1867:                 ret=line.replace('#basket: ','')
 1868:                 ret=ret.split('_')[0]
 1869: 
 1870:             else:
 1871:                 if (len(line.lstrip())>0) and (line.lstrip()[0]=="&"): #newfile
 1872:                     if nf:
 1873:                         nf.close() #close last file
 1874: 
 1875: 
 1876:                     filename=line[1:].split("=")[0].rstrip()+".atf"
 1877:                     if dir:
 1878:                         filename=os.path.join(dir,filename)
 1879:                     nf=file(filename,"w")
 1880:                     logging.info("open %s"%filename)
 1881:                 if nf:    
 1882:                     nf.write(line.replace("\n","")+"\n")
 1883: 
 1884:     try:        
 1885:         nf.close()
 1886:     except:
 1887:         pass
 1888:     
 1889:     if not((type(fh) is StringType) or (type(fh) is UnicodeType)):
 1890:         fh.close()
 1891:     return ret,len(os.listdir(dir))
 1892: 
 1893: 
 1894: class CDLIFileFolder(extVersionedFileFolder):
 1895:     """CDLI File Folder"""
 1896:     
 1897:     security=ClassSecurityInfo()
 1898:     meta_type="CDLI Folder"
 1899:     file_meta_type=['CDLI file']
 1900:     folder_meta_type=['CDLI Folder']
 1901: 
 1902:     file_catalog='CDLICatalog'
 1903: 
 1904:     #downloadCounter=0 # counts how many download for all files currently run, be mehr als 5 wird verweigert.
 1905:     tmpStore2={}
 1906: 
 1907:     def _newVersionedFile(self, id, title='', lockedBy=None, author=None):
 1908:         """factory for versioned files. to be overridden in derived classes."""
 1909:         logging.debug("_newVersionedFile(CDLI)")
 1910:         return CDLIFile(id, title, lockedBy=lockedBy, author=author)
 1911: 
 1912:     def setTemp(self,name,value):
 1913:         """set tmp"""
 1914: 
 1915:         setattr(self,name,value)
 1916:                                         
 1917:     deleteFileForm = PageTemplateFile("zpt/doDeleteFile", globals())
 1918:                                        
 1919:     def delete(self,ids,REQUEST=None):
 1920:         """delete these files"""
 1921:         if type(ids) is not ListType:
 1922:             ids=[ids]
 1923: 
 1924:         self.manage_delObjects(ids)
 1925:         
 1926:         if REQUEST is not None:
 1927:             return self.index_html()
 1928: 
 1929: 
 1930:     def getVersionNumbersFromIds(self,ids):
 1931:         """get the numbers of the current versions of documents described by their ids"""
 1932:         
 1933:         ret=[]
 1934:         searchStr=" OR ".join(ids)
 1935:         
 1936:         founds=self.CDLICatalog.search({'title':searchStr})
 1937:         
 1938:         for found in founds:
 1939:             lastVersion=found.getObject().getContentObject()
 1940:             ret.append((found.getId,lastVersion))
 1941:         
 1942:         return ret
 1943:     
 1944:     def getFile(self,fn):
 1945:         """get the content of the file fn"""
 1946:         logging.debug("getFile: %s"%repr(fn))
 1947:         if not self.hasObject(fn):
 1948:             # search deeper
 1949:             founds=getattr(self, self.file_catalog).search({'textid':fn})
 1950:             if founds:
 1951:                 obj=founds[0].getObject().getContentObject()
 1952:             else:
 1953:                 return "" 
 1954:         else:
 1955:             obj = self[fn].getContentObject()
 1956: 
 1957:         return obj.getData()[0:] 
 1958:  
 1959:     
 1960:     def checkCatalog(self,fn):
 1961:         """check if fn is in the catalog"""
 1962:         #TODO add checkCatalog
 1963:         
 1964:                                    
 1965:     def findObjectsFromListWithVersion(self,list,author=None):
 1966:         """find objects from a list with versions
 1967:         @param list: list of tuples  (cdliFile,version)
 1968:         """
 1969:         #self.REQUEST.SESSION['fileIds']=list#store fieldIds in session for further usage
 1970:         #self.REQUEST.SESSION['searchList']=self.REQUEST.SESSION['fileIds']
 1971:         
 1972:         pt=getattr(self,'filelistVersioned.html')
 1973:             
 1974:         return pt(search=list,author=author)
 1975:     
 1976:     
 1977:     def getAllPNumbers(self):
 1978:         """get a list of all files (resp their p-numbers) stored"""
 1979:         
 1980:         ret=[x.getId for x in  self.CDLICatalog()]
 1981:      
 1982:         return ret
 1983:     
 1984:     def expandFile(self,fileId,fileTree):
 1985:         """wildcard in fileID suche alle Treffer"""
 1986:         founds=self.CDLICatalog({'title':fileId})
 1987:         for found in founds:
 1988:             fileTree.add(found.getId)
 1989:             logging.debug("ADDD:"+found.getId)
 1990:          
 1991:     def findObjectsFromList(self,enterList=None,display=False,start=None,upload=None,list=None,basketName=None,numberOfObjects=None,RESPONSE=None,REQUEST=None,returnHash=False,hash=None):
 1992:         """findObjectsFromList (, TAB oder LINE separated)"""
 1993:                                        
 1994:         logging.debug("start: findObjectsFromList")
 1995:         #logging.debug("start: findObjectsFromList"+repr(list))
 1996:         
 1997:             
 1998:         if upload: # list from file upload
 1999:             txt=upload.read()
 2000:                                        
 2001:         if enterList:
 2002:             txt=enterList
 2003:             
 2004:         if upload or enterList:
 2005:             txt=txt.replace(",","\n")
 2006:             txt=txt.replace("\t","\n")
 2007:             txt=txt.replace("\r","\n")
 2008:             idsTmp=txt.split("\n")
 2009:             ids=[]
 2010:             for id in idsTmp: # make sure that no empty lines
 2011:                 idTmp=id.lstrip().rstrip()
 2012:                 if len(idTmp)>0:
 2013:                     
 2014:                     ids.append(idTmp)
 2015:                     
 2016:             #self.REQUEST.SESSION['ids']=" OR ".join(ids)
 2017: 
 2018:             pt=getattr(self,'filelist.html')
 2019:             self.REQUEST.SESSION['searchList']=ids
 2020:             return pt(search=ids)
 2021:         
 2022:         if basketName:
 2023:             #TODO: get rid of one of these..
 2024:             
 2025:             pt=getattr(self,'filelist.html')
 2026:             return pt(basketName=basketName,numberOfObjects=numberOfObjects)
 2027:         
 2028:         if hash is not None and hasattr(self.cdliRoot,'v_tmpStore') and self.cdliRoot.v_tmpStore.has_key(hash): 
 2029:                
 2030:                logging.debug("asking for storage2")
 2031:                result =self.cdliRoot.v_tmpStore[hash]
 2032:                if result:
 2033:                    logging.debug("give result from storage2")
 2034:                    return hash,self.cdliRoot.v_tmpStore[hash]
 2035:           
 2036:         if list is not None: # got already a list
 2037:             
 2038:             logging.debug(" ----List version")
 2039:             ret=[]
 2040:             fileTree=Set()
 2041:             
 2042:             for fileId in list:
 2043:                
 2044:                 if fileId.find("*")>-1: #check for wildcards
 2045:                         self.expandFile(fileId,fileTree)
 2046:                         
 2047:                 elif len(fileId.split("."))==1:
 2048:                         fileId=fileId+".atf"
 2049:                         fileTree.add(fileId)
 2050:                 #logging.debug("   -----:"+fileId)
 2051:                 #ret+=self.CDLICatalog({'title':fileId})
 2052:                 #x =self.getFileObject(fileId)
 2053:                 #if x is not None:
 2054:                 #    ret.append(x)
 2055:                 
 2056:             
 2057:             
 2058:             ids = fileTree & self.v_file_ids
 2059:             #self.REQUEST.SESSION['fileIds']=ids#store fieldIds in session for further usage
 2060:             l=makelist(fileTree)[0:]
 2061:             logging.debug("l-list:"+repr(l))
 2062:             self.REQUEST.SESSION['fileIds']=l#store fieldIds in session for further usage
 2063:             self.REQUEST.SESSION['searchList']=l
 2064:             #self.REQUEST.SESSION['searchList']=['P000001.atf']
 2065:           
 2066:             
 2067:             hash = md5.new(repr(makelist(fileTree))).hexdigest() # erzeuge hash als identification
 2068:             self.REQUEST.SESSION['hash']=hash
 2069:             #TODO: do I need garbage collection for v_tmpStore ?
 2070:             
 2071:             #logging.debug("Hash:"+repr(hash))
 2072: #        
 2073: #            if hasattr(self.cdliRoot,'v_tmpStore') and self.cdliRoot.v_tmpStore.has_key(hash): 
 2074: #               logging.debug("asking for storage")
 2075: #               res=self.cdliRoot.v_tmpStore[hash]
 2076: #               if res:
 2077: #                   if returnHash == True:
 2078: #                       return hash,res
 2079: #                   return res
 2080:           
 2081:             #TODO: get rid of one of these..
 2082:             #ids=[x.getObject().getId() for x in ret]
 2083:             ret=[(self.getFileObject(x),self.getFileObjectLastVersion(x)) for x in ids]
 2084:             
 2085:             #self.REQUEST.SESSION['fileIds']=ids#store fieldIds in session for further usage
 2086:             #self.REQUEST.SESSION['searchList']=self.REQUEST.SESSION['fileIds']
 2087:            
 2088:             if display:
 2089:                 pt=getattr(self,'filelist.html')
 2090:                 
 2091:                 return pt(search=ids)
 2092:             else:     
 2093:                 #self.REQUEST.SESSION['hash'] = ret # store in session 
 2094:                 if not hasattr(self,'v_tmpStore'):
 2095:                     self.cdliRoot.v_tmpStore={}
 2096:                 #logging.debug("HHHHHHNEU:"+repr(self.makelist(ids)))
 2097:                 #logging.debug("HHHHHHNEU:"+repr(hash))
 2098:                 self.cdliRoot.v_tmpStore[hash] = ret # store in session 
 2099:                 if returnHash == True:
 2100:                     return hash,ret
 2101:                 return ret
 2102:         
 2103:         
 2104:         
 2105:         if start:
 2106:             RESPONSE.redirect("filelist.html?start:int="+str(start))
 2107: 
 2108:     security.declareProtected('Manage','createAllFilesAsSingleFile')
 2109:     def createAllFilesAsSingleFile(self,RESPONSE=None):
 2110:         """download all files"""
 2111:         
 2112:         def sortF(x,y):
 2113:             return cmp(x[0],y[0])
 2114:         
 2115:         catalog=getattr(self,self.file_catalog)
 2116:         #tf,tfilename=mkstemp()
 2117:         if not hasattr(self.temp_folder,'downloadCounter'):
 2118:             self.temp_folder.downloadCounter=0
 2119: 
 2120:         if getattr(self.temp_folder,'downloadCounter',0) > 5:
 2121:             return """I am sorry, currently the server has to many requests for downloads, please come back later!"""
 2122: 
 2123:         self.temp_folder.downloadCounter+=1
 2124:         self._p_changed=1
 2125:         transaction.get().commit()
 2126:        
 2127:         list=[(x.getId,x) for x in catalog()]
 2128:         list.sort(sortF)
 2129:         
 2130: 
 2131:         
 2132:         RESPONSE.setHeader("Content-Disposition","""attachement; filename=%s"""%"all.atf")
 2133:         RESPONSE.setHeader("Content-Type","application/octet-stream")
 2134:         tmp=""
 2135:         for l in list:
 2136:             obj=l[1].getObject()
 2137:             
 2138:             if obj.meta_type=="CDLI file":
 2139:                 
 2140:                 #os.write(tf,obj.getLastVersion().data)
 2141:                 if RESPONSE:
 2142:                     RESPONSE.write(obj.getData()[0:])
 2143:                     RESPONSE.write("\n")
 2144:                 self.temp_folder.downloadCounter-=1 
 2145:                 self._p_changed=1
 2146:         transaction.get().commit()
 2147:         #os.close(tf)
 2148:         #RESPONSE.redirect(self.absolute_url()+"/downloadFile?fn="%tfilename)
 2149:         return True
 2150:     
 2151:     def downloadFile(self,fn):
 2152:         """download fn - not used yet"""
 2153:         self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename=%s"""%self.getLastVersion().getId())
 2154:         self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream")
 2155:         self.REQUEST.RESPONSE.write(file(fn).read())
 2156:         
 2157:       
 2158:                 
 2159:     def hasParent(self):
 2160:         """returns true falls subfolder"""
 2161:       
 2162:         if self.aq_parent.meta_type in self.folder_meta_type:
 2163:             return True
 2164:         else:
 2165:             return False
 2166:         
 2167:     def getFolders(self):
 2168:         """get all subfolders"""
 2169:         ret=[]
 2170:         folders=self.ZopeFind(self,obj_metatypes=self.folder_meta_type)
 2171:         for folder in folders:
 2172:             ret.append((folder[1],
 2173:                         len(self.ZopeFind(folder[1],obj_metatypes=self.folder_meta_type)),
 2174:                         len(self.ZopeFind(folder[1],obj_metatypes=self.file_meta_type))
 2175:                         ))
 2176:         return ret
 2177:     
 2178:             
 2179:     security.declareProtected('manage','index_html')
 2180:     def index_html(self):
 2181:         """main"""
 2182:         ext=self.ZopeFind(self,obj_ids=["index.html"])
 2183:         if ext:
 2184:             return ext[0][1]()
 2185:         
 2186:         pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','CDLIFileFolderMain')).__of__(self)
 2187:         return pt()
 2188:     
 2189:     
 2190: manage_addCDLIFileFolderForm=DTMLFile('dtml/folderAdd', globals())
 2191: 
 2192:     
 2193: def manage_addCDLIFileFolder(self, id, title='',
 2194:                      createPublic=0,
 2195:                      createUserF=0,
 2196:                      REQUEST=None):
 2197:     """Add a new Folder object with id *id*.
 2198: 
 2199:     If the 'createPublic' and 'createUserF' parameters are set to any true
 2200:     value, an 'index_html' and a 'UserFolder' objects are created respectively
 2201:     in the new folder.
 2202:     """
 2203:     ob=CDLIFileFolder()
 2204:     ob.id=str(id)
 2205:     ob.title=title
 2206:     self._setObject(id, ob)
 2207:     ob=self._getOb(id)
 2208: 
 2209:     checkPermission=getSecurityManager().checkPermission
 2210: 
 2211:     if createUserF:
 2212:         if not checkPermission('Add User Folders', ob):
 2213:             raise Unauthorized, (
 2214:                   'You are not authorized to add User Folders.'
 2215:                   )
 2216:         ob.manage_addUserFolder()
 2217: 
 2218:   
 2219:     if REQUEST is not None:
 2220:         return self.manage_main(self, REQUEST, update_menu=1)
 2221:     
 2222: class CDLIRoot(Folder):
 2223:     """main folder for cdli"""
 2224:     
 2225:     meta_type="CDLIRoot"
 2226:     downloadCounterBaskets=0 # counts the current basket downloads if counter > 10 no downloads are possible
 2227:     
 2228:     file_catalog = 'CDLICatalog'
 2229:     
 2230:     # word splitter for search
 2231:     splitter = {'words':cdliSplitter.wordSplitter(),
 2232:                 'graphemes':cdliSplitter.graphemeSplitter()}
 2233:     
 2234:     
 2235:     def viewATF(self,id,RESPONSE):
 2236:         """view an Object"""
 2237:         ob = self.CDLICatalog({'title':id})
 2238:         logging.debug(ob[0].getObject().getLastVersion().absolute_url()+"/view")
 2239:         if len(ob)>0:
 2240:             RESPONSE.redirect(ob[0].getObject().getLastVersion().absolute_url()+"/view")
 2241:         return "not found"
 2242:     
 2243:     def history(self,id,RESPONSE):
 2244:         """view an Object"""
 2245:         ob = self.CDLICatalog({'title':id})
 2246:         if len(ob)>0:
 2247:             RESPONSE.redirect(ob[0].absolute_url+"/history")
 2248:         return "not found"
 2249:     
 2250: 
 2251:     def downloadLocked(self,id,RESPONSE):
 2252:         """view an Object"""
 2253:         ob = self.CDLICatalog({'title':id})
 2254:         if len(ob)>0:
 2255:             RESPONSE.redirect(ob[0].absolute_url+"/downloadLocked")
 2256:         return "not found"
 2257:     
 2258:     def download(self,id,RESPONSE):
 2259:         """view an Object"""
 2260:         ob = self.CDLICatalog({'title':id})
 2261:         if len(ob)>0:
 2262:             RESPONSE.redirect(ob[0].getLastVersion().absolute_url())
 2263:         return "not found"
 2264:     def addCDLIFileObjectForm(self,id,RESPONSE):
 2265:         """view an Object"""
 2266:         ob = self.CDLICatalog({'title':id})
 2267:         if len(ob)>0:
 2268:             RESPONSE.redirect(ob[0].absolute_url+"/addCDLIFileObjectForm")
 2269:         return "not found"
 2270:     
 2271:     def addVersionedFileObjectForm(self,id,RESPONSE):
 2272:         """view an Object"""
 2273:         ob = self.CDLICatalog({'title':id})
 2274:         if len(ob)>0:
 2275:             RESPONSE.redirect(ob[0].absolute_url+"/addVersionedFileObjectForm")
 2276:         return "not found"
 2277:     
 2278:     def unlock(self,id,RESPONSE):
 2279:         """view an Object"""
 2280:         ob = self.CDLICatalog({'title':id})
 2281:         if len(ob)>0:
 2282:             RESPONSE.redirect(ob[0].absolute_url+"/unlock")
 2283:         return "not found"
 2284:     
 2285:     def getFileObject(self,fileId):
 2286:         """get an object"""
 2287:         x=self.v_files.get(fileId)
 2288:         #logging.debug(x)
 2289:         return x
 2290:     
 2291:     def getFileObjectLastVersion(self,fileId):
 2292:         """get an object"""
 2293:         x=self.v_files_lastVersion.get(fileId)
 2294:         #logging.debug(x)
 2295:         return x
 2296:     
 2297:     def showFileIds(self):
 2298:         """showIds"""
 2299:         return self.v_file_ids
 2300:     
 2301:     def generateFileBTree(self):
 2302:         """erzeuge einen Btree aus allen Files"""
 2303:         self.v_files = OOBTree()
 2304:         self.v_files_lastVersion = OOBTree()
 2305:         self.v_file_ids = Set()
 2306:         
 2307:         for x in self.CDLICatalog.searchResults():
 2308:             
 2309:             self.v_files.update({x.getId:x.getObject()})
 2310:             self.v_files_lastVersion.update({x.getId:x.getObject().getLastVersion()})
 2311:             self.v_file_ids.add(x.getId)
 2312:             logging.debug("add:"+x.getId+"XXX"+repr(x.getObject()))
 2313:         
 2314:         return True
 2315:     
 2316:     
 2317:     def updateOrAddToFileBTree(self,obj):
 2318:         """update a BTree"""
 2319:         self.v_files.update({obj.getId():obj})
 2320:         self.v_files_lastVersion.update({obj.getId():obj.getLastVersion()})
 2321:         
 2322:         self.v_file_ids.add(obj.getId())
 2323:         logging.debug("update:"+obj.getId()+"XXX"+repr(obj))
 2324:         
 2325:     def deleteFromBTree(self,objId):
 2326:         """delete an obj"""
 2327:         self.v_files.pop(objId)
 2328:         self.v_files_lastVersion.pop(objId)
 2329:         self.v_file_ids.remove(objId)
 2330:         
 2331: 
 2332:  
 2333:     def deleteFiles(self,ids):
 2334:         """delete files"""
 2335:         for id in ids:
 2336:             founds=self.CDLICatalog.search({'title':id.split(".")[0]})
 2337:             if founds:
 2338:                 logging.debug("deleting %s"%founds)
 2339:                 folder=founds[0].getObject().aq_parent #get the parent folder of the object
 2340:                 logging.debug("deleting from %s"%folder)
 2341:                 cut=folder.delete([founds[0].getId]) #cut it out
 2342: 
 2343: 
 2344: 
 2345:     def searchText(self, query, index='graphemes'):
 2346:         """searches query in the fulltext index and returns a list of file ids/P-numbers"""
 2347:         # see also: http://www.plope.com/Books/2_7Edition/SearchingZCatalog.stx#2-13
 2348:         logging.debug("searchtext for '%s' in index %s"%(query,index))
 2349:         #import Products.ZCTextIndex.QueryParser
 2350:         #qp = QueryParser.QueryParser()
 2351:         #logging.debug()
 2352:         idxQuery = {index:{'query':query}}
 2353:         idx = getattr(self, self.file_catalog)
 2354:         # do search
 2355:         resultset = idx.search(query_request=idxQuery,sort_index='textid')
 2356:         # put only the P-Number in the result 
 2357:         results = [res.getId[:7] for res in resultset]
 2358:         logging.debug("searchtext: found %d texts"%len(results))
 2359:         return results
 2360: 
 2361: 
 2362:     def getFile(self, pnum):
 2363:         """get the translit file with the given pnum"""
 2364:         f = getattr(self, self.file_catalog).search({'textid':pnum})
 2365:         if not f:
 2366:             return ""
 2367:         
 2368:         return f[0].getObject().getData()
 2369:          
 2370: 
 2371:     def showFile(self,fileId,wholePage=False):
 2372:         """show a file
 2373:         @param fileId: P-Number of the document to be displayed
 2374:         """
 2375:         f=getattr(self, self.file_catalog).search({'textid':fileId})
 2376:         if not f:
 2377:             return ""
 2378:         
 2379:         if wholePage:
 2380:             logging.debug("show whole page")
 2381:             return f[0].getObject().getContentObject().view()
 2382:         else:
 2383:             return f[0].getObject().getLastVersionFormattedData()
 2384:     
 2385: 
 2386:     def showWordInFile(self,fileId,word,indexName='graphemes',regExp=False,):
 2387:         """get lines with word from FileId"""
 2388:         logging.debug("showwordinfile word='%s' index=%s file=%s"%(word,indexName,fileId)) 
 2389:         
 2390:         file = formatAtfFullLineNum(self.getFile(fileId))
 2391:         ret=[]
 2392:         
 2393:         # add whitespace before and whitespace and line-end to splitter bounds expressions
 2394:         bounds = self.splitter[indexName].bounds
 2395:         splitexp = "(%s|\s)(%%s)(%s|\s|\Z)"%(bounds,bounds)
 2396:         # clean word expression 
 2397:         # TODO: this should use QueryParser itself
 2398:         # take out double quotes
 2399:         word = word.replace('"','')
 2400:         # take out ignorable signs
 2401:         ignorable = self.splitter[indexName].ignorex
 2402:         word = ignorable.sub('', word)
 2403:         # compile into regexp objects and escape parens
 2404:         wordlist = [re.compile(splitexp%re.escape(w)) for w in word.split(' ')]
 2405:             
 2406:         for line in file.splitlines():
 2407:             for word in wordlist:
 2408:                 #logging.debug("showwordinfile: searching for %s in %s"%(word.pattern,ignoreable.sub('',line)))
 2409:                 if word.search(ignorable.sub('',line)):
 2410:                     line = formatAtfLineHtml(line)
 2411:                     ret.append(line)
 2412:                     break
 2413:                     
 2414:         return ret
 2415: 
 2416:     
 2417:     def showWordInFiles(self,fileIds,word,indexName='graphemes',regExp=False):
 2418:         """
 2419:         get lines with word from all ids in list FileIds.
 2420:         returns dict with id:lines pairs.
 2421:         """
 2422:         logging.debug("showwordinfiles word='%s' index=%s file=%s"%(word,indexName,fileIds))
 2423:         
 2424:         return dict([(id,self.showWordInFile(id, word, indexName, regExp)) for id in fileIds])
 2425:     
 2426: 
 2427:     def tagWordInFile(self,fileId,word,indexName='graphemes',regExp=False):
 2428:         """get text with word highlighted from FileId"""
 2429:         logging.debug("tagwordinfile word='%s' index=%s file=%s"%(word,indexName,fileId)) 
 2430:         
 2431:         file=self.getFile(fileId)
 2432:         tagStart=u'<span class="found">'
 2433:         tagEnd=u'</span>'
 2434:         tagStr=tagStart + u'%%s' + tagEnd
 2435:         ret=[]
 2436:         
 2437:         # add whitespace to splitter bounds expressions and compile into regexp object
 2438:         bounds = self.splitter[indexName].bounds
 2439:         wordsplit = re.compile("(%s|\s)"%bounds)
 2440:         # clean word expression 
 2441:         # TODO: this should use QueryParser itself
 2442:         word = word.replace('"','') # take out double quotes
 2443:         # take out ignoreable signs
 2444:         ignorable = self.splitter[indexName].ignorex
 2445:         word = ignorable.sub('', word)
 2446:         # split search terms by blanks
 2447:         words = word.split(' ')
 2448:         # split search terms again (for grapheme search with words)
 2449:         splitwords = dict(((w,self.splitter[indexName].process([w])) for w in words))
 2450:             
 2451:         for line in file.splitlines():
 2452:             line = unicodify(line)
 2453:             # ignore lemma and other lines
 2454:             if line.lstrip().startswith('#lem:'):
 2455:                 continue
 2456:             # ignore p-num line
 2457:             if line.startswith('&P'):
 2458:                 continue
 2459:             # ignore version lines
 2460:             if line.startswith('#version'):
 2461:                 continue
 2462:             # ignore atf type lines
 2463:             if line.startswith('#atf:'):
 2464:                 continue
 2465: 
 2466:             # first scan
 2467:             hitwords = []
 2468:             for w in words:
 2469:                 if ignorable.sub('',line).find(w) > -1:
 2470:                     # word is in line
 2471:                     # append split word for grapheme search with words
 2472:                     hitwords.extend(splitwords[w])
 2473:                     #hitwords.extend(wordsplit.split(w))
 2474:                    
 2475:             # examine hits closer
 2476:             if hitwords:
 2477:                 # split line into words
 2478:                 parts = wordsplit.split(line)
 2479:                 line = ""
 2480:                 for p in parts:
 2481:                     #logging.debug("tagwordinfile: searching for %s in %s"%(p,hitwords))
 2482:                     # reassemble line
 2483:                     if ignorable.sub('', p) in hitwords:
 2484:                         #logging.debug("tagwordinfile: found %s in %s"%(p,hitwords))
 2485:                         # this part was found
 2486:                         line += tagStart + formatAtfHtml(p) + tagEnd
 2487:                     else:
 2488:                         line += formatAtfHtml(p)
 2489:                 
 2490:             else:
 2491:                 # no hits
 2492:                 line = formatAtfHtml(line)
 2493:             
 2494:             ret.append(line)
 2495:                         
 2496:         return u'<br>\n'.join(ret)
 2497: 
 2498: 
 2499: 
 2500:     def tagWordInFiles(self,fileIds,word,indexName='graphemes',regExp=False):
 2501:         """
 2502:         get texts with highlighted word from all ids in list FileIds.
 2503:         returns dict with id:text pairs.
 2504:         """
 2505:         logging.debug("tagwordinfiles word='%s' index=%s file=%s"%(word,indexName,fileIds)) 
 2506:         return dict([(id,self.tagWordInFile(id, word, indexName, regExp)) for id in fileIds])
 2507:     
 2508: 
 2509:     def getFileVersionList(self, pnum):
 2510:         """get the version history as a list for the translit file with the given pnum"""
 2511:         f = getattr(self, self.file_catalog).search({'textid':pnum})
 2512:         if not f:
 2513:             return []
 2514:         
 2515:         return f[0].getObject().getVersionList()
 2516:          
 2517: 
 2518:     def URLquote(self,str):
 2519:         """quote url"""
 2520:         return urllib.quote(str)
 2521:     
 2522:     def URLunquote(self,str):
 2523:         """unquote url"""
 2524:         return urllib.unquote(str)
 2525:     
 2526:     def URLquote_plus(self,str):
 2527:         """quote url"""
 2528:         return urllib.quote_plus(str)
 2529:     
 2530:     def URLunquote_plus(self,str):
 2531:         """unquote url"""
 2532:         return urllib.unquote_plus(str)
 2533:     
 2534:     
 2535:     def forceunlock(self):
 2536:         "break all locks"
 2537:         ret=[]
 2538:         for f in self.ZopeFind(self,obj_metatypes="CDLI file",search_sub=1):
 2539:            un=f[1].forceunlock()
 2540: 
 2541:            if un and un !="":
 2542:                ret.append((f[0],un))
 2543: 
 2544:         return ret
 2545:                                         
 2546: 
 2547:     def getChangesByAuthor(self,author,n=100):
 2548:         """getChangesByAuthor"""
 2549:         zcat=self.CDLIObjectsCatalog
 2550:         res=zcat({'lastEditor':author,
 2551:                      'sort_on':'getTime',
 2552:                      'sort_order':'descending',
 2553:                      'sort_limit':n})[:n ]
 2554:                        
 2555:         return res
 2556:     
 2557:     def getChangesByAuthor_html(self,author,n=100):
 2558:         """html output for changes by author"""
 2559:         tmp={}
 2560:         list=[]                         
 2561:         for x in self.getChangesByAuthor(author):
 2562:            nr=x.getObject().getVersionNumber()
 2563:            id=x.getObject().aq_parent.getId()
 2564:            #hinzufuegen, wenn Version neuer als die 
 2565:            if tmp.get(id,(0,0))[1] < nr:
 2566:                 tmp[id]=(x.getObject().aq_parent,nr)
 2567: 
 2568:      
 2569:         return self.cdli_main.findObjectsFromListWithVersion(list=tmp.values(),author=author)           
 2570:         
 2571:     def getLastChanges(self,n=100):
 2572:         """get the last n changes""" 
 2573:         n=int(n)                   
 2574:         zcat=self.CDLICatalog
 2575:         return zcat({'sort_on':'getLastChangeDate',
 2576:                      'sort_order':'descending',
 2577:                      'sort_limit':n})[:n ]
 2578:      
 2579:     
 2580:     def getLastChanges_html(self,n=100):
 2581:         """get the last n changes"""
 2582:         list = [x.getId for x in self.getLastChanges(n)]
 2583:         return self.cdli_main.findObjectsFromList(list=list,display=True)
 2584:                                        
 2585:     def refreshTxt(self,txt="",threadName=None):
 2586:         """txt fuer refresh"""
 2587:   
 2588:         return """ 2;url=%s?repeat=%s """%(self.absolute_url()+txt,threadName)
 2589: 
 2590:     
 2591:     def getResult(self,threadName=None):
 2592:        """result of thread"""
 2593:        try:
 2594:         return self._v_uploadATF[threadName].getResult()
 2595:        except:
 2596:         return "One moment, please"
 2597:     
 2598:         
 2599:     def checkThreads(self):
 2600:         """check threads"""
 2601:         ret="<html><body>"
 2602:         for thread in threading.enumerate():
 2603:            ret+="<p>%s (%s): %s</p>"%(repr(thread),thread.getName(),thread.isAlive())
 2604:        
 2605:         return ret
 2606:                                        
 2607:                                            
 2608:     def uploadATFRPC(self,data,username):
 2609:         """upload an atffile via xml-rpc"""
 2610:         uploader=uploadATFThread()
 2611:         
 2612:         #generate an random id for the upload object
 2613:         from random import randint
 2614:         if (not self.REQUEST.SESSION.get('idTmp',None)):
 2615: 
 2616:             idTmp=str(randint(0,1000000000))
 2617:             self.REQUEST.SESSION['idTmp']=idTmp
 2618:         else:
 2619:             idTmp=self.REQUEST.SESSION.get('idTmp',None)
 2620:             
 2621:         
 2622:         uploader.set(data,0,username,idTmp)
 2623:         
 2624:         stObj=uploader.run()
 2625:         
 2626:         processor=uploadATFfinallyThread()
 2627:         
 2628:         basketname=stObj.returnValue['basketNameFromFile']
 2629:         
 2630:         processor.set("uploadchanged",basketname=basketname,SESSION=stObj.returnValue,username=username,serverport=self.REQUEST['SERVER_PORT'])
 2631:         
 2632:         processor.run()
 2633:         
 2634:         
 2635:         return generateXMLReturn(stObj.returnValue)
 2636:         
 2637:     def uploadATF(self,repeat=None,upload=None,basketId=0,RESPONSE=None):
 2638:         """upload an atf file / basket file"""
 2639:         #self._v_uploadATF.returnValue=None
 2640:         
 2641:         #generate an random id for the upload thread
 2642:         from random import randint
 2643:         if (not self.REQUEST.SESSION.get('idTmp',None)):
 2644: 
 2645:             idTmp=str(randint(0,1000000000))
 2646:             self.REQUEST.SESSION['idTmp']=idTmp
 2647:         else:
 2648:             idTmp=self.REQUEST.SESSION.get('idTmp',None)
 2649:             
 2650:     
 2651:         threadName=repeat
 2652:         if not threadName or threadName=="":
 2653:             #new thread not called from the waiting page
 2654:             tmpVar=False
 2655:        
 2656:             thread=uploadATFThread()
 2657:             threadName=thread.getName()[0:]                                
 2658:             if (not hasattr(self,'_v_uploadATF')):
 2659:                    self._v_uploadATF={}
 2660:                                        
 2661:             self._v_uploadATF[threadName]=thread
 2662:             #self._xmltrans.start()
 2663:             #thread=Thread(target=self._v_uploadATF)
 2664:             logging.info("set thread. extern")
 2665:             self._v_uploadATF[threadName].set(upload,basketId,self.REQUEST['AUTHENTICATED_USER'],idTmp,serverport=self.REQUEST['SERVER_PORT'])
 2666:             #thread.start()
 2667:             logging.info("start thread. extern")
 2668:             self._v_uploadATF[threadName].start()
 2669: 
 2670:             
 2671:             self.threadName=self._v_uploadATF[threadName].getName()[0:]
 2672:             wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
 2673: 
 2674:             if wait_template:
 2675:                 return wait_template[0][1]()
 2676:             pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self)
 2677:             return pt(txt='/uploadATF',threadName=threadName)
 2678:             #_v_xmltrans.run()
 2679:             
 2680:         else:
 2681:             #recover thread, if lost
 2682:             if (not hasattr(self,'_v_uploadATF')):
 2683:                self._v_uploadATF={}
 2684:             if not self._v_uploadATF.get(threadName,None):
 2685:                  for thread in threading.enumerate():
 2686:                          if threadName == thread.getName():
 2687:                                        self._v_uploadATF[threadName]=thread
 2688:                                        
 2689:             if self._v_uploadATF.get(threadName,None) and (not self._v_uploadATF[threadName].returnValue):
 2690:         
 2691: 
 2692:                 wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
 2693:                 if wait_template:
 2694:                         return wait_template[0][1]()
 2695:                 
 2696:                 pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self)
 2697: 
 2698:                 return pt(txt='/uploadATF',threadName=threadName)
 2699:                 
 2700:             else:
 2701:                 tmp=getattr(self.temp_folder,idTmp).returnValue
 2702:  
 2703:                 pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadCheck.zpt')).__of__(self)
 2704: 
 2705:                 return pt(changed=tmp['changed'],lockerrors=tmp['lockerrors'],errors=tmp['errors'],dir=tmp['dir'],newPs=tmp['newPs'],basketLen=tmp['basketLen'],numberOfFiles=tmp['numberOfFiles'],
 2706:                   basketNameFromId=tmp['basketNameFromId'],basketNameFromFile=tmp['basketNameFromFile'],basketId=tmp['basketId'])
 2707:                      
 2708:     def redoUpload(self,threadName):
 2709:        """redo the upload"""
 2710:        tmp=self.cdli_main.tmpStore2[threadName]
 2711:        pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadCheck.zpt')).__of__(self)
 2712:        return pt(changed=tmp['changed'],lockerrors=tmp['lockerrors'],errors=tmp['errors'],dir=tmp['dir'],newPs=tmp['newPs'],basketLen=tmp['basketLen'],numberOfFiles=tmp['numberOfFiles'],
 2713:                   basketNameFromId=tmp['basketNameFromId'],basketNameFromFile=tmp['basketNameFromFile'],basketId=tmp['basketId'])
 2714:                  
 2715:     def uploadATFfinally(self,procedure='',comment="",basketname='',unlock=None,repeat=None,RESPONSE=None):
 2716:         """nowupload the files"""
 2717:        
 2718:        
 2719:        
 2720:         threadName=repeat
 2721:         if not threadName or threadName=="":
 2722:             thread=uploadATFfinallyThread()
 2723:             threadName=thread.getName()[0:]
 2724: 
 2725:             if (not hasattr(self,'_v_uploadATF')):
 2726:                                 self._v_uploadATF={}
 2727: 
 2728: 
 2729:             self._v_uploadATF[threadName]=thread
 2730: 
 2731:             idTmp=self.REQUEST.SESSION['idTmp']
 2732:             stObj=getattr(self.temp_folder,idTmp)
 2733:             self._v_uploadATF[threadName].set(procedure,comment=comment,basketname=basketname,unlock=unlock,SESSION=stObj.returnValue,username=self.REQUEST['AUTHENTICATED_USER'],serverport=self.REQUEST['SERVER_PORT'])
 2734: 
 2735:             self._v_uploadATF[threadName].start()
 2736: 
 2737:             
 2738:             
 2739:             wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
 2740: 
 2741:             if wait_template:
 2742:                 return wait_template[0][1]()
 2743:             pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self)
 2744: 
 2745:             return pt(txt='/uploadATFfinally',threadName=threadName)
 2746:             #_v_xmltrans.run()
 2747:         
 2748:         else:
 2749:             #recover thread, if lost
 2750:             if not hasattr(self,'_v_uploadATF'):
 2751:                self._v_uploadATF={}
 2752:             if not self._v_uploadATF.get(threadName,None):
 2753:                  for thread in threading.enumerate():
 2754:                          if threadName == thread.getName():
 2755:                                        self._v_uploadATF[threadName]=thread
 2756:                                        
 2757:             if self._v_uploadATF.get(threadName,None) and (self._v_uploadATF[threadName] is not None) and (not self._v_uploadATF[threadName].end) :
 2758: 
 2759:                 wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
 2760:                 if wait_template:
 2761:                         return wait_template[0][1]()
 2762:                 
 2763:                 pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self)
 2764:                 return pt(txt='/uploadATFfinally',threadName=threadName)
 2765:             else:
 2766:               
 2767:              
 2768:               idTmp=self.REQUEST.SESSION['idTmp']
 2769:               stObj=getattr(self.temp_folder,idTmp) 
 2770:               self.REQUEST.SESSION['idTmp']=None
 2771:              
 2772:               #update changed
 2773:               logging.debug("dir:"+repr(stObj.returnValue['changed']))
 2774:               for x in stObj.returnValue['changed']:
 2775:                     ob=self.CDLICatalog.search({'title':x[0]})
 2776:                    
 2777:                     self.cdliRoot.updateOrAddToFileBTree(ob[0].getObject())
 2778:               if RESPONSE is not None:
 2779:                   RESPONSE.redirect(self.absolute_url())
 2780: 
 2781:     def importFiles(self,comment="",author="" ,folderName="/Users/dwinter/atf", files=None,ext=None):
 2782:         """import files"""
 2783:         logging.debug("importFiles folderName=%s files=%s ext=%s"%(folderName,files,ext))
 2784:         root=self.cdli_main
 2785:         count=0
 2786:         if not files:
 2787:             files=os.listdir(folderName)
 2788:             
 2789:         for f in files:
 2790:             folder=f[0:3]
 2791:             f2=f[0:5]
 2792:             
 2793:             #check if main folder PXX already exists
 2794:             obj=self.ZopeFind(root,obj_ids=[folder])
 2795:             logging.debug("importFiles: folder=%s f2=%s obj=%s"%(folder,f2,obj)) 
 2796:             if ext:
 2797:                 ext.result="<p>adding: %s </p>"%f+ext.result
 2798: 
 2799:             
 2800:             if not obj: # if not create it
 2801:                 manage_addCDLIFileFolder(root,folder,folder)
 2802:                 fobj=getattr(root,folder)
 2803:                 #transaction.get().commit()                           
 2804: 
 2805:             else:
 2806:                 fobj=obj[0][1]
 2807:             
 2808:             #check IF PYYYYY already exist
 2809:             obj2=fobj.ZopeFind(fobj,obj_ids=[f2])
 2810:             logging.debug("importFiles: fobj=%s obj2=%s"%(fobj,obj2)) 
 2811:         
 2812:             if not obj2:# if not create it
 2813:                 manage_addCDLIFileFolder(fobj,f2,f2)
 2814:                 fobj2=getattr(fobj,f2)
 2815:         
 2816:             else:
 2817:                 fobj2=obj2[0][1]
 2818:               
 2819:             # not add the file
 2820:             file2=os.path.join(folderName,f)  
 2821:             id=f
 2822:             logging.debug("importFiles: addCDLIFile fobj2=%s, f=%s file2=%s"%(fobj2,repr(f),repr(file2)))
 2823:             fobj2.addFile(vC='',file=file(file2),author=author,newName=f)
 2824:             count+=1
 2825:             
 2826:             #now add the file to the storage
 2827:             ob = getattr(fobj2,f)
 2828:             self.cdliRoot.updateOrAddToFileBTree(ob)
 2829:             
 2830:             if count%100==0:
 2831:                 logging.debug("importfiles: committing")
 2832:                 transaction.get().commit()
 2833: 
 2834:         transaction.get().commit()
 2835:         return "ok"
 2836:          
 2837: 
 2838: manage_addCDLIRootForm=DTMLFile('dtml/rootAdd', globals())
 2839: 
 2840:     
 2841: def manage_addCDLIRoot(self, id, title='',
 2842:                      createPublic=0,
 2843:                      createUserF=0,
 2844:                      REQUEST=None):
 2845:     """Add a new Folder object with id *id*.
 2846: 
 2847:     If the 'createPublic' and 'createUserF' parameters are set to any true
 2848:     value, an 'index_html' and a 'UserFolder' objects are created respectively
 2849:     in the new folder.
 2850:     """
 2851:     ob=CDLIRoot()
 2852:     ob.id=str(id)
 2853:     ob.title=title
 2854:     try:
 2855:         self._setObject(id, ob)
 2856:     except:
 2857:         pass
 2858:     ob=self._getOb(id)
 2859: 
 2860:     checkPermission=getSecurityManager().checkPermission
 2861: 
 2862:     if createUserF:
 2863:         if not checkPermission('Add User Folders', ob):
 2864:             raise Unauthorized, (
 2865:                   'You are not authorized to add User Folders.'
 2866:                   )
 2867:         ob.manage_addUserFolder()
 2868: 
 2869:   
 2870:     if REQUEST is not None:
 2871:         return self.manage_main(self, REQUEST, update_menu=1)    
 2872: