--- cdli/cdli_files.py 2008/10/08 14:35:56 1.95
+++ cdli/cdli_files.py 2011/07/20 09:33:35 1.119
@@ -1,10 +1,8 @@
"""CDLI extensions of the filearchive"""
from Products.versionedFile.extVersionedFile import *
from Products.ZCatalog.CatalogPathAwareness import CatalogAware
-from tempfile import mkstemp,mkdtemp
import os.path
import os
-from types import *
import urlparse
import urllib
import cgi
@@ -21,1605 +19,23 @@ from ZPublisher.HTTPRequest import HTTPR
from ZPublisher.HTTPResponse import HTTPResponse
from ZPublisher.BaseRequest import RequestContainer
import threading
-from BTrees.OOBTree import OOBTree, OOTreeSet
import logging
import transaction
import copy
import codecs
import sys
from BTrees.IOBTree import IOBTree
+from BTrees.OOBTree import OOBTree
import cdliSplitter
from sets import Set
import md5
from DownloadBasket import DownloadBasketFinallyThread
-
-def makelist(mySet):
- x = list(mySet)
- x.sort()
- return x
-
-def unicodify(s):
- """decode str (utf-8 or latin-1 representation) into unicode object"""
- if not s:
- return u""
- if isinstance(s, str):
- try:
- return s.decode('utf-8')
- except:
- return s.decode('latin-1')
- else:
- return s
-
-def utf8ify(s):
- """encode unicode object or string into byte string in utf-8 representation.
- assumes string objects to be utf-8"""
- if not s:
- return ""
- if isinstance(s, str):
- return s
- else:
- return s.encode('utf-8')
-
-def formatAtfHtml(l):
- """escape special ATF characters for HTML"""
- if not l:
- return ""
-
- # replace &
- l = l.replace('&','&')
- # replace angular brackets
- l = l.replace('<','<')
- l = l.replace('>','>')
- return l
-
-def formatAtfLineHtml(l, nolemma=True):
- """format ATF line for HTML"""
- if not l:
- return ""
-
- if nolemma:
- # ignore lemma lines
- if l.lstrip().startswith('#lem:'):
- return ""
-
- return formatAtfHtml(l)
-
-
-
-def formatAtfFullLineNum(txt, nolemma=True):
- """format full line numbers in ATF text"""
- # surface codes
- surfaces = {'@obverse':'obv',
- '@reverse':'rev',
- '@surface':'surface',
- '@edge':'edge',
- '@left':'left',
- '@right':'right',
- '@top':'top',
- '@bottom':'bottom',
- '@face':'face',
- '@seal':'seal'}
-
- if not txt:
- return ""
-
- ret = []
- surf = ""
- col = ""
- for line in txt.splitlines():
- line = unicodify(line)
- if line and line[0] == '@':
- # surface or column
- words = line.split(' ')
- if words[0] in surfaces:
- surf = line.replace(words[0],surfaces[words[0]]).strip()
-
- elif words[0] == '@column':
- col = ' '.join(words[1:])
-
- elif line and line[0] in '123456789':
- # ordinary line -> add line number
- line = "%s:%s:%s"%(surf,col,line)
-
- ret.append(line)
-
- return '\n'.join(ret)
-
-
-def generateXMLReturn(hash):
- """erzeugt das xml file als returnwert fuer uploadATFRPC"""
-
- ret=""
-
- ret+=""
- for error in hash['errors']:
- ret+="""%s"""%error
-
- ret+=""
-
- ret+=""
- for changed in hash['changed']:
- ret+="""%s"""%changed
- ret+=""
-
- ret+=""
- for new in hash['newPs']:
- ret+=""""""%new
- ret+=""
-
- ret+=""
- return ret
-
-
-def unique(s):
- """Return a list of the elements in s, but without duplicates.
-
- For example, unique([1,2,3,1,2,3]) is some permutation of [1,2,3],
- unique("abcabc") some permutation of ["a", "b", "c"], and
- unique(([1, 2], [2, 3], [1, 2])) some permutation of
- [[2, 3], [1, 2]].
-
- For best speed, all sequence elements should be hashable. Then
- unique() will usually work in linear time.
-
- If not possible, the sequence elements should enjoy a total
- ordering, and if list(s).sort() doesn't raise TypeError it's
- assumed that they do enjoy a total ordering. Then unique() will
- usually work in O(N*log2(N)) time.
-
- If that's not possible either, the sequence elements must support
- equality-testing. Then unique() will usually work in quadratic
- time.
- (from the python cookbook)
- """
-
- n = len(s)
- if n == 0:
- return []
-
- # Try using a dict first, as that's the fastest and will usually
- # work. If it doesn't work, it will usually fail quickly, so it
- # usually doesn't cost much to *try* it. It requires that all the
- # sequence elements be hashable, and support equality comparison.
- u = {}
- try:
- for x in s:
- u[x] = 1
- except TypeError:
- del u # move on to the next method
- else:
- return u.keys()
-
- # We can't hash all the elements. Second fastest is to sort,
- # which brings the equal elements together; then duplicates are
- # easy to weed out in a single pass.
- # NOTE: Python's list.sort() was designed to be efficient in the
- # presence of many duplicate elements. This isn't true of all
- # sort functions in all languages or libraries, so this approach
- # is more effective in Python than it may be elsewhere.
- try:
- t = list(s)
- t.sort()
- except TypeError:
- del t # move on to the next method
- else:
- assert n > 0
- last = t[0]
- lasti = i = 1
- while i < n:
- if t[i] != last:
- t[lasti] = last = t[i]
- lasti += 1
- i += 1
- return t[:lasti]
-
- # Brute force is all that's left.
- u = []
- for x in s:
- if x not in u:
- u.append(x)
- return u
-
-
-class BasketContent(SimpleItem):
- """classe fuer den Inhalt eines Baskets"""
-
- def __init__(self,content=[]):
- """content"""
- self.contentList=content[0:]
-
- def getContent(self):
- """get content"""
-
- return self.contentList
-
- def setContent(self,content):
- self.contentList=content[0:]
-
- def numberOfItems(self):
- """number"""
-
- return len(self.getContent())
-
-
-class uploadATFfinallyThread(Thread):
- """class for adding uploaded filed (temporarily stored in the staging area at /tmp"""
-
- def __init__(self):
- """init for uploadATFfinallyThread"""
- self.continueVar=True
- self.returnValue=None
- self.end=False
- Thread.__init__(self)
-
- def set(self,procedure,comment="",basketname='',unlock=None,SESSION=None,username=None,serverport="8080"):
- """set start values for the thread"""
- self.procedure=procedure
- self.comment=comment
- self.basketname=basketname
- self.unlock=unlock
- self.SESSION=SESSION
- self.username=username
- self.serverport=serverport
-
-
- def __call__(self):
- """call of the thread (equals run)"""
- self.run()
- return True
-
- def getContext(self, app,serverport="8080"):
- """get the context within the ZODB"""
-
- resp = HTTPResponse(stdout=None)
- env = {
- 'SERVER_NAME':'localhost',
- 'SERVER_PORT':serverport,
- 'REQUEST_METHOD':'GET'
- }
- req = HTTPRequest(None, env, resp)
- return app.__of__(RequestContainer(REQUEST = req))
-
-
- def run(self):
- """run"""
-
- self.result=""
- #find context within ZODB
- from Zope import DB
- conn = DB.open()
- root = conn.root()
- app = root['Application']
- ctx = self.getContext(app,serverport=self.serverport)
-
- #add the files
- self.uploadATFfinallyThread(ctx,self.procedure,comment=self.comment,basketname=self.basketname,unlock=self.unlock,SESSION=self.SESSION,username=self.username)
- #commit the transactions
- transaction.get().commit()
- conn.close()
- #set flag for end of this method
- self.end=True
- logging.info("ended")
- return True
-
- def __del__(self):
- """delete"""
-
-
-
- def getResult(self):
- """method for accessing result"""
-
- return self.result
-
- def uploadATFfinallyThread(self,ctx,procedure,comment="",basketname='',unlock=None,RESPONSE=None,SESSION=None,username=None):
- """upload the files"""
- #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot
- ctx2=ctx.cdliRoot
-
- self.result+="
Start processing
"
-
- #shall I only upload the changed files?
- logging.debug("uploadATFfinally procedure: %s"%procedure)
- if procedure=="uploadchanged":
- changed=[x[0] for x in SESSION.get('changed',[])]
- uploadFns=changed+SESSION.get('newPs',[])
-
- #or all
- elif procedure=="uploadAll":
- uploadFns=[]
- for x in os.listdir(SESSION['tmpdir']):
- if not x in SESSION['lockerrors']:
- uploadFns.append(x)
-
- #or maybe nothing
- elif procedure=="noupload":
- return True
- else:
- uploadFns=[]
-
- #do first the changed files
- i=0
- for fn in uploadFns:
- logging.debug("uploadATFfinally uploadFn=%s"%fn)
- i+=1
- founds=ctx2.CDLICatalog.search({'title':fn})
- if len(founds)>0:
- SESSION['author']=str(username)
- self.result="Changing : %s"%fn+self.result
- logging.debug("uploadatffinallythread changing:%s"%fn+self.result)
- founds[0].getObject().manage_addCDLIFileObject('',comment,SESSION['author'],file=os.path.join(SESSION['tmpdir'],fn),from_tmp=True)
- if i%200==0:
- transaction.get().commit()
- logging.debug("uploadatffinallythread changing: do commit")
-
- transaction.get().commit()
- logging.debug("uploadatffinallythread changing: last commit")
-
- #now add the new files
- newPs=SESSION['newPs']
- if len(newPs)>0:
- tmpDir=SESSION['tmpdir']
- logging.debug("uploadatffinallythread adding start")
- self.result="
Adding files
"+self.result
- #TODO: make this configurable, at the moment base folder for the files has to be cdli_main
- ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self)
- logging.debug("uploadatffinallythread adding finished")
-
- #unlock locked files?
- if unlock:
- logging.debug("uploadatffinallythread unlocking start")
- self.result="Unlock files
"+self.result
- unlockFns=[]
- for x in os.listdir(SESSION['tmpdir']):
- if not x in SESSION['errors']:
- unlockFns.append(x)
-
- logging.debug("unlocking have now what to unlock")
-
- for fn in unlockFns:
- #logging.info("will unlock: %s"%fn)
- founds=ctx2.CDLICatalog.search({'title':fn})
- #logging.info("found it: %s"%repr(founds))
- if len(founds)>0:
- #logging.info("unlock: %s"%founds[0].getObject().getId())
- SESSION['author']=str(username)
- founds[0].getObject().lockedBy=""
-
- logging.debug("uploadatffinallythread unlocking done")
-
- #if a basketname is given, add files to the basket
- if not (basketname ==''):
- logging.debug("uploadatffinallythread add to basket %s"%basketname)
- self.result="Add to basket
"+self.result
- basketId=ctx2.basketContainer.getBasketIdfromName(basketname)
-
- if not basketId: # create new basket
- logging.debug("uploadatffinallythread create basket %s"%basketname)
- self.result="Create a new basket
"+self.result
- ob=ctx2.basketContainer.addBasket(basketname)
- basketId=ob.getId()
- basket=getattr(ctx2.basketContainer,str(basketId))
- ids=os.listdir(SESSION['tmpdir'])
- #logging.debug("should add:"+repr(ids))
- basket.addObjects(ids,deleteOld=True,username=str(username))
-
- logging.debug("uploadatffinallythread uploadfinally done")
-
- if RESPONSE is not None:
- RESPONSE.redirect(self.aq_parent.absolute_url())
-
- return True
-
-class tmpStore(SimpleItem):
- """simple item"""
- meta_type="cdli_upload"
-
- def __init__(self,id):
- """init tmp"""
- self.id=id
-
-class uploadATFThread(Thread):
- """class for checking the files befor uploading"""
-
- def __init__(self):
- """initialise"""
-
- self.continueVar=True
- self.returnValue=None
-
- Thread.__init__(self)
-
-
- def set(self,upload,basketId,username,idTmp,serverport="8080"):
- """set start values for the thread"""
- self.result=""
- self.upload=upload
- self.basketId=basketId
- self.username=username
- self.serverport=serverport
- self.idTmp=idTmp
-
- def __call__(self):
- """call method """
- self.run()
- return True
-
- def getContext(self, app,serverport="8080"):
- """get the context within the ZODB"""
- resp = HTTPResponse(stdout=None)
- env = {
- 'SERVER_NAME':'localhost',
- 'SERVER_PORT':serverport,
- 'REQUEST_METHOD':'GET'
- }
- req = HTTPRequest(None, env, resp)
- return app.__of__(RequestContainer(REQUEST = req))
-
- def run(self):
- idTmp=self.idTmp
- self.result=""
- #find context within ZODB
- from Zope import DB
- conn = DB.open()
- root = conn.root()
- app = root['Application']
- ctx = self.getContext(app,serverport=self.serverport)
- logging.info("run intern")
- try:
- logging.info("created: %s"%idTmp)
- ctx.temp_folder._setObject(idTmp,tmpStore(idTmp))
- except:
- logging.error("thread upload: %s %s"%sys.exc_info()[0:2])
-
- logging.info("call thread intern")
- self.uploadATFThread(ctx,self.upload,idTmp,self.basketId)
-
- #ctx.cdliRoot.cdli_main.tmpStore2[self.getName()[0:]]=self.returnValue
-
-
- transaction.get().commit()
-
- conn.close()
-
- return getattr(ctx.temp_folder,idTmp)
-
- def getResult(self):
- """method for accessing result"""
- return self.result
-
- def uploadATFThread(self,ctx,upload,idTmp,basketId=0):
- """upload an atf file"""
- #TODO: add comments
- #TODO: finish uploadATF
-
- stObj=getattr(ctx.temp_folder,idTmp)
- logging.info("start, upload thread")
- self.result="I got your file, start now to split it into single atf-files!
"
-
- #make sure that id is a string and not an integer
- basketId=str(basketId)
-
- #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot
- ctx2=ctx.cdliRoot
-
- #get temporary file for staging the downloaded and splitted files
- dir=mkdtemp()
-
-
- changed=[] # changed files
- errors=[] # files with errors
- lockerrors=[] # files with errors
-
- newPs=[] # new p filed
- psNotInCatalog=[] # files not in the catalog
-
- #split the uploadedd atf file
- basketNameFromFile, numberOfFiles=splitatf(upload,dir,ext=self)
-
- #find basketId if not set
-
- #get active abaket
- if basketId == '0':
- basketObj=ctx2.basketContainer.getActiveBasket()
- if basketObj:
- basketId=basketObj.getId()
-
- #if there is no active basket and no basketid given, id is empty, else get besketname and length
- if basketId == '0':
- basketNameFromId=""
- basketLen=0
- else:
- basketNameFromId=getattr(ctx2.basketContainer,basketId).title
- basketLen=getattr(ctx2.basketContainer,basketId).getLastVersion().numberOfItems()
-
- logging.info("got the file, upload thread")
- self.result+="""
I got the files
<
- p>I am computing the differences to the exisiting files"""
-
- #start to check the files
- for fn in os.listdir(dir):
-
- self.result="process:%s
"%fn+self.result
-
- # check if file is in the catalog
- #TODO: checkCatalog is not implemented yet
- if ctx2.cdli_main.checkCatalog(fn):
- psNotInCatalog.append(fn)
-
- #check if p-file already at the server
- founds=ctx2.CDLICatalog.search({'title':fn})
-
- #if not than add filename to the list of newfiles
-
- data=file(os.path.join(dir,fn)).read()
- status,msg=checkFile(fn,data,dir)
- #status=True
-
-
- if not status: # error
- errors.append((fn,msg))
-
- else:
- if len(founds)==0:
- newPs.append(fn)
-
- #if p file alread at the server
- for found in founds:
- #analyse the differences to the actual file
- obj=found.getObject()
-
- if (not (str(obj.lockedBy))=='') and (not (str(obj.lockedBy)==str(self.username))):
- lockerrors.append((fn,str(obj.lockedBy)))
- else:
-
- diffs=obj.diff(data)
- if diffs[0]>0:
- changed.append((obj,diffs)) #hochladen
-
- #ready, set the returnValues
- self.result+="Done
"
-
- stObj.returnValue={}
-
- stObj.returnValue['errors']=errors
-
- stObj.returnValue['newPs']=newPs
- stObj.returnValue['tmpdir']=dir
- stObj.returnValue['basketLen']=basketLen
- stObj.returnValue['numberOfFiles']=numberOfFiles
- stObj.returnValue['basketNameFromId']=basketNameFromId
- stObj.returnValue['basketNameFromFile']=basketNameFromFile
- stObj.returnValue['basketId']=basketId
- stObj.returnValue['dir']=dir
- #stObj.returnValue['changed']=copy.copy(changed)
- stObj.returnValue['changed']=[(x[0].getId(),x[1][0]) for x in changed]
- #stObj.returnValue['lockerrors']=[x[0].getId() for x in lockerrors]
- stObj.returnValue['lockerrors']=[x for x in lockerrors]
- self.returnValue=True
- #ctx2.cdli_main.setTemp('v_uploadATF_returnValue',True)
-
-
-class CDLIBasketContainer(OrderedFolder):
- """contains the baskets"""
-
-
- security=ClassSecurityInfo()
- meta_type="CDLIBasketContainer"
-
- def getPNumbersOfBasket(self,basketName):
- """get all pnumbers of a basket as a list, returns an empty list if basket not found
- @param basketName: name of the basket
- """
- ret=[]
- basketId=self.getBasketIdfromName(basketName)
- if not basketId:
- return []
-
- ob=getattr(self,basketId).getContent()
-
- ret=[x[0].split(".")[0] for x in ob]
-
- return ret
-
- security.declareProtected('manage','getBasketAsOneFile')
- def getBasketAsOneFile(self,basketName,current="no"):
- """returns all files of the basket combined in one file
- @param basketName: Name of the basket
- @param current: (optional) if current is set to "yes" then the most current version of
- all files are downloaded and not the versions of the files as stored in the basket
- """
- ret=""
- basketId=self.getBasketIdfromName(basketName)
- if not basketId:
- return ""
-
- ob=getattr(self,basketId).getLastVersion()
- for object in ob.getContent():
- if current=="no": #version as they are in the basket
- ret+=str(object[0].getData())+"\n"
- elif current=="yes":
- #search current object
- #logging.debug("current: %s"%object[1].getId().split(".")[0])
- founds=self.CDLICatalog.search({'title':object[1].getId().split(".")[0]})
- if len(founds)>0:
- ret+=str(founds[0].getObject().getLastVersion().getData())+"\n"
- return ret
-
- security.declareProtected('manage','upDateBaskets')
- def upDateBaskets(self):
- """update content in to objects"""
-
- founds=self.ZopeFind(self,obj_metatypes=['CDLIBasketVersion'],search_sub=1)
-
- for found in founds:
- found[1].updateBasket()
-
- security.declareProtected('manage','deleteBaskets')
- def deleteBaskets(self,ids=None):
- """delete baskets, i.e. move them into trash folder"""
-
-
- found=self.ZopeFind(self,obj_ids=['trash'])
-
- if len(found)<1:
- manage_addFolder(self, 'trash')
- trash=self._getOb('trash')
- else:
- trash=found[0][1]
-
- if type(ids) is not ListType:
- ids=[ids]
- cut=self.manage_cutObjects(ids)
- trash.manage_pasteObjects(cut)
-
- security.declareProtected('manage','manageBaskets')
- def manageBaskets(self,submit,ids=None,basket1="",basket2="",joinBasket="",subtractBasket="",REQUEST=None,RESPONSE=None):
- """manage baskets, delete or copy"""
- if submit=="delete":
- self.deleteBaskets(ids)
-
- elif submit=="join":
- flag,msg=self.joinBasket(joinBasket, ids)
- logging.info("joining %s %s"%(flag,msg))
-
- elif submit=="subtract":
- logging.info("BBBb %s %s"%(basket1,basket2))
- flag,msg=self.subtractBasket(subtractBasket, basket1,basket2)
- logging.info("subtract %s %s"%(flag,msg))
-
- if RESPONSE:
- RESPONSE.redirect(self.absolute_url())
-
- security.declareProtected('View','getBasketIdfromName')
- def getBasketIdfromName(self,basketname):
- """get id from name"""
-
- for basket in self.ZopeFind(self,obj_metatypes=["CDLIBasket"]):
- if basket[1].title==basketname:
- return basket[0]
- else:
- None
-
- security.declareProtected('manage','uploadBasket_html')
-
- def uploadBasket_html(self,basketId='0'):
- """upload an atf file, html form"""
-
-
- basketId=str(basketId)
- if not basketId=='0':
- basketName=getattr(self.basketContainer,basketId).title
- else:
- basketName=""
-
- pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadBasket_html.zpt')).__of__(self)
- return pt(basketId=basketId,basketName=basketName)
-
-
- security.declareProtected('manage','index_html')
- def index_html(self):
- """stanadard ansicht"""
-
-
-
- ext=self.ZopeFind(self,obj_ids=["index.html"])
- if ext:
- return ext[0][1]()
-
- pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketContainerMain')).__of__(self)
- return pt()
-
- def getStorageFolderRoot(self):
- """root des storage folders"""
- return self.cdli_main
-
- def __init__(self,id,title):
- """ init basket container"""
- self.id=id
- self.title=title
-
-
- def getBasketsId(self):
- """get all baskets als klartext"""
-
- ret=""
- baskets=self.ZopeFind(self,obj_metatypes=['CDLIBasket'])
- for basket in baskets:
- com,user,time,values = basket[1].getContentIds()
- ret+= "BASKET:"+com+"\t"+user+"\t"+time+"\n"
- for x in values:
- ret+= x[0]+"\t"+x[1]+"\n"
- return ret
-
- def getBaskets(self,sortField='title'):
- """get all baskets files"""
-
- def sortName(x,y):
- return cmp(x[1].title.lower(),y[1].title.lower())
-
- def sortDate(x,y):
- return cmp(y[1].getLastVersion().getTime(),x[1].getLastVersion().getTime())
-
-
- def sortComment(x,y):
-
-
-
- try:
- xc=getattr(x[1],'comment','ZZZZZZZZZZZZZ').lower()
- except:
- xc='ZZZZZZZZZZZZZ'.lower()
- try:
- yc=getattr(y[1],'comment','ZZZZZZZZZZZZZ').lower()
- except:
- yc='ZZZZZZZZZZZZZ'.lower()
-
-
- if (xc=='') or (xc=='ZZZZZZZZZZZZZ'.lower()):
-
- try:
- xc=x[1].getLastVersion().getComment().lower()
- except:
- xc='ZZZZZZZZZZZZZ'.lower()
-
- if (yc=='') or (yc=='ZZZZZZZZZZZZZ'.lower()):
- try:
- yc=y[1].getLastVersion().getComment().lower()
- except:
- yc='ZZZZZZZZZZZZZ'.lower()
-
-
- return cmp(xc,yc)
-
- def sortAuthor(x,y):
-
- return cmp(x[1].getLastVersion().getUser().lower(),y[1].getLastVersion().getUser().lower())
-
- baskets=self.ZopeFind(self,obj_metatypes=['CDLIBasket'])
-
-
- if sortField=='title':
- baskets.sort(sortName)
- elif sortField=='date':
- baskets.sort(sortDate)
- elif sortField=='author':
- baskets.sort(sortAuthor)
- elif sortField=='comment':
- baskets.sort(sortComment)
-
- return baskets
-
-
- def subtractBasket(self,newBasket,basket1,basket2):
- """subtract basket2 from basket1
- (i.e. newbasket will contain alle elements of basket1 which are not in basket2),
- if basket2 contains files which are not in basket1, then theses files fill be ignored
-
- @param newbasket: name of the new basket
- @param basket1: basket where basket2 will be subtracted from
- @param basket2: see above
-
- """
- logging.info("CCCCC %s %s"%(basket1,basket2))
-
- try:
- newB=self.addBasket(newBasket)
- except:
- return False, "cannot create the new basket"
-
-
-
-
-
- bas2= getattr(self,basket2)
- bas2content=bas2.getContent()
- bas2ids=[x[0] for x in bas2content]
-
-
-
- bas1= getattr(self,basket1)
- bas1content=bas1.getContent()
-
-
- newBasketContent={}
-
- for id,version in bas1content:
- if not (id in bas2ids):
- newBasketContent[id]=version
-
- username=self.getActualUserName()
-
- logging.info("sbc %s"%newBasketContent)
- newB.addObjectsWithVersion(newBasketContent,username=username,catalog=self.CDLICatalog)
-
- return True, ""
-
-
- def joinBasket(self,newBasket,oldBaskets):
- """join two baskets
- @param newbasket: name of the new basket
- @param oldbaskets: list of baskets to be joined
- """
- try:
- newB=self.addBasket(newBasket)
- except:
- return False, "cannot create the new basket"
-
- newBasketContent={}
- for ob in oldBaskets:
- x= getattr(self,ob,None)
- if x is None:
- return False, "cannot find basket: %s"%ob
-
- ids=x.getContent() # hole den Inhalt
-
- for id,version in ids:
- if newBasketContent.has_key(id): # p number gibt's schon
- newBasketContent[id]=max(newBasketContent[id],version) # speichere die groessere Versionsnumber
- else:
- newBasketContent[id]=version
- username=self.getActualUserName()
-
- logging.info("nbc %s"%newBasketContent)
- newB.addObjectsWithVersion(newBasketContent,username=username,catalog=self.CDLICatalog)
-
- return True, ""
-
- def getNewId(self):
- """createIds"""
- last=getattr(self,'last',0)
- last +=1
- while len(self.ZopeFind(self,obj_ids=[str(last)]))>0:
- last+=1
-
- self.last=last
- return last
-
- def setActiveBasket(self,basketId,REQUEST=None):
- """store active basketId in a cookie"""
- self.REQUEST.RESPONSE.setCookie("CDLIActiveBasket",basketId,path="/")
- try:
- qs=cgi.parse_qs(REQUEST['QUERY_STRING'])
- del(qs['basketId'])
- except:
- qs={}
- if REQUEST:
- REQUEST.RESPONSE.redirect(REQUEST['URL1']+'?'+urllib.urlencode(qs))
-
- def getActiveBasket(self):
- """get active basket from cookie"""
-
- id= self.REQUEST.cookies.get('CDLIActiveBasket',None)
- if id:
- obj=getattr(self,str(id),None)
- else:
- obj=None
- return obj
-
- def getActualUserName(self):
- """get name of the actualuser"""
- return str(self.REQUEST['AUTHENTICATED_USER'])
-
- security.declareProtected('manage','addBasket')
- def addBasket(self,newBasketName):
- """add a new basket"""
-
- ob=manage_addCDLIBasket(self,newBasketName)
- return ob
-
- def storeInBasket(self,submit,ids=None,newBasketName=None,fromFileList=None,RESPONSE=None,REQUEST=None):
- """store it"""
- if not ids:
- ids=self.REQUEST.SESSION['fileIds']
-
- if (type(ids) is not ListType) and (not isinstance(ids,Set)):
- ids=[ids]
-
- if isinstance(ids,Set):
- ids=list(ids)
-
- if (submit.lower()=="store in new basket") or (submit.lower()=="new basket"):
- basketRet=self.addBasket(newBasketName)
- self.setActiveBasket(basketRet.getId())
- basket=getattr(self,basketRet.getId())
- elif (submit.lower()=="store in active basket") or (submit.lower()=="active basket"):
- basket=self.getActiveBasket()
-
- added=basket.addObjects(ids)
- back=self.REQUEST['HTTP_REFERER'].split("?")[0]+"?basketName="+basket.title+"&numberOfObjects="+str(added)
-
-
- if fromFileList:
-
- return self.cdli_main.findObjectsFromList(list=ids,basketName=basket.title,numberOfObjects=added)
-
- if RESPONSE:
-
- RESPONSE.redirect(back)
-
- return True
-
-def manage_addCDLIBasketContainerForm(self):
- """add the CDLIBasketContainer form"""
- pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIBasketContainer.zpt')).__of__(self)
- return pt()
-
-def manage_addCDLIBasketContainer(self,id,title,RESPONSE=None):
- """add the basket"""
- ob=CDLIBasketContainer(id,title)
-
- self._setObject(id, ob)
-
- if RESPONSE is not None:
- RESPONSE.redirect('manage_main')
-
-class CDLIBasket(Folder,CatalogAware):
- """basket"""
-
- meta_type="CDLIBasket"
- default_catalog="CDLIBasketCatalog"
-
- def searchInBasket(self,indexName,searchStr,regExp=False):
- """searchInBasket"""
-
- lst=self.searchInLineIndexDocs(indexName,searchStr,uniq=True,regExp=regExp) #TODO: fix this
- ret={}
-
- lv=self.getLastVersion()
-
-
- for obj in lv.content.getContent():
- id=obj[1].getId().split(".")[0]
- if id in lst:
-
- ret[id]=self.showWordInFile(id,searchStr,lineList=self.getLinesFromIndex(indexName,searchStr,id,regExp=regExp),regExp=regExp,indexName=indexName)
-
-
- pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','searchResultsInBasket')).__of__(self)
- return pt(result=ret,indexName=indexName,regExp=regExp,word=searchStr)
-
-
-
-
- def searchInBasket_v1(self,searchStr):
- """search occurences of searchStr in files im basket"""
- ret=[]
- lv=self.getLastVersion()
- logging.info("searching")
- for obj in lv.content.getContent():
- txt=obj[0].getData()
- for x in txt.split("\n"):
- logging.info("search %s"%x)
- if re.match(searchStr,x):
- ret.append(x)
-
- return "\n".join(ret)
-
-
- def getFile(self,obj):
- return obj[1]
-
- def getFileLastVersion(self,obj):
- return obj[0]
-
- def getFileNamesInLastVersion(self):
- """get content of the last version as list"""
-
- return [x[1].getId() for x in self.getLastVersion().getContent()]
-
-
- def isActual(self,obj):
- """teste ob im basket die aktuelle version ist"""
- try:
- #logging.debug("isActual:"+repr(obj))
- actualNo=obj[1].getLastVersion().getVersionNumber()
- storedNo=obj[0].getVersionNumber()
-
-
- #actualNo=self.getFileObjectLastVersion(obj.getId()).getVersionNumber()
-
- #if len(founds)>0 and founds[0].getObject().aq_parent.getId()==".trash":
- # return False, -1
-
- if actualNo==storedNo:
- return True , 0
- else:
- return False, actualNo
- except:
- logging.error( """is actual: %s (%s %s)"""%(repr(obj),sys.exc_info()[0],sys.exc_info()[1]))
-
- return False, -1
-
- def history(self):
- """history"""
-
- ext=self.ZopeFind(self.aq_parent,obj_ids=["history_template.html"])
- if ext:
- return getattr(self,ext[0][1].getId())()
-
- pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketHistory')).__of__(self)
- return pt()
-
- def getStorageFolderRoot(self):
- """root des storage folders"""
- return self.aq_parent.cdli_main
-
- def __init__(self,id,title,shortDescription="",comment=""):
- """init a CDLIBasket"""
-
- self.id=id
- self.title=title
- self.shortDescription=shortDescription
- self.comment=comment
-
- def getActualUserName(self):
- """get name of the actualuser"""
-
- return str(self.REQUEST['AUTHENTICATED_USER'])
-
-
- def getLastVersion(self):
- """hole letzte version"""
-
- ids=[]
- idsTmp= self.objectIds()
- for x in idsTmp:
- try:
- ids.append(int(x))
- except:
- pass
- ids.sort()
-
- if len(ids)==0:
- return None
- else:
- ob=getattr(self,str(ids[-1]))
-
-
- return ob
-
- def getVersions(self):
- """get versions"""
- versions=self.ZopeFind(self,obj_metatypes=["CDLIBasketVersion"])
- return versions
-
-
- def updateObjects(self,ids,RESPONSE=None,REQUEST=None):
- """update ids, ids not in the basket the add"""
- if type(ids) is not ListType:
- ids=[ids]
-
- lastVersion=self.getLastVersion()
- oldContent=lastVersion.content.getContent()
- newContent=[]
-
- #first copy the old
- for obj in oldContent:
- if obj[1].getId() not in ids:
- newContent.append(obj)
- #now add the new
-
- for id in ids:
- founds=self.CDLICatalog.search({'title':id})
-
- for found in founds:
- if found.getObject() not in oldContent:
- #TODO: was passiert wenn, man eine Object dazufgt, das schon da ist aber eine neuere version
- newContent.append((found.getObject().getLastVersion(),found.getObject()))
-
-
- content=newContent
- user=self.getActualUserName()
-
- ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=newContent)
-
- obj=self._getOb(ob.getId())
- if RESPONSE:
-
- RESPONSE.redirect(obj.absolute_url())
-
- return obj
-
- def addObjectsWithVersion(self,ids,deleteOld=None,username=None,catalog=None):
- """generate a new version of the basket with objects added,
- hier wird jedoch nicht die letzte Version jedes Files hinzugefuegt, s
- ondern ids is ein Tupel mit der Id (d.h. der p-number) und der Versionsnummer.
- """
- logging.info("add to basket (%s)"%(self.getId()))
- lastVersion=self.getLastVersion()
-
- if not catalog:
- catalog=self.CDLICatalog
-
- if lastVersion is None:
- oldContent=[]
- else:
- oldContent=lastVersion.content.getContent()
-
- if deleteOld:
- oldContent=[]
-
- newContent=[]
- added=0
-
- for id,version in ids.iteritems():
- logging.info("adding %s %s"%(id,version))
- id=id.split(".")[0] # title nur die pnumber ohne atf
-
- try:
- founds=catalog.search({'title':id})
- except:
- founds=[]
- logging.info(" found %s "%(founds))
- for found in founds:
- if found.getObject() not in oldContent:
-
- #TODO: was passiert wenn, man eine Object dazufgt, das schon da ist aber eine neuere version
- newContent.append((found.getObject().getVersions()[version-1][1],found.getObject()))
- added+=1
-
- content=oldContent+newContent
- if not username:
- logging.error("XXXXXXXXXXX %s"%repr(self))
- user=self.getActualUserName()
- else:
- user = username
-
- ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=content)
- logging.info("add to basket (%s) done"%(self.getId()))
- return added
-
-
- def addObjects(self,ids,deleteOld=None,username=None):
- """generate a new version of the basket with objects added"""
-
- def swap(x):
- return (x[1],x[0])
-
- logging.info("add to basket (%s)"%(self.getId()))
- lastVersion=self.getLastVersion()
-
- if lastVersion is None:
- oldContent=[]
- else:
- oldContent=lastVersion.content.getContent()
-
- if deleteOld:
- oldContent=[]
-
- added=0
-# for id in ids:
-# logging.debug("adding:"+id)
-# try:
-# founds=self.CDLICatalog.search({'title':id})
-# except:
-# founds=[]
-#
-# for found in founds:
-# if found.getObject() not in oldContent:
-# #TODO: was passiert wenn, man eine Object dazufgt, das schon da ist aber eine neuere version
-# newContent.append((found.getObject().getLastVersion(),found.getObject()))
-# added+=1
-
- hash = md5.new(repr(makelist(ids))).hexdigest() # erzeuge hash als identification
- #logging.debug("JJJJJJJ:"+repr(self.makelist(ids)))
-
+from types import *
+import pickle
+import tempfile
- if hasattr(self.cdliRoot,'v_tmpStore') and self.cdliRoot.v_tmpStore.has_key("hash"): #TODO: muss eigentlich self.cdliRoot.v_tmpStore.has_key(hash): heissen (ohne "), erstmal so gesetzt damit der hash hier nie benutzt wird
- logging.debug("from store!")
- newContent=Set(map(swap,self.cdliRoot.v_tmpStore[hash]))
-
- else:
- logging.debug("not from store!")
- newContent=Set([(self.getFileObjectLastVersion(x),self.getFileObject(x)) for x in ids])
-
-
- content=Set(oldContent).union(newContent)
- added = len(content)-len(oldContent)
- if not username:
- user=self.getActualUserName()
- else:
- user = username
-
- #logging.debug("content:"+repr(list(content)))
- ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=list(content))
- logging.info("add to basket (%s) done"%(self.getId()))
- return added
-
-
-
- def getContent(self):
- """print content"""
- ret=[]
-
- lv=self.getLastVersion()
- for obj in lv.content.getContent():
- #logging.info("XXXXXXXXXX %s"%repr(obj))
- ret.append((obj[1].getId(),obj[0].versionNumber))
-
- return ret
-
- def getContentIds(self):
- """print basket content"""
- ret=[]
- lv=self.getLastVersion()
- for obj in lv.content.getContent():
- ret.append((obj[0].getId(),obj[1].getId()))
-
-
- return lv.getComment(),lv.getUser(),lv.getTime(),ret
-
- def changeBasket(self,ids,submit,RESPONSE=None,REQUEST=None):
- """change a basket"""
- if submit=="update":
- return self.updateObjects(ids,RESPONSE=RESPONSE,REQUEST=REQUEST)
- elif submit=="delete":
- return self.deleteObjects(ids,RESPONSE=RESPONSE,REQUEST=REQUEST)
-
- def deleteObjects(self,ids,RESPONSE=None,REQUEST=None):
- """delete objects"""
-
- if type(ids) is not ListType:
- ids=[ids]
-
- lastVersion=self.getLastVersion()
- oldContent=lastVersion.content.getContent()
- newContent=[]
- for obj in oldContent:
- if obj[1].getId() not in ids:
- newContent.append(obj)
-
-
- user=self.getActualUserName()
-
- ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=newContent)
-
- if RESPONSE:
- obj=self._getOb(ob.getId())
- RESPONSE.redirect(obj.absolute_url())
-
-def manage_addCDLIBasketForm(self):
- """add the CDLIBasketContainer form"""
- pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIBasket.zpt')).__of__(self)
- return pt()
+from cdli_helpers import *
-def manage_addCDLIBasket(self,title,shortDescription="",comment="",RESPONSE=None):
- """add the basket"""
-
- id=str(self.getNewId())
-
- ob=CDLIBasket(id,title,shortDescription,comment)
-
- self._setObject(id, ob)
-
- if RESPONSE is not None:
- RESPONSE.redirect('manage_main')
- else:
- return ob
-
-class CDLIBasketVersion(Implicit,Persistent,Folder):
- """version of a basket"""
-
- meta_type="CDLIBasketVersion"
- security=ClassSecurityInfo()
-
- def updateBasket(self):
- """update"""
- try:
- self._setObject('content',BasketContent(self.basketContent))
- except:
- try:
- if len(self.basketContent)>0:
- self.content.setContent(self.basketContent)
- except:
- print "error",self.getId(),self.aq_parent.getId()
- self.basketContent=[]
-
-
- def containsNonActualFiles(self):
- """returns True if basket contains one or more non current files"""
-
- objs=self.getContent()
- for obj in objs:
- if not self.isActual(obj)[0]:
- return True
- return False
-
- def downloadListOfPnumbers(self):
- """download pnumbers of the basket as list"""
-
- basket_name=self.aq_parent.title
-
- ids=self.getContent() # get the list of objects
- logging.error(ids)
- ret="\n".join([x[1].getId().split(".")[0] for x in ids])
-
- self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.txt" """%basket_name)
- self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream")
- length=len(ret)
- self.REQUEST.RESPONSE.setHeader("Content-Length",length)
- self.REQUEST.RESPONSE.write(ret)
-
- security.declareProtected('manage','downloadObjectsAsOneFile')
- def downloadObjectsAsOneFile(self,lock=None,procedure=None,REQUEST=None,check="yes",current="no"):
- """download all selected files in one file"""
-
- if self.temp_folder.downloadCounterBaskets > 10000:
- return """I am sorry, currently the server has to many requests for downloads, please come back later!"""
-
-
- if (check=="yes") and self.containsNonActualFiles():
- pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadObjectAsOneFile_check.zpt')).__of__(self)
-
- return pt(lock=lock)
-
- else:
-
- return self.downloadObjectsAsOneFileFinally(lock=lock,procedure=procedure,REQUEST=REQUEST,current="no")
-
- def downloadObjectsAsOneFileFinally(self,lock=None,procedure=None,REQUEST=None,current="no",repeat=None):
- """print do the download"""
-
-
- ret=""
- lockedObjects={}
-
-
-
- if lock:
- logging.debug("------lock:"+repr(lock))
- if str(self.REQUEST['AUTHENTICATED_USER'])=='Anonymous User':
-
- return "please login first"
-
- #check if a locked object exist in the basket.
- lockedObjects={}
- for object in self.content.getContent():
-
- if (not str(object[1].lockedBy)=="") and (not (str(object[1].lockedBy)==str(self.REQUEST['AUTHENTICATED_USER']))):
- lockedObjects[object[1].title]=repr(object[1].lockedBy)
-
-
- keys=lockedObjects.keys()
-
-
- if len(keys)>0 and (not procedure):
- self.REQUEST.SESSION['lockedObjects']=lockedObjects
- pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','lockedObjects.zpt')).__of__(self)
-
-
- return pt()
-
- elif not procedure: #keine fails gesperrt dann alle donwloaden
- procedure="downloadAll"
-
-
-
-
- threadName=repeat
- if not threadName or threadName=="":
- thread=DownloadBasketFinallyThread()
- threadName=thread.getName()[0:]
-
- if (not hasattr(self,'_v_downloadBasket')):
- self._v_downloadBasket={}
-
-
- self._v_downloadBasket[threadName]=thread
- logging.debug("dwonloadfinally:"+repr(self))
- if isinstance(self,CDLIBasketVersion):
- obj=self
- else:
- obj=self.aq_parent
- logging.debug("dwonloadfinally2:"+repr(obj))
- logging.debug("dwonloadfinally2:"+repr(obj.aq_parent))
-
- obj2=obj.aq_parent
- if not isinstance(obj2,CDLIBasket):
- obj2=obj2.aq_parent
-
- basketID=obj2.getId()
- versionNumber=obj.getId()
- logging.debug("dwonloadfinally2:"+repr(basketID))
- logging.debug("dwonloadfinally2:"+repr(versionNumber))
-
- if lock:
- logging.debug("-----start locking")
- for object in self.content.getContent():
- if object[1].lockedBy =='':
- object[1].lockedBy=self.REQUEST['AUTHENTICATED_USER']
- logging.debug("-----finished locking")
-
- #obj.lockedBy=user
- self._v_downloadBasket[threadName].set(lock,procedure,self.REQUEST['AUTHENTICATED_USER'],current,basketID,versionNumber)
-
- self._v_downloadBasket[threadName].start()
-
-
-
- wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
-
- if wait_template:
- return wait_template[0][1]()
- pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadBasketWait.zpt')).__of__(self)
-
- return pt(txt=self.absolute_url()+'/downloadObjectsAsOneFileFinally',threadName=threadName,
- counter=self._v_downloadBasket[threadName].getCounter(),
- number=self._v_downloadBasket[threadName].getNumberOfFiles())
- #_v_xmltrans.run()
-
- else:
- #recover thread, if lost
- if not hasattr(self,'_v_downloadBasket'):
- self._v_downloadBasket={}
- if not self._v_downloadBasket.get(threadName,None):
- for thread in threading.enumerate():
- if threadName == thread.getName():
- self._v_downloadBasket[threadName]=thread
-
- if self._v_downloadBasket.get(threadName,None) and (self._v_downloadBasket[threadName] is not None) and (not self._v_downloadBasket[threadName].end) :
-
- wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
- if wait_template:
- return wait_template[0][1]()
-
- pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadBasketWait.zpt')).__of__(self)
- return pt(txt=self.absolute_url()+'/downloadObjectsAsOneFileFinally',threadName=threadName,
- counter=self._v_downloadBasket[threadName].getCounter(),
- number=self._v_downloadBasket[threadName].getNumberOfFiles())
- else:
-
-
- logging.debug("FINISHED")
- if not self._v_downloadBasket.get(threadName,None):
- for thread in threading.enumerate():
- if threadName == thread.getName():
- self._v_downloadBasket[threadName]=thread
-
- #files = self._v_downloadBasket[threadName].result
- files=self.basketContainer.resultHash[threadName]
- lockedFiles=self.basketContainer.resultLockedHash[threadName]
-
- # fh=file("/var/tmp/test")
- #ret =fh.read()
-
- if (not isinstance(self.aq_parent,CDLIBasket)):
- basket_name=self.aq_parent.aq_parent.title+"_V"+self.getId()
- else:
- basket_name=self.aq_parent.title+"_V"+self.getId()
-
-
-
- #write basketname to header of atf file
-
-
- self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.atf" """%basket_name)
- self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream")
- #length=len(ret)
- #self.REQUEST.RESPONSE.setHeader("Content-Length",length)
-
- ret="#basket: %s\n"%basket_name
- self.REQUEST.RESPONSE.write(ret)
-
- for fileName in files:
- try:
- self.REQUEST.RESPONSE.write(file(fileName).read())
- except:
- logging.error("downloadasonefile: cannot read %s"%fileName)
-
-
- self.REQUEST.RESPONSE.write("\n# locked files\n")
- for fileName in lockedFiles:
- self.REQUEST.RESPONSE.write("# %s by %s\n"%fileName)
-
- self.REQUEST.RESPONSE.write("# locked files end\n")
-
- del self.basketContainer.resultHash[threadName]
- del self.basketContainer.resultLockedHash[threadName]
-
- def numberOfItems(self):
- """return anzahl der elemente im basket"""
- return self.content.numberOfItems()
-
- def getTime(self):
- """getTime"""
- #return self.bobobase_modification_time().ISO()
-
- if hasattr(self,'time'):
- return time.strftime("%Y-%m-%d %H:%M:%S",self.time)
- elif hasattr(self,'timefixed'):
- return self.timefixed
- else:
- setattr(self,'timefixed',self.bobobase_modification_time().ISO())
- return self.bobobase_modification_time().ISO()
-
- def getContent(self):
- """get Basket Content"""
- return self.content.getContent()
-
-
- def __init__(self,id,user,comment="",basketContent=[]):
- """ init a basket version"""
- self.id=id
- self.comment=comment
- self._setObject('content',BasketContent(basketContent))
- #self.basketContent=basketContent[0:]a
- self.user=user
- self.time=time.localtime()
-
- def getUser(self):
- """get user"""
- return self.user
-
- def getComment(self):
- """get Comment"""
- return self.comment
-
- security.declareProtected('manage','index_html')
- def index_html(self):
- """view the basket"""
-
- if self.REQUEST.get('change',False):
- ob=self.aq_parent.updateObjects(self.REQUEST['change'])
-
- self.REQUEST.RESPONSE.redirect(ob.absolute_url())#go to new basket, because changing generates a new basket
-
- pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketVersionMain.zpt')).__of__(self)
- return pt()
-
- def getObjUrl(self,result):
- """getUrl of the version of the object"""
- objId=result[1].getTitle()
- founds=self.CDLICatalog.search({'title':objId})
- if len(founds)>0:
- return founds[0].getObject().getLastVersion().absolute_url()
-
- else: #assume version number
- splitted=objId.split("_")
- founds=self.CDLICatalog.search({'title':splitted[1]})
- return founds[0].getObject().getLastVersion().absolute_url()+'/'+objId
-
-def manage_addCDLIBasketVersion(self,user,comment="",basketContent=[],RESPONSE=None):
- """add a version"""
-
- #check for already existing versions
-
- lastVersion=self.getLastVersion()
- if lastVersion is None:
- newId=str(1)
- else:
- newId=str(int(lastVersion.getId())+1)
-
- ob=CDLIBasketVersion(newId,user,comment,basketContent)
-
- self._setObject(newId, ob)
-
- if RESPONSE is not None:
- RESPONSE.redirect('manage_main')
- else:
- return ob
-
class CDLIFileObject(CatalogAware,extVersionedFileObject):
"""CDLI file object"""
@@ -1674,7 +90,7 @@ class CDLIFileObject(CatalogAware,extVer
def getPNumber(self):
"""get the pnumber"""
try:
- txt=re.match("&[Pp](\d*)\s*=([^\r\n]*)",self.getData()[0:])
+ txt=re.match("&[PpSs](\d*)\s*=([^\r\n]*)",self.getData()[0:])
except:
txt=self.getData()[0:]
@@ -1688,7 +104,7 @@ class CDLIFileObject(CatalogAware,extVer
def getDesignation(self):
"""get the designation out of the file"""
try:
- txt=re.match("&[Pp](\d*)\s*=([^\r\n]*)",self.getData()[0:])
+ txt=re.match("&[PpSs](\d*)\s*=([^\r\n]*)",self.getData()[0:])
except:
txt=self.getData()[0:]
@@ -1890,10 +306,15 @@ def manage_addCDLIFile(self,id,title,loc
def checkUTF8(data):
"""check utf 8"""
+ if not isinstance(data, str):
+ logging.error("checkUTF8 data is not string! (%s)"%repr(data))
+
try:
- data.encode('utf-8')
+ data.decode('utf-8')
+ logging.debug("checkUTF8: ok!")
return True
except:
+ logging.debug("checkUTF8: false!")
return False
@@ -1902,8 +323,8 @@ def checkFile(filename,data,folder):
# first check the file name
fn=filename.split(".") # no extension
- if not fn[0][0]=="P":
- return False,"P missing in the filename"
+ if not (fn[0][0]=="P" or fn[0][0]=="S"):
+ return False,"P/S missing in the filename"
elif len(fn[0])!=7:
return False,"P number has not the right length 6"
elif not checkUTF8(data):
@@ -1919,7 +340,7 @@ def splitatf(fh,dir=None,ext=None):
i=0
#ROC: why split \n first and then \r???
- if (type(fh) is StringType) or (type(fh) is UnicodeType):
+ if isinstance(fh, basestring):
iter=fh.split("\n")
else:
iter=fh.readlines()
@@ -1927,7 +348,7 @@ def splitatf(fh,dir=None,ext=None):
for lineTmp in iter:
lineTmp=lineTmp.replace(codecs.BOM_UTF8,'') # make sure that all BOM are removed..
for line in lineTmp.split("\r"):
- #logging.log("Deal with: %s"%line)
+ #logging.info("Deal with: %s"%line)
if ext:
i+=1
if (i%100)==0:
@@ -1953,7 +374,7 @@ def splitatf(fh,dir=None,ext=None):
if dir:
filename=os.path.join(dir,filename)
nf=file(filename,"w")
- logging.info("open %s"%filename)
+ logging.debug("open %s"%filename)
if nf:
nf.write(line.replace("\n","")+"\n")
@@ -1962,8 +383,9 @@ def splitatf(fh,dir=None,ext=None):
except:
pass
- if not((type(fh) is StringType) or (type(fh) is UnicodeType)):
+ if not isinstance(fh, basestring):
fh.close()
+
return ret,len(os.listdir(dir))
@@ -2101,14 +523,12 @@ class CDLIFileFolder(extVersionedFileFol
pt=getattr(self,'filelist.html')
return pt(basketName=basketName,numberOfObjects=numberOfObjects)
- if hash is not None and hasattr(self.cdliRoot,'v_tmpStore') and self.cdliRoot.v_tmpStore.has_key(hash):
-
- logging.debug("asking for storage2")
- result =self.cdliRoot.v_tmpStore[hash]
- if result:
- logging.debug("give result from storage2")
- return hash,self.cdliRoot.v_tmpStore[hash]
-
+
+ result =self.CDLICache.retrieve(hash)
+ if result:
+ logging.debug("give result from storage2")
+ return hash,result
+
if list is not None: # got already a list
logging.debug(" ----List version")
@@ -2167,11 +587,11 @@ class CDLIFileFolder(extVersionedFileFol
return pt(search=ids)
else:
#self.REQUEST.SESSION['hash'] = ret # store in session
- if not hasattr(self,'v_tmpStore'):
- self.cdliRoot.v_tmpStore={}
+
#logging.debug("HHHHHHNEU:"+repr(self.makelist(ids)))
#logging.debug("HHHHHHNEU:"+repr(hash))
- self.cdliRoot.v_tmpStore[hash] = ret # store in session
+ self.CDLICache.store(hash,ret)
+
if returnHash == True:
return hash,ret
return ret
@@ -2307,6 +727,26 @@ class CDLIRoot(Folder):
splitter = {'words':cdliSplitter.wordSplitter(),
'graphemes':cdliSplitter.graphemeSplitter()}
+ def redirect(self,RESPONSE,url):
+ """mache ein redirect mit einem angehaengten time stamp um ein reload zu erzwingen"""
+
+ timeStamp=time.time()
+
+ if url.find("?")>-1: #giebt es schon parameter
+ addStr="&time=%s"
+ else:
+ addStr="?time=%s"
+
+ RESPONSE.setHeader('Last-Modified',email.Utils.formatdate().split("-")[0]+'GMT')
+ logging.error(email.Utils.formatdate()+' GMT')
+ RESPONSE.redirect(url+addStr%timeStamp)
+
+ def unicodify(self,txt):
+ return unicodify(txt)
+ def invalidateOldCacheVersion(self):
+ """loescht die alte Version des Cache"""
+ del self.v_tmpStore
+ return "done"
def viewATF(self,id,RESPONSE):
"""view an Object"""
@@ -2335,7 +775,9 @@ class CDLIRoot(Folder):
"""view an Object"""
ob = self.CDLICatalog({'title':id})
if len(ob)>0:
- RESPONSE.redirect(ob[0].getLastVersion().absolute_url())
+ logging.info("objekt:"+repr(ob[0]))
+ #RESPONSE.redirect(ob[0].getLastVersion().absolute_url())
+ RESPONSE.redirect(ob[0].absolute_url+"/download")
return "not found"
def addCDLIFileObjectForm(self,id,RESPONSE):
"""view an Object"""
@@ -2358,16 +800,42 @@ class CDLIRoot(Folder):
RESPONSE.redirect(ob[0].absolute_url+"/unlock")
return "not found"
+
def getFileObject(self,fileId):
"""get an object"""
+ logging.debug("getFileObj:"+repr(fileId))
+ if isinstance(fileId,CDLIFileObject): # support for old baskets
+ return fileId
x=self.v_files.get(fileId)
- #logging.debug(x)
+ logging.debug("obj: "+repr(x))
+ if x is None:
+ logging.debug("fileId"+repr(fileId))
+ folder=fileId[0:3]
+ f2=fileId[0:5]
+ fObj = getattr(self.cdliRoot.cdli_main,folder);
+ f2Obj = getattr(fObj,f2)
+
+ o = getattr(f2Obj,fileId)
+ logging.debug(o);
+ self.updateOrAddToFileBTree(o)
+ return o
return x
def getFileObjectLastVersion(self,fileId):
"""get an object"""
x=self.v_files_lastVersion.get(fileId)
- #logging.debug("lastVersion: "+repr(x))
+ logging.debug("lastVersion: "+repr(x))
+ if x==None:
+ folder=fileId[0:3]
+ f2=fileId[0:5]
+ fObj = getattr(self.cdliRoot.cdli_main,folder);
+ f2Obj = getattr(fObj,f2)
+
+ o =getattr(f2Obj,fileId)
+ logging.debug(o);
+ return o.getLastVersion()
+
+
return x
def showFileIds(self):
@@ -2396,8 +864,17 @@ class CDLIRoot(Folder):
self.v_files_lastVersion.update({obj.getId():obj.getLastVersion()})
self.v_file_ids.add(obj.getId())
- logging.debug("update:"+obj.getId()+"XXX"+repr(obj))
+ #change everthing around to make it persistent...
+ tmp = self.v_files
+ self.v_files=tmp
+ tmp2=self.v_file_ids
+ self.v_file_ids=tmp2
+
+ self.CDLICache.cleanCache() #be sure that the cache is clean
+ logging.debug("update:"+obj.getId()+"XXX"+repr(obj))
+
+
def deleteFromBTree(self,objId):
"""delete an obj"""
self.v_files.pop(objId)
@@ -2418,8 +895,9 @@ class CDLIRoot(Folder):
- def searchText(self, query, index='graphemes'):
- """searches query in the fulltext index and returns a list of file ids/P-numbers"""
+ def searchText(self, query, index='graphemes', resultFilter=None):
+ """searches query in the fulltext index and returns a list of file IDs/P-numbers
+ resultFilter is matched against the beginning of the file ID"""
# see also: http://www.plope.com/Books/2_7Edition/SearchingZCatalog.stx#2-13
logging.debug("searchtext for '%s' in index %s"%(query,index))
#import Products.ZCTextIndex.QueryParser
@@ -2429,8 +907,11 @@ class CDLIRoot(Folder):
idx = getattr(self, self.file_catalog)
# do search
resultset = idx.search(query_request=idxQuery,sort_index='textid')
- # put only the P-Number in the result
- results = [res.getId[:7] for res in resultset]
+ # put only the P-Number in the result
+ if resultFilter is None:
+ results = [res.getId[:7] for res in resultset]
+ else:
+ results = [res.getId[:7] for res in resultset if res.getId.startswith(resultFilter)]
logging.debug("searchtext: found %d texts"%len(results))
return results
@@ -2530,7 +1011,7 @@ class CDLIRoot(Folder):
if line.lstrip().startswith('#lem:'):
continue
# ignore p-num line
- if line.startswith('&P'):
+ if line.startswith('&P') or line.startswith('&S'):
continue
# ignore version lines
if line.startswith('#version'):
@@ -2607,19 +1088,68 @@ class CDLIRoot(Folder):
"""unquote url"""
return urllib.unquote_plus(str)
+ def changeUserForPObjectFromFile(self,user,fname):
+ """aendere user fuer alle p in fiele"""
+ pns = file(os.path.join(package_home(globals()),'inputs',fname),"r")
+ for p in pns.readlines():
+ p=p.lstrip().rstrip()
+ logging.info(str(p+".atf"))
+ pObj=self.getFileObject(p+".atf")
+ if pObj is not None:
+ logging.info(pObj)
+ lv=pObj.getContentObject()
+ logging.info("author:"+lv.author)
+ lv.author=user
+ lv.versionComment="XXXXXXX"
- def forceunlock(self):
+ def forceunlock(self,REQUEST=None,user=None,fid=None):
"break all locks"
+ if fid is not None:
+ self.getFileObject(fid).forceunlock()
+ return fid
ret=[]
+
for f in self.ZopeFind(self,obj_metatypes="CDLI file",search_sub=1):
- un=f[1].forceunlock()
-
+ un=f[1].forceunlock(user=user)
+ logging.info("check:"+f[0])
if un and un !="":
ret.append((f[0],un))
+ if REQUEST is not None:
+ pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','breakLockResponse.zpt')).__of__(self)
+
+ return pt(ret=ret)
+
return ret
+ def getLockedFiles(self,REQUEST=None,user=None):
+ """hole alle gesperrten files"""
+ ret={}
+
+ for nm,f in self.v_files.items():
+ lb = str(f.lockedBy)
+ add=False
+ if (lb is not None) and (lb!=""):
+
+ if user is None:
+ add=True
+ else:
+ if str(lb)==user:
+ add=True
+ if add==True:
+ if not ret.has_key(lb):
+ ret[lb]=[]
+ ret[lb].append(nm)
+
+
+ if REQUEST is not None:
+ pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','showlockResponse.zpt')).__of__(self)
+
+ return pt(ret=ret)
+
+ return ret
+
def getChangesByAuthor(self,author,n=100):
"""getChangesByAuthor"""
zcat=self.CDLIObjectsCatalog
@@ -2714,7 +1244,39 @@ class CDLIRoot(Folder):
return generateXMLReturn(stObj.returnValue)
+
+ def uploadATFAsync(self,repeat=None,upload=None,basketId=0,RESPONSE=None):
+ """upload an atf file / basket file"""
+ #self._v_uploadATF.returnValue=None
+
+
+ idTmp=str(randint(0,1000000000))
+
+ if upload is None:
+ return "ERROR NO FILE!"
+
+ tmpFile = File("/tmp/idTmp","w")
+ # sicher basket in file
+ for x in upload.read():
+ tmpFile.write(x)
+
+ tmpFile.close();
+ uploadATFAsync(baskerId,idTmp,basketId,self.REQUEST['AUTHENTICATED_USER'],idTmp,serverport=self.REQUEST['SERVER_PORT'])
+
+ return idTMP
+
+ def viewTicketAsync(self,ticketNr=1):
+ """viewticket"""
+
+ tmp = pickle.load(file("/tmp/"+str(ticketNr)+".result"))
+
+ pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadCheckAsync.zpt')).__of__(self)
+ return pt(changed=tmp['changed'],lockerrors=tmp['lockerrors'],errors=tmp['errors'],dir=tmp['dir'],newPs=tmp['newPs'],basketLen=tmp['basketLen'],numberOfFiles=tmp['numberOfFiles'],
+ basketNameFromId=tmp['basketNameFromId'],basketNameFromFile=tmp['basketNameFromFile'],basketId=tmp['basketId'])
+
+
+
def uploadATF(self,repeat=None,upload=None,basketId=0,RESPONSE=None):
"""upload an atf file / basket file"""
#self._v_uploadATF.returnValue=None
@@ -2786,6 +1348,7 @@ class CDLIRoot(Folder):
return pt(changed=tmp['changed'],lockerrors=tmp['lockerrors'],errors=tmp['errors'],dir=tmp['dir'],newPs=tmp['newPs'],basketLen=tmp['basketLen'],numberOfFiles=tmp['numberOfFiles'],
basketNameFromId=tmp['basketNameFromId'],basketNameFromFile=tmp['basketNameFromFile'],basketId=tmp['basketId'])
+
def redoUpload(self,threadName):
"""redo the upload"""
tmp=self.cdli_main.tmpStore2[threadName]
@@ -2795,9 +1358,6 @@ class CDLIRoot(Folder):
def uploadATFfinally(self,procedure='',comment="",basketname='',unlock=None,repeat=None,RESPONSE=None):
"""nowupload the files"""
-
-
-
threadName=repeat
if not threadName or threadName=="":
thread=uploadATFfinallyThread()
@@ -2854,8 +1414,14 @@ class CDLIRoot(Folder):
logging.debug("dir:"+repr(stObj.returnValue['changed']))
for x in stObj.returnValue['changed']:
ob=self.CDLICatalog.search({'title':x[0]})
-
- self.cdliRoot.updateOrAddToFileBTree(ob[0].getObject())
+ try:
+ self.cdliRoot.updateOrAddToFileBTree(ob[0].getObject())
+ except:
+
+ logging.error("uploadATFfinally - cannot update Object %s Error: %s %s"%(ob[1],sys.exc_info()[0],sys.exc_info()[1]))
+ for x in stObj.returnValue['newPs']:
+ obj=self.getFileObject(x) #updates the object in the cache
+ logging.debug("Got:"+repr(obj))
if RESPONSE is not None:
RESPONSE.redirect(self.absolute_url())
@@ -2875,7 +1441,11 @@ class CDLIRoot(Folder):
obj=self.ZopeFind(root,obj_ids=[folder])
logging.debug("importFiles: folder=%s f2=%s obj=%s"%(folder,f2,obj))
if ext:
- ext.result="adding: %s
"%f+ext.result
+
+ if type(ext.result) is types.FileType:
+ ext.result.write("adding: %s
\n"%f)
+ else:
+ ext.result="adding: %s
"%f+ext.result
if not obj: # if not create it
@@ -2902,17 +1472,21 @@ class CDLIRoot(Folder):
id=f
logging.debug("importFiles: addCDLIFile fobj2=%s, f=%s file2=%s"%(fobj2,repr(f),repr(file2)))
fobj2.addFile(vC='',file=file(file2),author=author,newName=f)
+ logging.debug("importfiles: fobj2.add")
count+=1
#now add the file to the storage
ob = getattr(fobj2,f)
- self.cdliRoot.updateOrAddToFileBTree(ob)
-
+ logging.debug("importfiles: btree_start")
+ #self.cdliRoot.updateOrAddToFileBTree(ob)
+ logging.debug("importfiles: btree_end")
if count%100==0:
logging.debug("importfiles: committing")
transaction.get().commit()
+ logging.debug("importfiles: committing")
transaction.get().commit()
+ logging.debug("importfiles: committing done")
return "ok"
@@ -2951,3 +1525,37 @@ def manage_addCDLIRoot(self, id, title='
if REQUEST is not None:
return self.manage_main(self, REQUEST, update_menu=1)
+
+import cdli_basket
+
+
+# Die folgenden Klassen sollte nicht mehr aus diesem Paket benutzt werden, sondern direkt aus
+# cdli_basket importiert werden.
+class uploadATFfinallyThread(cdli_basket.uploadATFfinallyThread):
+ """depricates"""
+ pass
+
+class tmpStore(cdli_basket.tmpStore):
+ """depricated"""
+ pass
+
+class uploadATFThread(cdli_basket.uploadATFThread):
+ """depricated"""
+ pass
+
+class CDLIBasketContainer(cdli_basket.CDLIBasketContainer):
+ """depricated"""
+ pass
+
+class CDLIBasket(cdli_basket.CDLIBasket):
+ """depricated"""
+ pass
+
+class CDLIBasketVersion(cdli_basket.CDLIBasketVersion):
+ """depricated"""
+ pass
+
+class BasketContent(cdli_basket.BasketContent):
+ """depricated"""
+ pass
+