"""CDLI extensions of the filearchive"""
from Products.versionedFile.extVersionedFile import *
from Products.ZCatalog.CatalogPathAwareness import CatalogAware
import os.path
import os
import urlparse
import urllib
import cgi
from OFS.OrderedFolder import OrderedFolder
from OFS.SimpleItem import SimpleItem
import time
from OFS.Folder import manage_addFolder
import re
from AccessControl import ClassSecurityInfo
from Acquisition import Implicit
from Globals import Persistent
from threading import Thread
from ZPublisher.HTTPRequest import HTTPRequest
from ZPublisher.HTTPResponse import HTTPResponse
from ZPublisher.BaseRequest import RequestContainer
import threading
import logging
import transaction
import copy
import codecs
import sys
from BTrees.IOBTree import IOBTree
import cdliSplitter
from sets import Set
import md5
from DownloadBasket import DownloadBasketFinallyThread
from types import *
import pickle
import tempfile
from cdli_files import CDLIFile
from cdli_files import splitatf,checkFile
from cdli_helpers import *
class BasketContent(SimpleItem):
"""classe fuer den Inhalt eines Baskets"""
def getFileAndVersionFromId(self,pnum,versionNr):
obj=self.cdliRoot.getFileObject(pnum)
logging.debug("obj : %s"%obj)
version=obj.getVersionNr(versionNr)
logging.debug("-------vs: %s"%version.getFileName())
return version,obj
def __init__(self,content=[]):
"""content"""
self.setContent(content[0:])
def getContent(self,filtered=True):
return self.contentList
def getContentOld(self,filtered=True):
"""get content"""
logging.debug("content object: content List %s"%self.contentList)
ret=[]
return [self.getFileAndVersionFromId(x[0],x[1]) for x in self.contentList]
#
# if filtered:
# for x in self.contentList:
# if not((x[0] is None) or (x[1] is None)):
# ret.append(x)
# logging.debug("content object: content List -done filtered")
# return ret
#
# else:
# logging.debug("content object: content List -done not filtered")
# return self.contentList
def allContent(self):
"""get all content"""
return self.getContent(filtered=False)
def setContent(self,content):
contentList=[]
for x in content:
if not((x[0] is None) or (x[1] is None)):
contentList.append((x[1].getId(),x[0].getVersionNumber()))
logging.debug("cl: %s"%contentList)
self.contentList=contentList[0:]
def numberOfItems(self):
"""number"""
return len(self.getContent())
class uploadATFfinallyThread(Thread):
"""class for adding uploaded filed (temporarily stored in the staging area at /tmp"""
def __init__(self):
"""init for uploadATFfinallyThread"""
self.continueVar=True
self.returnValue=None
self.end=False
Thread.__init__(self)
def set(self,procedure,comment="",basketname='',unlock=None,SESSION=None,username=None,serverport="8080"):
"""set start values for the thread"""
self.procedure=procedure
self.comment=comment
self.basketname=basketname
self.unlock=unlock
self.SESSION=SESSION
self.username=username
self.serverport=serverport
def __call__(self):
"""call of the thread (equals run)"""
self.run()
return True
def getContext(self, app,serverport="8080"):
"""get the context within the ZODB"""
resp = HTTPResponse(stdout=None)
env = {
'SERVER_NAME':'localhost',
'SERVER_PORT':serverport,
'REQUEST_METHOD':'GET'
}
req = HTTPRequest(None, env, resp)
return app.__of__(RequestContainer(REQUEST = req))
def run(self):
"""run"""
self.result=""
#find context within ZODB
from Zope import DB
conn = DB.open()
root = conn.root()
app = root['Application']
ctx = self.getContext(app,serverport=self.serverport)
#add the files
self.uploadATFfinallyThread(ctx,self.procedure,comment=self.comment,basketname=self.basketname,unlock=self.unlock,SESSION=self.SESSION,username=self.username)
#commit the transactions
transaction.get().commit()
conn.close()
#set flag for end of this method
self.end=True
logging.info("ended")
return True
def __del__(self):
"""delete"""
def getResult(self):
"""method for accessing result"""
return self.result
def uploadATFfinallyThread(self,ctx,procedure,comment="",basketname='',unlock=None,RESPONSE=None,SESSION=None,username=None):
"""upload the files"""
#TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot
ctx2=ctx.cdliRoot
self.result+="<h2>Start processing</h2>"
#shall I only upload the changed files?
logging.debug("uploadATFfinally procedure: %s"%procedure)
if procedure=="uploadchanged":
changed=[x[0] for x in SESSION.get('changed',[])]
uploadFns=changed+SESSION.get('newPs',[])
#or all
elif procedure=="uploadAll":
uploadFns=[]
for x in os.listdir(SESSION['tmpdir']):
if not x in SESSION['lockerrors']:
uploadFns.append(x)
#or maybe nothing
elif procedure=="noupload":
return True
else:
uploadFns=[]
#do first the changed files
i=0
for fn in uploadFns:
logging.debug("uploadATFfinally uploadFn=%s"%fn)
i+=1
founds=ctx2.CDLICatalog.search({'title':fn})
if len(founds)>0:
SESSION['author']=str(username)
self.result="<p>Changing : %s"%fn+self.result
logging.debug("uploadatffinallythread changing:%s"%fn+self.result)
founds[0].getObject().manage_addCDLIFileObject('',comment,SESSION['author'],file=os.path.join(SESSION['tmpdir'],fn),from_tmp=True)
if i%200==0:
transaction.get().commit()
logging.debug("uploadatffinallythread changing: do commit")
transaction.get().commit()
logging.debug("uploadatffinallythread changing: last commit")
#now add the new files
newPs=SESSION['newPs']
if len(newPs)>0:
tmpDir=SESSION['tmpdir']
logging.debug("uploadatffinallythread adding start")
self.result="<p>Adding files</p>"+self.result
#TODO: make this configurable, at the moment base folder for the files has to be cdli_main
ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self)
logging.debug("uploadatffinallythread adding finished")
#unlock locked files?
if unlock:
logging.debug("uploadatffinallythread unlocking start")
self.result="<p>Unlock files</p>"+self.result
unlockFns=[]
for x in os.listdir(SESSION['tmpdir']):
if not x in SESSION['errors']:
unlockFns.append(x)
logging.debug("unlocking have now what to unlock")
for fn in unlockFns:
#logging.info("will unlock: %s"%fn)
founds=ctx2.CDLICatalog.search({'title':fn})
#logging.info("found it: %s"%repr(founds))
if len(founds)>0:
#logging.info("unlock: %s"%founds[0].getObject().getId())
SESSION['author']=str(username)
founds[0].getObject().lockedBy=""
logging.debug("uploadatffinallythread unlocking done")
#if a basketname is given, add files to the basket
if not (basketname ==''):
logging.debug("uploadatffinallythread add to basket %s"%basketname)
self.result="<p>Add to basket</p>"+self.result
basketId=ctx2.basketContainer.getBasketIdfromName(basketname)
if not basketId: # create new basket
logging.debug("uploadatffinallythread create basket %s"%basketname)
self.result="<p>Create a new basket</p>"+self.result
ob=ctx2.basketContainer.addBasket(basketname)
basketId=ob.getId()
basket=getattr(ctx2.basketContainer,str(basketId))
ids=os.listdir(SESSION['tmpdir'])
#logging.debug("should add:"+repr(ids))
basket.addObjects(ids,deleteOld=True,username=str(username))
logging.debug("uploadatffinallythread uploadfinally done")
if RESPONSE is not None:
RESPONSE.redirect(self.aq_parent.absolute_url())
return True
class tmpStore(SimpleItem):
"""simple item"""
meta_type="cdli_upload"
def __init__(self,id):
"""init tmp"""
self.id=id
class uploadATFThread(Thread):
"""class for checking the files befor uploading"""
def __init__(self):
"""initialise"""
self.continueVar=True
self.returnValue=None
Thread.__init__(self)
def set(self,upload,basketId,username,idTmp,serverport="8080"):
"""set start values for the thread"""
self.result=""
self.upload=upload
self.basketId=basketId
self.username=username
self.serverport=serverport
self.idTmp=idTmp
def __call__(self):
"""call method """
self.run()
return True
def getContext(self, app,serverport="8080"):
"""get the context within the ZODB"""
resp = HTTPResponse(stdout=None)
env = {
'SERVER_NAME':'localhost',
'SERVER_PORT':serverport,
'REQUEST_METHOD':'GET'
}
req = HTTPRequest(None, env, resp)
return app.__of__(RequestContainer(REQUEST = req))
def run(self):
idTmp=self.idTmp
self.result=""
#find context within ZODB
from Zope import DB
conn = DB.open()
root = conn.root()
app = root['Application']
ctx = self.getContext(app,serverport=self.serverport)
logging.info("run intern")
try:
logging.info("created: %s"%idTmp)
ctx.temp_folder._setObject(idTmp,tmpStore(idTmp))
except:
logging.error("thread upload: %s %s"%sys.exc_info()[0:2])
logging.info("call thread intern")
self.uploadATFThread(ctx,self.upload,idTmp,self.basketId)
#ctx.cdliRoot.cdli_main.tmpStore2[self.getName()[0:]]=self.returnValue
transaction.get().commit()
conn.close()
return getattr(ctx.temp_folder,idTmp)
def getResult(self):
"""method for accessing result"""
return self.result
def uploadATFThread(self,ctx,upload,idTmp,basketId=0):
"""upload an atf file"""
#TODO: add comments
#TODO: finish uploadATF
stObj=getattr(ctx.temp_folder,idTmp)
logging.info("start, upload thread")
self.result="<html><body><h2>I got your file, start now to split it into single atf-files!</h2><p>"
#make sure that id is a string and not an integer
basketId=str(basketId)
logging.info("basketID:"+basketId)
#TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot
ctx2=ctx.cdliRoot
#get temporary file for staging the downloaded and splitted files
dir=tempfile.mkdtemp()
logging.info("tmpfFile:"+str(dir))
changed=[] # changed files
errors=[] # files with errors
lockerrors=[] # files with errors
newPs=[] # new p filed
psNotInCatalog=[] # files not in the catalog
#split the uploadedd atf file
logging.info("start splitting")
basketNameFromFile, numberOfFiles=splitatf(upload,dir,ext=self)
#find basketId if not set
#get active abaket
if basketId == '0':
basketObj=ctx2.basketContainer.getActiveBasket()
if basketObj:
basketId=basketObj.getId()
#if there is no active basket and no basketid given, id is empty, else get besketname and length
if basketId == '0':
basketNameFromId=""
basketLen=0
else:
basketNameFromId=getattr(ctx2.basketContainer,basketId).title
basketLen=getattr(ctx2.basketContainer,basketId).getLastVersion().numberOfItems()
logging.info("got the file, upload thread")
self.result+="""<html><body><h2>I got the files</h2><
p>I am computing the differences to the exisiting files</p>"""
#start to check the files
for fn in os.listdir(dir):
self.result="<p>process:%s</p>"%fn+self.result
# check if file is in the catalog
#TODO: checkCatalog is not implemented yet
if ctx2.cdli_main.checkCatalog(fn):
psNotInCatalog.append(fn)
#check if p-file already at the server
founds=ctx2.CDLICatalog.search({'title':fn})
#if not than add filename to the list of newfiles
data=file(os.path.join(dir,fn)).read()
status,msg=checkFile(fn,data,dir)
#status=True
if not status: # error
errors.append((fn,msg))
else:
if len(founds)==0:
newPs.append(fn)
#if p file alread at the server
for found in founds:
#analyse the differences to the actual file
obj=found.getObject()
if (not (str(obj.lockedBy))=='') and (not (str(obj.lockedBy)==str(self.username))):
lockerrors.append((fn,str(obj.lockedBy)))
else:
diffs=obj.diff(data)
if diffs[0]>0:
changed.append((obj,diffs)) #hochladen
#ready, set the returnValues
self.result+="<h3>Done</h3></body></html>"
stObj.returnValue={}
stObj.returnValue['errors']=errors
stObj.returnValue['newPs']=newPs
stObj.returnValue['tmpdir']=dir
stObj.returnValue['basketLen']=basketLen
stObj.returnValue['numberOfFiles']=numberOfFiles
stObj.returnValue['basketNameFromId']=basketNameFromId
stObj.returnValue['basketNameFromFile']=basketNameFromFile
stObj.returnValue['basketId']=basketId
stObj.returnValue['dir']=dir
#stObj.returnValue['changed']=copy.copy(changed)
stObj.returnValue['changed']=[(x[0].getId(),x[1][0]) for x in changed]
#stObj.returnValue['lockerrors']=[x[0].getId() for x in lockerrors]
stObj.returnValue['lockerrors']=[x for x in lockerrors]
self.returnValue=True
#ctx2.cdli_main.setTemp('v_uploadATF_returnValue',True)
class CDLIBasketContainer(OrderedFolder):
"""contains the baskets"""
security=ClassSecurityInfo()
meta_type="CDLIBasketContainer"
def getResultHash(self):
"""get the result hash for debug purposes"""
return self.resultHash.keys()
def getPNumbersOfBasket(self,basketName):
"""get all pnumbers of a basket as a list, returns an empty list if basket not found
@param basketName: name of the basket
"""
ret=[]
basketId=self.getBasketIdfromName(basketName)
if not basketId:
return []
ob=getattr(self,basketId).getContent() #get the content of a basket
ret=[x[0].split(".")[0] for x in ob]
return ret
security.declareProtected('manage','getBasketAsOneFile')
def getBasketAsOneFile(self,basketName,current="no"):
"""returns all files of the basket combined in one file
@param basketName: Name of the basket
@param current: (optional) if current is set to "yes" then the most current version of
all files are downloaded and not the versions of the files as stored in the basket
"""
ret=""
basketId=self.getBasketIdfromName(basketName)
if not basketId:
return ""
ob=getattr(self,basketId).getLastVersion()
for pnum,versionNr in ob.getContent():
obj=self.cdliRoot.getFileObject(pnum)
# logging.debug("obj : %s"%obj)
# version=obj.getVersionNr(versionNr)
if current=="no": #version as they are in the basket
cur= obj.getVersionNr(versionNr)
ret+=str(cur.getData())+"\n"
elif current=="yes":
#search current object
#logging.debug("current: %s"%object[1].getId().split(".")[0])
obj.getData()
return ret
security.declareProtected('manage','upDateBaskets')
def upDateBaskets(self):
"""update content in to objects"""
founds=self.ZopeFind(self,obj_metatypes=['CDLIBasketVersion'],search_sub=1)
for found in founds:
found[1].updateBasket()
security.declareProtected('manage','deleteBaskets')
def deleteBaskets(self,ids=None):
"""delete baskets, i.e. move them into trash folder"""
if ids is None:
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','cdliError_html.zpt')).__of__(self)
txt="Sorry, no basket selected!"
return pt(txt=txt)
found=self.ZopeFind(self,obj_ids=['trash'])
if len(found)<1:
manage_addFolder(self, 'trash')
trash=self._getOb('trash')
else:
trash=found[0][1]
if type(ids) is not ListType:
ids=[ids]
logging.error("XERXON:"+repr(ids))
if len(ids)==0:
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','cdliError_html.zpt')).__of__(self)
txt="Sorry, no basket selected!"
return pt(txt=txt)
cut=self.manage_cutObjects(ids)
trash.manage_pasteObjects(cut)
return None
security.declareProtected('manage','manageBaskets')
def manageBaskets(self,submit,ids=None,basket1="",basket2="",joinBasket="",subtractBasket="",REQUEST=None,RESPONSE=None):
"""manage baskets, delete or copy"""
if submit=="delete":
ret= self.deleteBaskets(ids)
if ret:
return ret
elif submit=="join":
flag,msg=self.joinBasket(joinBasket, ids)
logging.info("joining %s %s"%(flag,msg))
if not flag:
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','cdliError_html.zpt')).__of__(self)
return pt(txt=msg)
elif submit=="subtract":
logging.info("BBBb %s %s"%(basket1,basket2))
flag,msg=self.subtractBasket(subtractBasket, basket1,basket2)
logging.info("subtract %s %s"%(flag,msg))
if RESPONSE:
RESPONSE.redirect(self.absolute_url())
security.declareProtected('View','getBasketIdfromName')
def getBasketIdfromName(self,basketname):
"""get id from name"""
for basket in self.ZopeFind(self,obj_metatypes=["CDLIBasket"]):
if basket[1].title==basketname:
return basket[0]
else:
None
security.declareProtected('manage','uploadBasket_html')
def uploadBasket_html(self,basketId='0'):
"""upload an atf file, html form"""
basketId=str(basketId)
if not basketId=='0':
basketName=getattr(self.basketContainer,basketId).title
else:
basketName=""
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadBasket_html.zpt')).__of__(self)
return pt(basketId=basketId,basketName=basketName)
security.declareProtected('manage','index_html')
def index_html(self):
"""stanadard ansicht"""
ext=self.ZopeFind(self,obj_ids=["index.html"])
if ext:
return ext[0][1]()
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketContainerMain')).__of__(self)
return pt()
def getStorageFolderRoot(self):
"""root des storage folders"""
return self.cdli_main
def __init__(self,id,title):
""" init basket container"""
self.id=id
self.title=title
def getBasketsId(self):
"""get all baskets als klartext"""
ret=""
baskets=self.ZopeFind(self,obj_metatypes=['CDLIBasket'])
for basket in baskets:
com,user,time,values = basket[1].getContentIds()
ret+= "BASKET:"+com+"\t"+user+"\t"+time+"\n"
for x in values:
ret+= x[0]+"\t"+x[1]+"\n"
return ret
def getBaskets(self,sortField='title'):
"""get all baskets files"""
def sortName(x,y):
return cmp(x[1].title.lower(),y[1].title.lower())
def sortDate(x,y):
return cmp(y[1].getLastVersion().getTime(),x[1].getLastVersion().getTime())
def sortComment(x,y):
try:
xc=getattr(x[1],'comment','ZZZZZZZZZZZZZ').lower()
except:
xc='ZZZZZZZZZZZZZ'.lower()
try:
yc=getattr(y[1],'comment','ZZZZZZZZZZZZZ').lower()
except:
yc='ZZZZZZZZZZZZZ'.lower()
if (xc=='') or (xc=='ZZZZZZZZZZZZZ'.lower()):
try:
xc=x[1].getLastVersion().getComment().lower()
except:
xc='ZZZZZZZZZZZZZ'.lower()
if (yc=='') or (yc=='ZZZZZZZZZZZZZ'.lower()):
try:
yc=y[1].getLastVersion().getComment().lower()
except:
yc='ZZZZZZZZZZZZZ'.lower()
return cmp(xc,yc)
def sortAuthor(x,y):
return cmp(x[1].getLastVersion().getUser().lower(),y[1].getLastVersion().getUser().lower())
baskets=self.ZopeFind(self,obj_metatypes=['CDLIBasket'])
if sortField=='title':
baskets.sort(sortName)
elif sortField=='date':
baskets.sort(sortDate)
elif sortField=='author':
baskets.sort(sortAuthor)
elif sortField=='comment':
baskets.sort(sortComment)
return baskets
def subtractBasket(self,newBasket,basket1,basket2):
"""subtract basket2 from basket1
(i.e. newbasket will contain alle elements of basket1 which are not in basket2),
if basket2 contains files which are not in basket1, then theses files fill be ignored
@param newbasket: name of the new basket
@param basket1: basket where basket2 will be subtracted from
@param basket2: see above
"""
logging.info("CCCCC %s %s"%(basket1,basket2))
try:
newB=self.addBasket(newBasket)
except:
return False, "cannot create the new basket"
bas2= getattr(self,basket2)
bas2content=bas2.getContent()
bas2ids=[x[0] for x in bas2content]
bas1= getattr(self,basket1)
bas1content=bas1.getContent()
newBasketContent={}
for id,version in bas1content:
if not (id in bas2ids):
newBasketContent[id]=version
username=self.getActualUserName()
logging.info("sbc %s"%newBasketContent)
newB.addObjectsWithVersion(newBasketContent,username=username,catalog=self.CDLICatalog)
return True, ""
def joinBasket(self,newBasket,oldBaskets):
"""join two baskets
@param newbasket: name of the new basket
@param oldbaskets: list of baskets to be joined
"""
if oldBaskets is None:
return False, "No Baskets selected!"
try:
newB=self.addBasket(newBasket)
except:
return False, "cannot create the new basket"
newBasketContent={}
for ob in oldBaskets:
x= getattr(self,ob,None)
if x is None:
return False, "cannot find basket: %s"%ob
ids=x.getContent() # hole den Inhalt
for id,version in ids:
if newBasketContent.has_key(id): # p number gibt's schon
newBasketContent[id]=max(newBasketContent[id],version) # speichere die groessere Versionsnumber
else:
newBasketContent[id]=version
username=self.getActualUserName()
logging.info("nbc %s"%newBasketContent)
newB.addObjectsWithVersion(newBasketContent,username=username,catalog=self.CDLICatalog)
return True, ""
def getNewId(self):
"""createIds"""
last=getattr(self,'last',0)
last +=1
while len(self.ZopeFind(self,obj_ids=[str(last)]))>0:
last+=1
self.last=last
return last
def setActiveBasket(self,basketId,REQUEST=None):
"""store active basketId in a cookie"""
self.REQUEST.RESPONSE.setCookie("CDLIActiveBasket",basketId,path="/")
try:
qs=cgi.parse_qs(REQUEST['QUERY_STRING'])
del(qs['basketId'])
except:
qs={}
if REQUEST:
REQUEST.RESPONSE.redirect(REQUEST['URL1']+'?'+urllib.urlencode(qs))
def getActiveBasket(self):
"""get active basket from cookie"""
id= self.REQUEST.cookies.get('CDLIActiveBasket',None)
if id:
obj=getattr(self,str(id),None)
else:
obj=None
return obj
def getActualUserName(self):
"""get name of the actualuser"""
return str(self.REQUEST['AUTHENTICATED_USER'])
security.declareProtected('manage','addBasket')
def addBasket(self,newBasketName):
"""add a new basket"""
ob=manage_addCDLIBasket(self,newBasketName)
return ob
def storeInBasket(self,submit,ids=None,newBasketName=None,fromFileList=None,RESPONSE=None,REQUEST=None):
"""store it"""
if not ids:
ids=self.REQUEST.SESSION['fileIds']
if (type(ids) is not ListType) and (not isinstance(ids,Set)):
ids=[ids]
if isinstance(ids,Set):
ids=list(ids)
if (submit.lower()=="store in new basket") or (submit.lower()=="new basket"):
basketRet=self.addBasket(newBasketName)
self.setActiveBasket(basketRet.getId())
basket=getattr(self,basketRet.getId())
elif (submit.lower()=="store in active basket") or (submit.lower()=="active basket"):
basket=self.getActiveBasket()
added=basket.addObjects(ids)
back=self.REQUEST['HTTP_REFERER'].split("?")[0]+"?basketName="+basket.title+"&numberOfObjects="+str(added)
if fromFileList:
return self.cdli_main.findObjectsFromList(list=ids,basketName=basket.title,numberOfObjects=added)
if RESPONSE:
RESPONSE.redirect(back)
return True
def manage_addCDLIBasketContainerForm(self):
"""add the CDLIBasketContainer form"""
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIBasketContainer.zpt')).__of__(self)
return pt()
def manage_addCDLIBasketContainer(self,id,title,RESPONSE=None):
"""add the basket"""
ob=CDLIBasketContainer(id,title)
self._setObject(id, ob)
if RESPONSE is not None:
RESPONSE.redirect('manage_main')
class CDLIBasket(Folder,CatalogAware):
"""basket"""
meta_type="CDLIBasket"
default_catalog="CDLIBasketCatalog"
def searchInBasket(self,indexName,searchStr,regExp=False):
"""searchInBasket"""
lst=self.searchInLineIndexDocs(indexName,searchStr,uniq=True,regExp=regExp) #TODO: fix this
ret={}
lv=self.getLastVersion()
for obj in lv.content.getContent():
id=obj[1].getId().split(".")[0]
if id in lst:
ret[id]=self.showWordInFile(id,searchStr,lineList=self.getLinesFromIndex(indexName,searchStr,id,regExp=regExp),regExp=regExp,indexName=indexName)
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','searchResultsInBasket')).__of__(self)
return pt(result=ret,indexName=indexName,regExp=regExp,word=searchStr)
def searchInBasket_v1(self,searchStr):
"""search occurences of searchStr in files im basket"""
ret=[]
lv=self.getLastVersion()
logging.info("searching")
for obj in lv.content.getContent():
txt=obj[0].getData()
for x in txt.split("\n"):
logging.info("search %s"%x)
if re.match(searchStr,x):
ret.append(x)
return "\n".join(ret)
def getFile(self,obj):
return obj[1]
def getFileLastVersion(self,obj):
return obj[0]
def getFileNamesInLastVersion(self):
"""get content of the last version as list"""
return [x[1].getId() for x in self.getLastVersion().getContent()]
def isActual(self,obj,nummer):
"""teste ob im basket die aktuelle version ist, obj kann entweder ein CDLIFile sein oder eine
eine pnummer, die auf ein CDLIFile verweist"""
try:
#logging.debug("isActual:"+repr(obj))
if isinstance(obj, CDLIFile):
actualNo=obj.getLastVersion().getVersionNumber()
else:
actualNo=self.cdliRoot.getFileObjectLastVersion(obj).getVersionNumber()
if actualNo==nummer:
return True , 0
else:
return False, actualNo
except:
logging.error( """is actual: %s (%s %s)"""%(repr(obj),sys.exc_info()[0],sys.exc_info()[1]))
logging.error(""" PARAMS: %s %s"""%(obj,nummer))
return False, -1
def isActualOld(self,obj):
"""teste ob im basket die aktuelle version ist"""
try:
#logging.debug("isActual:"+repr(obj))
actualNo=obj[1].getLastVersion().getVersionNumber()
storedNo=obj[0].getVersionNumber()
#actualNo=self.getFileObjectLastVersion(obj.getId()).getVersionNumber()
#if len(founds)>0 and founds[0].getObject().aq_parent.getId()==".trash":
# return False, -1
if actualNo==storedNo:
return True , 0
else:
return False, actualNo
except:
logging.error( """is actual: %s (%s %s)"""%(repr(obj),sys.exc_info()[0],sys.exc_info()[1]))
return False, -1
def history(self):
"""history"""
ext=self.ZopeFind(self.aq_parent,obj_ids=["history_template.html"])
if ext:
return getattr(self,ext[0][1].getId())()
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketHistory')).__of__(self)
return pt()
def getStorageFolderRoot(self):
"""root des storage folders"""
return self.aq_parent.cdli_main
def __init__(self,id,title,shortDescription="",comment=""):
"""init a CDLIBasket"""
self.id=id
self.title=title
self.shortDescription=shortDescription
self.comment=comment
def getActualUserName(self):
"""get name of the actualuser"""
return str(self.REQUEST['AUTHENTICATED_USER'])
def getLastVersion(self):
"""hole letzte version"""
ids=[]
idsTmp= self.objectIds()
for x in idsTmp:
try:
ids.append(int(x))
except:
pass
ids.sort()
if len(ids)==0:
return None
else:
ob=getattr(self,str(ids[-1]))
return ob
def getVersions(self):
"""get versions"""
versions=self.ZopeFind(self,obj_metatypes=["CDLIBasketVersion"])
return versions
def updateObjects(self,ids,RESPONSE=None,REQUEST=None):
"""update ids, ids not in the basket the add"""
if type(ids) is not ListType:
ids=[ids]
lastVersion=self.getLastVersion()
oldContent=lastVersion.content.getContent()
newContent=[]
#first copy the old
for obj in oldContent:
if obj[1].getId() not in ids:
newContent.append(obj)
#now add the new
for id in ids:
founds=self.CDLICatalog.search({'title':id})
for found in founds:
if found.getObject() not in oldContent:
#TODO: was passiert wenn, man eine Object dazufuegt, das schon da ist aber eine neuere version
newContent.append((found.getObject().getLastVersion(),found.getObject()))
content=newContent
user=self.getActualUserName()
ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=newContent)
obj=self._getOb(ob.getId())
if RESPONSE:
RESPONSE.redirect(obj.absolute_url())
return obj
def addObjectsWithVersion(self,ids,deleteOld=None,username=None,catalog=None):
"""generate a new version of the basket with objects added,
hier wird jedoch nicht die letzte Version jedes Files hinzugefuegt, s
ondern ids is ein Tupel mit der Id (d.h. der p-number) und der Versionsnummer.
"""
logging.info("add to basket (%s)"%(self.getId()))
lastVersion=self.getLastVersion()
if not catalog:
catalog=self.CDLICatalog
if lastVersion is None:
oldContent=[]
else:
oldContent=lastVersion.content.getContent()
if deleteOld:
oldContent=[]
newContent=[]
added=0
for id,version in ids.iteritems():
logging.info("adding %s %s"%(id,version))
id=id.split(".")[0] # title nur die pnumber ohne atf
try:
founds=catalog.search({'title':id})
except:
founds=[]
logging.info(" found %s "%(founds))
for found in founds:
if found.getObject() not in oldContent:
#TODO: was passiert wenn, man eine Object dazufuegt, das schon da ist aber eine neuere version
newContent.append((found.getObject().getVersions()[version-1][1],found.getObject()))
added+=1
content=oldContent+newContent
if not username:
logging.error("XXXXXXXXXXX %s"%repr(self))
user=self.getActualUserName()
else:
user = username
ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=content)
logging.info("add to basket (%s) done"%(self.getId()))
return added
def addObjects(self,ids,deleteOld=None,username=None):
"""generate a new version of the basket with objects added"""
def swap(x):
return (x[1],x[0])
logging.info("add to basket (%s)"%(repr(ids)))
logging.info("add to basket (%s)"%(self.getId()))
lastVersion=self.getLastVersion()
if lastVersion is None:
oldContent=[]
else:
oldContent=lastVersion.content.getContent()
if deleteOld:
oldContent=[]
added=0
# for id in ids:
# logging.debug("adding:"+id)
# try:
# founds=self.CDLICatalog.search({'title':id})
# except:
# founds=[]
#
# for found in founds:
# if found.getObject() not in oldContent:
# #TODO: was passiert wenn, man eine Object dazufugt, das schon da ist aber eine neuere version
# newContent.append((found.getObject().getLastVersion(),found.getObject()))
# added+=1
hash = md5.new(repr(makelist(ids))).hexdigest() # erzeuge hash als identification
#logging.debug("JJJJJJJ:"+repr(self.makelist(ids)))
retrieved = self.CDLICache.retrieve(hash)
if retrieved:
newContent=Set(map(swap,retrieved))
else:
newContent=Set([(self.getFileObjectLastVersion(x),self.getFileObject(x)) for x in ids])
#remove all Elements which are not stored
if (None,None) in newContent:
newContent.remove((None,None))
content=Set(oldContent).union(newContent)
added = len(content)-len(oldContent)
if not username:
user=self.getActualUserName()
else:
user = username
#logging.debug("content:"+repr(list(content)))
ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=list(content))
logging.info("add to basket (%s) done"%(self.getId()))
return added
def getContent(self):
"""print content"""
ret=[]
lv=self.getLastVersion()
#for obj in lv.content.getContent():
#logging.info("XXXXXXXXXX %s"%repr(obj))
# ret.append((obj[1].getId(),obj[0].versionNumber))
return lv
def getContentIds(self):
"""print basket content"""
ret=[]
lv=self.getLastVersion()
for obj in lv.content.getContent():
ret.append((obj[0].getId(),obj[1].getId()))
return lv.getComment(),lv.getUser(),lv.getTime(),ret
def changeBasket(self,ids,submit,RESPONSE=None,REQUEST=None):
"""change a basket"""
if submit=="update":
return self.updateObjects(ids,RESPONSE=RESPONSE,REQUEST=REQUEST)
elif submit=="delete":
return self.deleteObjects(ids,RESPONSE=RESPONSE,REQUEST=REQUEST)
def deleteObjects(self,ids,RESPONSE=None,REQUEST=None):
"""delete objects"""
if type(ids) is not ListType:
ids=[ids]
lastVersion=self.getLastVersion()
oldContent=lastVersion.content.getContent()
newContent=[]
for obj in oldContent:
if obj[1].getId() not in ids:
newContent.append(obj)
user=self.getActualUserName()
ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=newContent)
if RESPONSE:
obj=self._getOb(ob.getId())
RESPONSE.redirect(obj.absolute_url())
def manage_addCDLIBasketForm(self):
"""add the CDLIBasketContainer form"""
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIBasket.zpt')).__of__(self)
return pt()
def manage_addCDLIBasket(self,title,shortDescription="",comment="",RESPONSE=None):
"""add the basket"""
id=str(self.getNewId())
ob=CDLIBasket(id,title,shortDescription,comment)
self._setObject(id, ob)
if RESPONSE is not None:
RESPONSE.redirect('manage_main')
else:
return ob
class CDLIBasketVersion(Implicit,Persistent,Folder):
"""version of a basket"""
meta_type="CDLIBasketVersion"
security=ClassSecurityInfo()
def updateBasket(self):
"""update"""
try:
self._setObject('content',BasketContent(self.basketContent))
except:
try:
if len(self.basketContent)>0:
self.content.setContent(self.basketContent)
except:
print "error",self.getId(),self.aq_parent.getId()
self.basketContent=[]
def containsNonActualFiles(self):
"""returns True if basket contains one or more non current files"""
objs=self.getContent()
for obj in objs:
if not self.isActual(obj[0],obj[1])[0]:
return True
return False
def downloadListOfPnumbers(self):
"""download pnumbers of the basket as list"""
basket_name=self.aq_parent.title
ids=self.getContent() # get the list of objects
logging.error(ids)
ret="\n".join([x[1].getId().split(".")[0] for x in ids])
self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.txt" """%basket_name)
self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream")
length=len(ret)
self.REQUEST.RESPONSE.setHeader("Content-Length",length)
self.REQUEST.RESPONSE.write(ret)
security.declareProtected('manage','downloadObjectsAsOneFile')
def downloadObjectsAsOneFile(self,lock=None,procedure=None,REQUEST=None,check="yes",current="no"):
"""download all selected files in one file"""
if self.temp_folder.downloadCounterBaskets > 10000:
return """I am sorry, currently the server has to many requests for downloads, please come back later!"""
#if (check=="yes") and self.containsNonActualFiles():
# pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadObjectAsOneFile_check.zpt')).__of__(self)
#
# return pt(lock=lock)
# neue Version aus Performancegruenden, es wird nicht mehr getestet, ob es nicht aktuelle Objekte gibt
# sondern lediglich gefragt.
if (check=="yes"):
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadObjectAsOneFile_ask.zpt')).__of__(self)
return pt(lock=lock)
else:
return self.downloadObjectsAsOneFileFinally(lock=lock,procedure=procedure,REQUEST=REQUEST,current="no")
def downloadObjectsAsOneFileFinally(self,lock=None,procedure=None,REQUEST=None,current="no",repeat=None):
"""print do the download"""
ret=""
lockedObjects={}
if lock:
logging.debug("------lock:"+repr(lock))
if str(self.REQUEST['AUTHENTICATED_USER'])=='Anonymous User':
return "please login first"
#check if a locked object exist in the basket.
lockedObjects={}
for object in self.content.getContent():
obj=self.getFileObject(object[0])
if (not str(obj.lockedBy)=="") and (not (str(obj.lockedBy)==str(self.REQUEST['AUTHENTICATED_USER']))):
lockedObjects[obj.title]=repr(obj.lockedBy)
keys=lockedObjects.keys()
if len(keys)>0 and (not procedure):
self.REQUEST.SESSION['lockedObjects']=lockedObjects
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','lockedObjects.zpt')).__of__(self)
return pt()
elif not procedure: #keine fails gesperrt dann alle donwloaden
procedure="downloadAll"
threadName=repeat
if not threadName or threadName=="":
thread=DownloadBasketFinallyThread()
threadName=thread.getName()[0:]
if (not hasattr(self,'_v_downloadBasket')):
self._v_downloadBasket={}
self._v_downloadBasket[threadName]=thread
logging.debug("dwonloadfinally:"+repr(self))
if isinstance(self,CDLIBasketVersion):
obj=self
else:
obj=self.aq_parent
logging.debug("dwonloadfinally2:"+repr(obj))
logging.debug("dwonloadfinally2:"+repr(obj.aq_parent))
obj2=obj.aq_parent
if not isinstance(obj2,CDLIBasket):
obj2=obj2.aq_parent
basketID=obj2.getId()
versionNumber=obj.getId()
logging.debug("dwonloadfinally2:"+repr(basketID))
logging.debug("dwonloadfinally2:"+repr(versionNumber))
if lock:
logging.debug("-----start locking")
for object in self.content.getContent():
obj=self.getFileObject(object[0])
if obj.lockedBy =='':
obj.lockedBy=self.REQUEST['AUTHENTICATED_USER']
logging.debug("-----finished locking")
#obj.lockedBy=user
self._v_downloadBasket[threadName].set(lock,procedure,self.REQUEST['AUTHENTICATED_USER'],current,basketID,versionNumber)
self._v_downloadBasket[threadName].start()
wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
if wait_template:
return wait_template[0][1]()
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadBasketWait.zpt')).__of__(self)
return pt(txt=self.absolute_url()+'/downloadObjectsAsOneFileFinally',threadName=threadName,
counter=self._v_downloadBasket[threadName].getCounter(),
number=self._v_downloadBasket[threadName].getNumberOfFiles())
#_v_xmltrans.run()
else:
#recover thread, if lost
if not hasattr(self,'_v_downloadBasket'):
self._v_downloadBasket={}
if not self._v_downloadBasket.get(threadName,None):
for thread in threading.enumerate():
if threadName == thread.getName():
self._v_downloadBasket[threadName]=thread
if self._v_downloadBasket.get(threadName,None) and (self._v_downloadBasket[threadName] is not None) and (not self._v_downloadBasket[threadName].end) :
wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
if wait_template:
return wait_template[0][1]()
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadBasketWait.zpt')).__of__(self)
return pt(txt=self.absolute_url()+'/downloadObjectsAsOneFileFinally',threadName=threadName,
counter=self._v_downloadBasket[threadName].getCounter(),
number=self._v_downloadBasket[threadName].getNumberOfFiles())
else:
logging.debug("FINISHED")
if not self._v_downloadBasket.get(threadName,None):
for thread in threading.enumerate():
if threadName == thread.getName():
self._v_downloadBasket[threadName]=thread
#files = self._v_downloadBasket[threadName].result
# lade die files und die locked files, bei grossen Baskets muss u.U. gewartet werden
# bis das Commit aus dem Thread alles geschrieben hat, in dem Falle existiert resultHash[threadName]
# noch nicht.
o1 = file("/tmp/"+threadName,'r')
files=pickle.load(o1)
os.remove("/tmp/"+threadName)
o2 = file("/tmp/"+threadName+'_lockedFiles','r')
lockedFiles=pickle.load(o2)
os.remove("/tmp/"+threadName+'_lockedFiles')
# try:
# files=self.basketContainer.resultHash[threadName]
# except:
# i=0
# while (not self.basketContainer.resultHash.has_key(threadName)) and (i<100):
# logging.debug(" downloadFinally: I am waiting for thread %s to write the resultHashfile: %s"%(threadName,i))
# time.sleep(5)
# i+=1
# files=self.basketContainer.resultHash[threadName]
#
# try:
# lockedFiles=self.basketContainer.resultLockedHash[threadName]
# except:
# i=0
# while (not self.basketContainer.resultLockedHash.has_key(threadName)) and (i<100):
# logging.debug(" downloadFinally: I am waiting for thread %s to write the LockedHashfile: %s"%(threadName,i))
# time.sleep(5)
# i+=1
# lockedFiles=self.basketContainer.resultLockedHash[threadName]
# fh=file("/var/tmp/test")
#ret =fh.read()
if (not isinstance(self.aq_parent,CDLIBasket)):
basket_name=self.aq_parent.aq_parent.title+"_V"+self.getId()
else:
basket_name=self.aq_parent.title+"_V"+self.getId()
#write basketname to header of atf file
self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.atf" """%basket_name)
self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream")
#length=len(ret)
#self.REQUEST.RESPONSE.setHeader("Content-Length",length)
ret="#basket: %s\n"%basket_name
self.REQUEST.RESPONSE.write(ret)
for fileName in files:
logging.debug("download: %s"%fileName)
try:
self.REQUEST.RESPONSE.write(file(fileName).read())
except:
logging.error("downloadasonefile: cannot read %s"%fileName)
self.REQUEST.RESPONSE.write("\n# locked files\n")
for fileName in lockedFiles:
self.REQUEST.RESPONSE.write("# %s by %s\n"%fileName)
self.REQUEST.RESPONSE.write("# locked files end\n")
#del self.basketContainer.resultHash[threadName]
#del self.basketContainer.resultLockedHash[threadName]
def numberOfItems(self):
"""return anzahl der elemente im basket"""
return self.content.numberOfItems()
def getTime(self):
"""getTime"""
#return self.bobobase_modification_time().ISO()
if hasattr(self,'time'):
return time.strftime("%Y-%m-%d %H:%M:%S",self.time)
elif hasattr(self,'timefixed'):
return self.timefixed
else:
setattr(self,'timefixed',self.bobobase_modification_time().ISO())
return self.bobobase_modification_time().ISO()
def getContent(self):
"""get Basket Content"""
logging.debug("retrieving content A")
cnt = self.content
logging.debug("retrieving content: obj %s"%cnt)
tmp = self.content.getContent()
logging.debug("got content")
return tmp
def __init__(self,id,user,comment="",basketContent=[]):
""" init a basket version"""
self.id=id
self.comment=comment
self._setObject('content',BasketContent(basketContent))
#self.basketContent=basketContent[0:]a
self.user=user
self.time=time.localtime()
def getUser(self):
"""get user"""
return self.user
def getComment(self):
"""get Comment"""
return self.comment
security.declareProtected('manage','index_html')
def index_html(self):
"""view the basket"""
logging.debug("start index_html - Basket version")
if self.REQUEST.get('change',False):
ob=self.aq_parent.updateObjects(self.REQUEST['change'])
self.REQUEST.RESPONSE.redirect(ob.absolute_url())#go to new basket, because changing generates a new basket
logging.debug("start index_html - Basket version:template")
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketVersionMain.zpt')).__of__(self)
return pt()
def getObjUrl(self,result):
"""getUrl of the version of the object"""
founds=self.CDLICatalog.search({'title':result})
if len(founds)>0:
return founds[0].getObject().getLastVersion().absolute_url()
else: #assume version number
splitted=result.split("_")
founds=self.CDLICatalog.search({'title':splitted[1]})
return founds[0].getObject().getLastVersion().absolute_url()+'/'+result
def manage_addCDLIBasketVersion(self,user,comment="",basketContent=[],RESPONSE=None):
"""add a version"""
#check for already existing versions
lastVersion=self.getLastVersion()
if lastVersion is None:
newId=str(1)
else:
newId=str(int(lastVersion.getId())+1)
ob=CDLIBasketVersion(newId,user,comment,basketContent)
self._setObject(newId, ob)
if RESPONSE is not None:
RESPONSE.redirect('manage_main')
else:
return ob
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>