cdli/cdli_files.py - diff

Return to cdli_files.py CVS log

Up to [Repository] / cdli

Diff for /cdli/cdli_files.py between versions 1.80.2.4 and 1.103

version 1.80.2.4, 2007/10/24 20:36:07	version 1.103, 2008/11/05 19:53:32
Line 1	Line 1
"""CDLI extensions of the filearchive"""	"""CDLI extensions of the filearchive"""
from Products.versionedFile.extVersionedFile import *	from Products.versionedFile.extVersionedFile import *
from Products.ZCatalog.CatalogPathAwareness import CatalogAware	from Products.ZCatalog.CatalogPathAwareness import CatalogAware
from tempfile import mkstemp,mkdtemp
import os.path	import os.path
import os	import os
from types import *
import urlparse	import urlparse
import urllib	import urllib
import cgi	import cgi
Line 21 from ZPublisher.HTTPRequest import HTTPR	Line 19 from ZPublisher.HTTPRequest import HTTPR
from ZPublisher.HTTPResponse import HTTPResponse	from ZPublisher.HTTPResponse import HTTPResponse
from ZPublisher.BaseRequest import RequestContainer	from ZPublisher.BaseRequest import RequestContainer
import threading	import threading
from BTrees.OOBTree import OOBTree
import logging	import logging
import transaction	import transaction
import copy	import copy
import codecs	import codecs
import sys	import sys
	from BTrees.IOBTree import IOBTree
import cdliSplitter	import cdliSplitter
	from sets import Set
	import md5
	from DownloadBasket import DownloadBasketFinallyThread
	from types import *
	import pickle

	def makelist(mySet):
	x = list(mySet)
	x.sort()
	return x

def unicodify(s):	def unicodify(s):
"""decode str (utf-8 or latin-1 representation) into unicode object"""	"""decode str (utf-8 or latin-1 representation) into unicode object"""
Line 53 def utf8ify(s):	Line 59 def utf8ify(s):
else:	else:
return s.encode('utf-8')	return s.encode('utf-8')

def formatAtfLineHtml(l, nolemma=True):	def formatAtfHtml(l):
"""escape special ATF characters for HTML"""	"""escape special ATF characters for HTML"""
if not l:	if not l:
return ""	return ""

if nolemma:
# ignore lemma lines
if l.lstrip().startswith('#lem:'):
return ""
# replace &	# replace &
l = l.replace('&','&')	l = l.replace('&','&')
# replace angular brackets	# replace angular brackets
Line 69 def formatAtfLineHtml(l, nolemma=True):	Line 71 def formatAtfLineHtml(l, nolemma=True):
l = l.replace('>','>')	l = l.replace('>','>')
return l	return l

	def formatAtfLineHtml(l, nolemma=True):
	"""format ATF line for HTML"""
	if not l:
	return ""

	if nolemma:
	# ignore lemma lines
	if l.lstrip().startswith('#lem:'):
	return ""

	return formatAtfHtml(l)



	def formatAtfFullLineNum(txt, nolemma=True):
	"""format full line numbers in ATF text"""
	# surface codes
	surfaces = {'@obverse':'obv',
	'@reverse':'rev',
	'@surface':'surface',
	'@edge':'edge',
	'@left':'left',
	'@right':'right',
	'@top':'top',
	'@bottom':'bottom',
	'@face':'face',
	'@seal':'seal'}

	if not txt:
	return ""

	ret = []
	surf = ""
	col = ""
	for line in txt.splitlines():
	line = unicodify(line)
	if line and line[0] == '@':
	# surface or column
	words = line.split(' ')
	if words[0] in surfaces:
	surf = line.replace(words[0],surfaces[words[0]]).strip()

	elif words[0] == '@column':
	col = ' '.join(words[1:])

	elif line and line[0] in '123456789':
	# ordinary line -> add line number
	line = "%s:%s:%s"%(surf,col,line)

	ret.append(line)

	return '\n'.join(ret)


def generateXMLReturn(hash):	def generateXMLReturn(hash):
"""erzeugt das xml file als returnwert fuer uploadATFRPC"""	"""erzeugt das xml file als returnwert fuer uploadATFRPC"""
Line 95 def generateXMLReturn(hash):	Line 150 def generateXMLReturn(hash):
return ret	return ret










def unique(s):	def unique(s):
"""Return a list of the elements in s, but without duplicates.	"""Return a list of the elements in s, but without duplicates.

Line 176 def unique(s):	Line 223 def unique(s):
class BasketContent(SimpleItem):	class BasketContent(SimpleItem):
"""classe fuer den Inhalt eines Baskets"""	"""classe fuer den Inhalt eines Baskets"""

	def getFileAndVersionFromId(self,pnum,versionNr):

	obj=self.cdliRoot.getFileObject(pnum)
	logging.debug("obj : %s"%obj)
	version=obj.getVersionNr(versionNr)
	logging.debug("-------vs: %s"%version.getFileName())
	return version,obj

def __init__(self,content=[]):	def __init__(self,content=[]):
"""content"""	"""content"""
self.contentList=content[0:]

def getContent(self):	self.setContent(content[0:])
"""get content"""

	def getContent(self,filtered=True):
return self.contentList	return self.contentList

	def getContentOld(self,filtered=True):
	"""get content"""
	logging.debug("content object: content List %s"%self.contentList)
	ret=[]

	return [self.getFileAndVersionFromId(x[0],x[1]) for x in self.contentList]
	#
	# if filtered:
	# for x in self.contentList:
	# if not((x[0] is None) or (x[1] is None)):
	# ret.append(x)
	# logging.debug("content object: content List -done filtered")
	# return ret
	#
	# else:
	# logging.debug("content object: content List -done not filtered")
	# return self.contentList

	def allContent(self):
	"""get all content"""
	return self.getContent(filtered=False)

def setContent(self,content):	def setContent(self,content):
self.contentList=content[0:]	contentList=[]
	for x in content:
	if not((x[0] is None) or (x[1] is None)):

	contentList.append((x[1].getId(),x[0].getVersionNumber()))
	logging.debug("cl: %s"%contentList)
	self.contentList=contentList[0:]

def numberOfItems(self):	def numberOfItems(self):
"""number"""	"""number"""
Line 272 class uploadATFfinallyThread(Thread):	Line 354 class uploadATFfinallyThread(Thread):
self.result+="<h2>Start processing</h2>"	self.result+="<h2>Start processing</h2>"

#shall I only upload the changed files?	#shall I only upload the changed files?
logging.info("uploadATFfinally procedure: %s"%procedure)	logging.debug("uploadATFfinally procedure: %s"%procedure)
if procedure=="uploadchanged":	if procedure=="uploadchanged":
changed=[x[0] for x in SESSION.get('changed',[])]	changed=[x[0] for x in SESSION.get('changed',[])]
uploadFns=changed+SESSION.get('newPs',[])	uploadFns=changed+SESSION.get('newPs',[])
Line 293 class uploadATFfinallyThread(Thread):	Line 375 class uploadATFfinallyThread(Thread):
#do first the changed files	#do first the changed files
i=0	i=0
for fn in uploadFns:	for fn in uploadFns:
	logging.debug("uploadATFfinally uploadFn=%s"%fn)
i+=1	i+=1
founds=ctx2.CDLICatalog.search({'title':fn})	founds=ctx2.CDLICatalog.search({'title':fn})
if len(founds)>0:	if len(founds)>0:
SESSION['author']=str(username)	SESSION['author']=str(username)
self.result="<p>Changing : %s"%fn+self.result	self.result="<p>Changing : %s"%fn+self.result
	logging.debug("uploadatffinallythread changing:%s"%fn+self.result)
founds[0].getObject().manage_addCDLIFileObject('',comment,SESSION['author'],file=os.path.join(SESSION['tmpdir'],fn),from_tmp=True)	founds[0].getObject().manage_addCDLIFileObject('',comment,SESSION['author'],file=os.path.join(SESSION['tmpdir'],fn),from_tmp=True)
if i==200:	if i%200==0:
i=0
transaction.get().commit()	transaction.get().commit()
logging.info("changing: do commit")	logging.debug("uploadatffinallythread changing: do commit")

transaction.get().commit()	transaction.get().commit()
logging.info("changing: last commit")	logging.debug("uploadatffinallythread changing: last commit")

#now add the new files	#now add the new files
newPs=SESSION['newPs']	newPs=SESSION['newPs']
if len(newPs)>0:	if len(newPs)>0:
tmpDir=SESSION['tmpdir']	tmpDir=SESSION['tmpdir']
logging.info("adding start")	logging.debug("uploadatffinallythread adding start")
self.result="<p>Adding files</p>"+self.result	self.result="<p>Adding files</p>"+self.result
#TODO: make this configurable, at the moment base folder for the files has to be cdli_main	#TODO: make this configurable, at the moment base folder for the files has to be cdli_main

ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self)	ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self)
logging.info("adding finished")	logging.debug("uploadatffinallythread adding finished")


#unlock locked files?	#unlock locked files?
if unlock:	if unlock:
logging.info("unlocking start")	logging.debug("uploadatffinallythread unlocking start")
self.result="<p>Unlock files</p>"+self.result	self.result="<p>Unlock files</p>"+self.result
unlockFns=[]	unlockFns=[]
for x in os.listdir(SESSION['tmpdir']):	for x in os.listdir(SESSION['tmpdir']):
if not x in SESSION['errors']:	if not x in SESSION['errors']:
unlockFns.append(x)	unlockFns.append(x)
logging.info("unlocking have now what to unlock")
	logging.debug("unlocking have now what to unlock")

for fn in unlockFns:	for fn in unlockFns:
#logging.info("will unlock: %s"%fn)	#logging.info("will unlock: %s"%fn)
Line 336 class uploadATFfinallyThread(Thread):	Line 418 class uploadATFfinallyThread(Thread):
if len(founds)>0:	if len(founds)>0:
#logging.info("unlock: %s"%founds[0].getObject().getId())	#logging.info("unlock: %s"%founds[0].getObject().getId())
SESSION['author']=str(username)	SESSION['author']=str(username)

founds[0].getObject().lockedBy=""	founds[0].getObject().lockedBy=""
logging.info("unlocking done")
	logging.debug("uploadatffinallythread unlocking done")

#if a basketname is given, add files to the basket	#if a basketname is given, add files to the basket
if not (basketname ==''):	if not (basketname ==''):
logging.info("add to basket %s"%basketname)	logging.debug("uploadatffinallythread add to basket %s"%basketname)
self.result="<p>Add to basket</p>"+self.result	self.result="<p>Add to basket</p>"+self.result
basketId=ctx2.basketContainer.getBasketIdfromName(basketname)	basketId=ctx2.basketContainer.getBasketIdfromName(basketname)

if not basketId: # create new basket	if not basketId: # create new basket
logging.info("create basket %s"%basketname)	logging.debug("uploadatffinallythread create basket %s"%basketname)
self.result="<p>Create a new basket</p>"+self.result	self.result="<p>Create a new basket</p>"+self.result
ob=ctx2.basketContainer.addBasket(basketname)	ob=ctx2.basketContainer.addBasket(basketname)
basketId=ob.getId()	basketId=ob.getId()
basket=getattr(ctx2.basketContainer,str(basketId))	basket=getattr(ctx2.basketContainer,str(basketId))
ids=os.listdir(SESSION['tmpdir'])	ids=os.listdir(SESSION['tmpdir'])
	#logging.debug("should add:"+repr(ids))
basket.addObjects(ids,deleteOld=True,username=str(username))	basket.addObjects(ids,deleteOld=True,username=str(username))

	logging.debug("uploadatffinallythread uploadfinally done")

if RESPONSE is not None:	if RESPONSE is not None:
RESPONSE.redirect(self.aq_parent.absolute_url())	RESPONSE.redirect(self.aq_parent.absolute_url())


logging.info("uploadfinally done")
return True	return True

class tmpStore(SimpleItem):	class tmpStore(SimpleItem):
Line 558 class CDLIBasketContainer(OrderedFolder)	Line 641 class CDLIBasketContainer(OrderedFolder)
security=ClassSecurityInfo()	security=ClassSecurityInfo()
meta_type="CDLIBasketContainer"	meta_type="CDLIBasketContainer"

	def getResultHash(self):
	"""get the result hash for debug purposes"""
	return self.resultHash.keys()

def getPNumbersOfBasket(self,basketName):	def getPNumbersOfBasket(self,basketName):
"""get all pnumbers of a basket as a list, returns an empty list if basket not found	"""get all pnumbers of a basket as a list, returns an empty list if basket not found
@param basketName: name of the basket	@param basketName: name of the basket
Line 567 class CDLIBasketContainer(OrderedFolder)	Line 654 class CDLIBasketContainer(OrderedFolder)
if not basketId:	if not basketId:
return []	return []

ob=getattr(self,basketId).getContent()	ob=getattr(self,basketId).getContent() #get the content of a basket

ret=[x[0].split(".")[0] for x in ob]	ret=[x[0].split(".")[0] for x in ob]

Line 586 class CDLIBasketContainer(OrderedFolder)	Line 673 class CDLIBasketContainer(OrderedFolder)
return ""	return ""

ob=getattr(self,basketId).getLastVersion()	ob=getattr(self,basketId).getLastVersion()
for object in ob.getContent():	for pnum,versionNr in ob.getContent():
	obj=self.cdliRoot.getFileObject(pnum)
	# logging.debug("obj : %s"%obj)
	# version=obj.getVersionNr(versionNr)

if current=="no": #version as they are in the basket	if current=="no": #version as they are in the basket
ret+=str(object[0].getData())+"\n"	cur= obj.getVersionNr(versionNr)
	ret+=str(cur.getData())+"\n"
elif current=="yes":	elif current=="yes":
#search current object	#search current object
logging.info("crrent: %s"%object[1].getId().split(".")[0])	#logging.debug("current: %s"%object[1].getId().split(".")[0])
founds=self.CDLICatalog.search({'title':object[1].getId().split(".")[0]})	obj.getData()
if len(founds)>0:
ret+=str(founds[0].getObject().getLastVersion().getData())+"\n"
return ret	return ret

security.declareProtected('manage','upDateBaskets')	security.declareProtected('manage','upDateBaskets')
Line 610 class CDLIBasketContainer(OrderedFolder)	Line 700 class CDLIBasketContainer(OrderedFolder)
def deleteBaskets(self,ids=None):	def deleteBaskets(self,ids=None):
"""delete baskets, i.e. move them into trash folder"""	"""delete baskets, i.e. move them into trash folder"""

	if ids is None:
	pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','cdliError_html.zpt')).__of__(self)
	txt="Sorry, no basket selected!"
	return pt(txt=txt)

found=self.ZopeFind(self,obj_ids=['trash'])	found=self.ZopeFind(self,obj_ids=['trash'])

Line 621 class CDLIBasketContainer(OrderedFolder)	Line 715 class CDLIBasketContainer(OrderedFolder)

if type(ids) is not ListType:	if type(ids) is not ListType:
ids=[ids]	ids=[ids]
	logging.error("XERXON:"+repr(ids))
	if len(ids)==0:
	pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','cdliError_html.zpt')).__of__(self)
	txt="Sorry, no basket selected!"
	return pt(txt=txt)

cut=self.manage_cutObjects(ids)	cut=self.manage_cutObjects(ids)
trash.manage_pasteObjects(cut)	trash.manage_pasteObjects(cut)
	return None
security.declareProtected('manage','manageBaskets')	security.declareProtected('manage','manageBaskets')
def manageBaskets(self,submit,ids=None,basket1="",basket2="",joinBasket="",subtractBasket="",REQUEST=None,RESPONSE=None):	def manageBaskets(self,submit,ids=None,basket1="",basket2="",joinBasket="",subtractBasket="",REQUEST=None,RESPONSE=None):
"""manage baskets, delete or copy"""	"""manage baskets, delete or copy"""
if submit=="delete":	if submit=="delete":
self.deleteBaskets(ids)	ret= self.deleteBaskets(ids)
	if ret:
	return ret
elif submit=="join":	elif submit=="join":
flag,msg=self.joinBasket(joinBasket, ids)	flag,msg=self.joinBasket(joinBasket, ids)
logging.info("joining %s %s"%(flag,msg))	logging.info("joining %s %s"%(flag,msg))
	if not flag:
	pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','cdliError_html.zpt')).__of__(self)

	return pt(txt=msg)

elif submit=="subtract":	elif submit=="subtract":
logging.info("BBBb %s %s"%(basket1,basket2))	logging.info("BBBb %s %s"%(basket1,basket2))
Line 772 class CDLIBasketContainer(OrderedFolder)	Line 877 class CDLIBasketContainer(OrderedFolder)
@param basket2: see above	@param basket2: see above

"""	"""

logging.info("CCCCC %s %s"%(basket1,basket2))	logging.info("CCCCC %s %s"%(basket1,basket2))

try:	try:
Line 812 class CDLIBasketContainer(OrderedFolder)	Line 918 class CDLIBasketContainer(OrderedFolder)
@param newbasket: name of the new basket	@param newbasket: name of the new basket
@param oldbaskets: list of baskets to be joined	@param oldbaskets: list of baskets to be joined
"""	"""
	if oldBaskets is None:
	return False, "No Baskets selected!"

try:	try:
newB=self.addBasket(newBasket)	newB=self.addBasket(newBasket)
except:	except:
return False, "cannot create the new basket"	return False, "cannot create the new basket"

newBasketContent={}	newBasketContent={}

for ob in oldBaskets:	for ob in oldBaskets:
x= getattr(self,ob,None)	x= getattr(self,ob,None)
if x is None:	if x is None:
Line 884 class CDLIBasketContainer(OrderedFolder)	Line 994 class CDLIBasketContainer(OrderedFolder)
if not ids:	if not ids:
ids=self.REQUEST.SESSION['fileIds']	ids=self.REQUEST.SESSION['fileIds']

if type(ids) is not ListType:	if (type(ids) is not ListType) and (not isinstance(ids,Set)):
ids=[ids]	ids=[ids]

	if isinstance(ids,Set):
	ids=list(ids)

if (submit.lower()=="store in new basket") or (submit.lower()=="new basket"):	if (submit.lower()=="store in new basket") or (submit.lower()=="new basket"):
basketRet=self.addBasket(newBasketName)	basketRet=self.addBasket(newBasketName)
self.setActiveBasket(basketRet.getId())	self.setActiveBasket(basketRet.getId())
Line 900 class CDLIBasketContainer(OrderedFolder)	Line 1013 class CDLIBasketContainer(OrderedFolder)

if fromFileList:	if fromFileList:

return self.cdli_main.findObjectsFromList(list=self.REQUEST.SESSION['fileIds'],basketName=basket.title,numberOfObjects=added)	return self.cdli_main.findObjectsFromList(list=ids,basketName=basket.title,numberOfObjects=added)

if RESPONSE:	if RESPONSE:

Line 977 class CDLIBasket(Folder,CatalogAware):	Line 1090 class CDLIBasket(Folder,CatalogAware):
return [x[1].getId() for x in self.getLastVersion().getContent()]	return [x[1].getId() for x in self.getLastVersion().getContent()]


def isActual(self,obj):	def isActual(self,obj,nummer):
	"""teste ob im basket die aktuelle version ist, obj kann entweder ein CDLIFile sein oder eine
	eine pnummer, die auf ein CDLIFile verweist"""
	try:
	#logging.debug("isActual:"+repr(obj))
	if isinstance(obj, CDLIFile):
	actualNo=obj.getLastVersion().getVersionNumber()
	else:
	actualNo=self.cdliRoot.getFileObjectLastVersion(obj).getVersionNumber()

	if actualNo==nummer:
	return True , 0
	else:
	return False, actualNo
	except:
	logging.error( """is actual: %s (%s %s)"""%(repr(obj),sys.exc_info()[0],sys.exc_info()[1]))
	logging.error(""" PARAMS: %s %s"""%(obj,nummer))
	return False, -1
	def isActualOld(self,obj):
"""teste ob im basket die aktuelle version ist"""	"""teste ob im basket die aktuelle version ist"""
	try:
	#logging.debug("isActual:"+repr(obj))
actualNo=obj[1].getLastVersion().getVersionNumber()	actualNo=obj[1].getLastVersion().getVersionNumber()
storedNo=obj[0].getVersionNumber()	storedNo=obj[0].getVersionNumber()

founds=self.CDLICatalog.search({'title':obj[0].getId()})
if len(founds)>0:
actualNo=founds[0].getObject().getLastVersion().getVersionNumber()

if len(founds)>0 and founds[0].getObject().aq_parent.getId()==".trash":	#actualNo=self.getFileObjectLastVersion(obj.getId()).getVersionNumber()
return False, -1
	#if len(founds)>0 and founds[0].getObject().aq_parent.getId()==".trash":
	# return False, -1

if actualNo==storedNo:	if actualNo==storedNo:
return True , 0	return True , 0
else:	else:
return False, actualNo	return False, actualNo
	except:
	logging.error( """is actual: %s (%s %s)"""%(repr(obj),sys.exc_info()[0],sys.exc_info()[1]))

	return False, -1

def history(self):	def history(self):
"""history"""	"""history"""
Line 1136 class CDLIBasket(Folder,CatalogAware):	Line 1272 class CDLIBasket(Folder,CatalogAware):

def addObjects(self,ids,deleteOld=None,username=None):	def addObjects(self,ids,deleteOld=None,username=None):
"""generate a new version of the basket with objects added"""	"""generate a new version of the basket with objects added"""

	def swap(x):
	return (x[1],x[0])

	logging.info("add to basket (%s)"%(repr(ids)))
logging.info("add to basket (%s)"%(self.getId()))	logging.info("add to basket (%s)"%(self.getId()))
lastVersion=self.getLastVersion()	lastVersion=self.getLastVersion()

Line 1147 class CDLIBasket(Folder,CatalogAware):	Line 1288 class CDLIBasket(Folder,CatalogAware):
if deleteOld:	if deleteOld:
oldContent=[]	oldContent=[]

newContent=[]
added=0	added=0
for id in ids:	# for id in ids:
try:	# logging.debug("adding:"+id)
founds=self.CDLICatalog.search({'title':id})	# try:
except:	# founds=self.CDLICatalog.search({'title':id})
founds=[]	# except:
	# founds=[]
	#
	# for found in founds:
	# if found.getObject() not in oldContent:
	# #TODO: was passiert wenn, man eine Object dazuf�gt, das schon da ist aber eine neuere version
	# newContent.append((found.getObject().getLastVersion(),found.getObject()))
	# added+=1

for found in founds:	hash = md5.new(repr(makelist(ids))).hexdigest() # erzeuge hash als identification
if found.getObject() not in oldContent:	#logging.debug("JJJJJJJ:"+repr(self.makelist(ids)))
#TODO: was passiert wenn, man eine Object dazuf�gt, das schon da ist aber eine neuere version
newContent.append((found.getObject().getLastVersion(),found.getObject()))
added+=1

content=oldContent+newContent	retrieved = self.CDLICache.retrieve(hash)
	if retrieved:
	newContent=Set(map(swap,retrieved))
	else:
	newContent=Set([(self.getFileObjectLastVersion(x),self.getFileObject(x)) for x in ids])



	#remove all Elements which are not stored
	if (None,None) in newContent:
	newContent.remove((None,None))
	content=Set(oldContent).union(newContent)
	added = len(content)-len(oldContent)
if not username:	if not username:
user=self.getActualUserName()	user=self.getActualUserName()
else:	else:
user = username	user = username

ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=content)	#logging.debug("content:"+repr(list(content)))
	ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=list(content))
logging.info("add to basket (%s) done"%(self.getId()))	logging.info("add to basket (%s) done"%(self.getId()))
return added	return added

Line 1178 class CDLIBasket(Folder,CatalogAware):	Line 1335 class CDLIBasket(Folder,CatalogAware):
ret=[]	ret=[]

lv=self.getLastVersion()	lv=self.getLastVersion()
for obj in lv.content.getContent():	#for obj in lv.content.getContent():
logging.info("XXXXXXXXXX %s"%repr(obj))	#logging.info("XXXXXXXXXX %s"%repr(obj))
ret.append((obj[1].getId(),obj[0].versionNumber))	# ret.append((obj[1].getId(),obj[0].versionNumber))

return ret	return lv

def getContentIds(self):	def getContentIds(self):
"""print basket content"""	"""print basket content"""
Line 1266 class CDLIBasketVersion(Implicit,Persist	Line 1423 class CDLIBasketVersion(Implicit,Persist

objs=self.getContent()	objs=self.getContent()
for obj in objs:	for obj in objs:
if not self.isActual(obj)[0]:	if not self.isActual(obj[0],obj[1])[0]:
return True	return True
return False	return False

Line 1293 class CDLIBasketVersion(Implicit,Persist	Line 1450 class CDLIBasketVersion(Implicit,Persist
return """I am sorry, currently the server has to many requests for downloads, please come back later!"""	return """I am sorry, currently the server has to many requests for downloads, please come back later!"""


if (check=="yes") and self.containsNonActualFiles():	#if (check=="yes") and self.containsNonActualFiles():
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadObjectAsOneFile_check.zpt')).__of__(self)	# pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadObjectAsOneFile_check.zpt')).__of__(self)
	#
	# return pt(lock=lock)

	# neue Version aus Performancegruenden, es wird nicht mehr getestet, ob es nicht aktuelle Objekte gibt
	# sondern lediglich gefragt.
	if (check=="yes"):
	pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadObjectAsOneFile_ask.zpt')).__of__(self)

return pt(lock=lock)	return pt(lock=lock)

Line 1302 class CDLIBasketVersion(Implicit,Persist	Line 1466 class CDLIBasketVersion(Implicit,Persist

return self.downloadObjectsAsOneFileFinally(lock=lock,procedure=procedure,REQUEST=REQUEST,current="no")	return self.downloadObjectsAsOneFileFinally(lock=lock,procedure=procedure,REQUEST=REQUEST,current="no")

def downloadObjectsAsOneFileFinally(self,lock=None,procedure=None,REQUEST=None,current="no"):	def downloadObjectsAsOneFileFinally(self,lock=None,procedure=None,REQUEST=None,current="no",repeat=None):
"""print do the download"""	"""print do the download"""


ret=""	ret=""
lockedObjects={}	lockedObjects={}

self.temp_folder.downloadCounterBaskets+=1
self._p_changed=1
transaction.get().commit()

if lock:

	if lock:
	logging.debug("------lock:"+repr(lock))
if str(self.REQUEST['AUTHENTICATED_USER'])=='Anonymous User':	if str(self.REQUEST['AUTHENTICATED_USER'])=='Anonymous User':
self.temp_folder.downloadCounterBaskets-=1
self._p_changed=1
transaction.get().commit()
self.temp_folder.downloadCounterBaskets-=1
self._p_changed=1
transaction.get().commit()
return "please login first"	return "please login first"

#check if a locked object exist in the basket.	#check if a locked object exist in the basket.
lockedObjects={}	lockedObjects={}
for object in self.content.getContent():	for object in self.content.getContent():
	obj=self.getFileObject(object[0])
if (not str(object[1].lockedBy)=="") and (not (str(object[1].lockedBy)==str(self.REQUEST['AUTHENTICATED_USER']))):	if (not str(obj.lockedBy)=="") and (not (str(obj.lockedBy)==str(self.REQUEST['AUTHENTICATED_USER']))):
lockedObjects[object[1].title]=repr(object[1].lockedBy)	lockedObjects[obj.title]=repr(obj.lockedBy)


keys=lockedObjects.keys()	keys=lockedObjects.keys()
Line 1338 class CDLIBasketVersion(Implicit,Persist	Line 1496 class CDLIBasketVersion(Implicit,Persist
self.REQUEST.SESSION['lockedObjects']=lockedObjects	self.REQUEST.SESSION['lockedObjects']=lockedObjects
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','lockedObjects.zpt')).__of__(self)	pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','lockedObjects.zpt')).__of__(self)

self.temp_folder.downloadCounterBaskets-=1
self._p_changed=1
transaction.get().commit()

return pt()	return pt()

Line 1350 class CDLIBasketVersion(Implicit,Persist	Line 1505 class CDLIBasketVersion(Implicit,Persist



	threadName=repeat
	if not threadName or threadName=="":
	thread=DownloadBasketFinallyThread()
	threadName=thread.getName()[0:]

	if (not hasattr(self,'_v_downloadBasket')):
	self._v_downloadBasket={}


	self._v_downloadBasket[threadName]=thread
	logging.debug("dwonloadfinally:"+repr(self))

	if isinstance(self,CDLIBasketVersion):
	obj=self
	else:
	obj=self.aq_parent
	logging.debug("dwonloadfinally2:"+repr(obj))
	logging.debug("dwonloadfinally2:"+repr(obj.aq_parent))

	obj2=obj.aq_parent
	if not isinstance(obj2,CDLIBasket):
	obj2=obj2.aq_parent

	basketID=obj2.getId()
	versionNumber=obj.getId()
	logging.debug("dwonloadfinally2:"+repr(basketID))
	logging.debug("dwonloadfinally2:"+repr(versionNumber))


	if lock:
	logging.debug("-----start locking")
for object in self.content.getContent():	for object in self.content.getContent():
	obj=self.ctx.getFileObject(object[0])
	if obj.lockedBy =='':
	obj.lockedBy=self.REQUEST['AUTHENTICATED_USER']
	logging.debug("-----finished locking")

	#obj.lockedBy=user
	self._v_downloadBasket[threadName].set(lock,procedure,self.REQUEST['AUTHENTICATED_USER'],current,basketID,versionNumber)

	self._v_downloadBasket[threadName].start()


if (procedure=="downloadAll") or (object[1].lockedBy=='') or (object[1].lockedBy==self.REQUEST['AUTHENTICATED_USER']):
if current=="no": #version as they are in the basket
ret+=str(object[0].getData())+"\n"
elif current=="yes":
#search current object
founds=self.CDLICatalog.search({'title':object[1].getId().split(".")[0]})
if len(founds)>0:
ret+=str(founds[0].getObject().getLastVersion().getData())+"\n"

if lock and object[1].lockedBy=='':	wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
object[1].lockedBy=self.REQUEST['AUTHENTICATED_USER']
	if wait_template:
	return wait_template[0][1]()
	pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadBasketWait.zpt')).__of__(self)

	return pt(txt=self.absolute_url()+'/downloadObjectsAsOneFileFinally',threadName=threadName,
	counter=self._v_downloadBasket[threadName].getCounter(),
	number=self._v_downloadBasket[threadName].getNumberOfFiles())
	#_v_xmltrans.run()

	else:
	#recover thread, if lost
	if not hasattr(self,'_v_downloadBasket'):
	self._v_downloadBasket={}
	if not self._v_downloadBasket.get(threadName,None):
	for thread in threading.enumerate():
	if threadName == thread.getName():
	self._v_downloadBasket[threadName]=thread

	if self._v_downloadBasket.get(threadName,None) and (self._v_downloadBasket[threadName] is not None) and (not self._v_downloadBasket[threadName].end) :

	wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template'])
	if wait_template:
	return wait_template[0][1]()

	pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadBasketWait.zpt')).__of__(self)
	return pt(txt=self.absolute_url()+'/downloadObjectsAsOneFileFinally',threadName=threadName,
	counter=self._v_downloadBasket[threadName].getCounter(),
	number=self._v_downloadBasket[threadName].getNumberOfFiles())
	else:


	logging.debug("FINISHED")
	if not self._v_downloadBasket.get(threadName,None):
	for thread in threading.enumerate():
	if threadName == thread.getName():
	self._v_downloadBasket[threadName]=thread

	#files = self._v_downloadBasket[threadName].result
	# lade die files und die locked files, bei grossen Baskets muss u.U. gewartet werden
	# bis das Commit aus dem Thread alles geschrieben hat, in dem Falle existiert resultHash[threadName]
	# noch nicht.
	o1 = file("/tmp/"+threadName,'r')
	files=pickle.load(o1)
	os.remove("/tmp/"+threadName)
	o2 = file("/tmp/"+threadName+'_lockedFiles','r')

	lockedFiles=pickle.load(o2)
	os.remove("/tmp/"+threadName+'_lockedFiles')
	# try:
	# files=self.basketContainer.resultHash[threadName]
	# except:
	# i=0
	# while (not self.basketContainer.resultHash.has_key(threadName)) and (i<100):
	# logging.debug(" downloadFinally: I am waiting for thread %s to write the resultHashfile: %s"%(threadName,i))
	# time.sleep(5)
	# i+=1
	# files=self.basketContainer.resultHash[threadName]
	#
	# try:
	# lockedFiles=self.basketContainer.resultLockedHash[threadName]
	# except:
	# i=0
	# while (not self.basketContainer.resultLockedHash.has_key(threadName)) and (i<100):
	# logging.debug(" downloadFinally: I am waiting for thread %s to write the LockedHashfile: %s"%(threadName,i))
	# time.sleep(5)
	# i+=1
	# lockedFiles=self.basketContainer.resultLockedHash[threadName]

	# fh=file("/var/tmp/test")
	#ret =fh.read()

	if (not isinstance(self.aq_parent,CDLIBasket)):
	basket_name=self.aq_parent.aq_parent.title+"_V"+self.getId()
	else:
basket_name=self.aq_parent.title+"_V"+self.getId()	basket_name=self.aq_parent.title+"_V"+self.getId()



#write basketname to header of atf file	#write basketname to header of atf file
ret="#basket: %s\n"%basket_name+ret

self.temp_folder.downloadCounterBaskets-=1
self._p_changed=1
transaction.get().commit()

self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.atf" """%basket_name)	self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.atf" """%basket_name)
self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream")	self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream")
length=len(ret)	#length=len(ret)
self.REQUEST.RESPONSE.setHeader("Content-Length",length)	#self.REQUEST.RESPONSE.setHeader("Content-Length",length)

	ret="#basket: %s\n"%basket_name
self.REQUEST.RESPONSE.write(ret)	self.REQUEST.RESPONSE.write(ret)
return True
	for fileName in files:
	logging.debug("download: %s"%fileName)
	try:
	self.REQUEST.RESPONSE.write(file(fileName).read())
	except:
	logging.error("downloadasonefile: cannot read %s"%fileName)


	self.REQUEST.RESPONSE.write("\n# locked files\n")
	for fileName in lockedFiles:
	self.REQUEST.RESPONSE.write("# %s by %s\n"%fileName)

	self.REQUEST.RESPONSE.write("# locked files end\n")

	del self.basketContainer.resultHash[threadName]
	del self.basketContainer.resultLockedHash[threadName]

def numberOfItems(self):	def numberOfItems(self):
"""return anzahl der elemente im basket"""	"""return anzahl der elemente im basket"""
Line 1397 class CDLIBasketVersion(Implicit,Persist	Line 1675 class CDLIBasketVersion(Implicit,Persist

def getContent(self):	def getContent(self):
"""get Basket Content"""	"""get Basket Content"""
return self.content.getContent()	logging.debug("retrieving content A")
	cnt = self.content
	logging.debug("retrieving content: obj %s"%cnt)
	tmp = self.content.getContent()
	logging.debug("got content")
	return tmp


def __init__(self,id,user,comment="",basketContent=[]):	def __init__(self,id,user,comment="",basketContent=[]):
Line 1420 class CDLIBasketVersion(Implicit,Persist	Line 1703 class CDLIBasketVersion(Implicit,Persist
security.declareProtected('manage','index_html')	security.declareProtected('manage','index_html')
def index_html(self):	def index_html(self):
"""view the basket"""	"""view the basket"""
	logging.debug("start index_html - Basket version")
if self.REQUEST.get('change',False):	if self.REQUEST.get('change',False):
ob=self.aq_parent.updateObjects(self.REQUEST['change'])	ob=self.aq_parent.updateObjects(self.REQUEST['change'])

self.REQUEST.RESPONSE.redirect(ob.absolute_url())#go to new basket, because changing generates a new basket	self.REQUEST.RESPONSE.redirect(ob.absolute_url())#go to new basket, because changing generates a new basket
	logging.debug("start index_html - Basket version:template")
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketVersionMain.zpt')).__of__(self)	pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketVersionMain.zpt')).__of__(self)
return pt()	return pt()

def getObjUrl(self,result):	def getObjUrl(self,result):
"""getUrl of the version of the object"""	"""getUrl of the version of the object"""
objId=result[1].getTitle()
founds=self.CDLICatalog.search({'title':objId})	founds=self.CDLICatalog.search({'title':result})
if len(founds)>0:	if len(founds)>0:
return founds[0].getObject().getLastVersion().absolute_url()	return founds[0].getObject().getLastVersion().absolute_url()

else: #assume version number	else: #assume version number
splitted=objId.split("_")	splitted=result.split("_")
founds=self.CDLICatalog.search({'title':splitted[1]})	founds=self.CDLICatalog.search({'title':splitted[1]})
return founds[0].getObject().getLastVersion().absolute_url()+'/'+objId	return founds[0].getObject().getLastVersion().absolute_url()+'/'+result

def manage_addCDLIBasketVersion(self,user,comment="",basketContent=[],RESPONSE=None):	def manage_addCDLIBasketVersion(self,user,comment="",basketContent=[],RESPONSE=None):
"""add a version"""	"""add a version"""
Line 1471 class CDLIFileObject(CatalogAware,extVer	Line 1754 class CDLIFileObject(CatalogAware,extVer

security.declareProtected('manage','index_html')	security.declareProtected('manage','index_html')

	security.declarePublic('view')
	view = PageTemplateFile('zpt/viewCDLIFile.zpt', globals())

	security.declarePublic('editATF')
	editATF = PageTemplateFile('zpt/editATFFile.zpt', globals())

def PrincipiaSearchSource(self):	def PrincipiaSearchSource(self):
"""Return cataloguable key for ourselves."""	"""Return cataloguable key for ourselves."""
return str(self)	return str(self)

	def setAuthor(self, author):
	"""change the author"""
	self.author = author

def makeThisVersionCurrent_html(self):	def makeThisVersionCurrent_html(self):
"""form for making this version current"""	"""form for mthis version current"""

pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','makeThisVersionCurrent.zpt')).__of__(self)	pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','makeThisVersionCurrent.zpt')).__of__(self)
return pt()	return pt()
Line 1485 class CDLIFileObject(CatalogAware,extVer	Line 1778 class CDLIFileObject(CatalogAware,extVer
def makeThisVersionCurrent(self,comment,author,RESPONSE=None):	def makeThisVersionCurrent(self,comment,author,RESPONSE=None):
"""copy this version to current"""	"""copy this version to current"""
parent=self.aq_parent	parent=self.aq_parent
newversion=parent.manage_addCDLIFileObject('',comment,author)	parent.manage_addVersionedFileObject(id=None,vC=comment,author=author,file=self.getData(),RESPONSE=RESPONSE)
newversion.manage_upload(self.getData())	#newversion=parent.manage_addCDLIFileObject('',comment,author)
	#newversion.manage_upload(self.getData())

if RESPONSE is not None:	#if RESPONSE is not None:
RESPONSE.redirect(self.aq_parent.absolute_url()+'/history')	# RESPONSE.redirect(self.aq_parent.absolute_url()+'/history')

return True	return True

Line 1499 class CDLIFileObject(CatalogAware,extVer	Line 1793 class CDLIFileObject(CatalogAware,extVer
# return re.sub("\s\#lem"," #lem",data) #remove return vor #lem	# return re.sub("\s\#lem"," #lem",data) #remove return vor #lem
return re.sub("#lem"," #lem",data) #remove return vor #lem	return re.sub("#lem"," #lem",data) #remove return vor #lem

security.declarePublic('view')
def view(self):
"""view file"""
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','viewCDLIFile.zpt')).__of__(self)
return pt()

security.declarePublic('getPNumber')	security.declarePublic('getPNumber')
def getPNumber(self):	def getPNumber(self):
Line 1533 class CDLIFileObject(CatalogAware,extVer	Line 1822 class CDLIFileObject(CatalogAware,extVer
except:	except:
return "ERROR"	return "ERROR"


manage_addCDLIFileObjectForm=DTMLFile('dtml/fileAdd', globals(),Kind='CDLIFileObject',kind='CDLIFileObject', version='1')	manage_addCDLIFileObjectForm=DTMLFile('dtml/fileAdd', globals(),Kind='CDLIFileObject',kind='CDLIFileObject', version='1')

def manage_addCDLIFileObject(self,id,vC='',author='', file='',title='',versionNumber=0,	def manage_addCDLIFileObject(self,id,vC='',author='', file='',title='',versionNumber=0,
Line 1565 def manage_addCDLIFileObject(self,id,vC=	Line 1855 def manage_addCDLIFileObject(self,id,vC=
if content_type:	if content_type:
fob.content_type=content_type	fob.content_type=content_type

logging.debug("manage_add: lastversion=%s"%self.getData())	#logging.debug("manage_add: lastversion=%s"%self.getData())
logging.debug("reindex1: %s in %s"%(repr(self),repr(self.default_catalog)))	logging.debug("reindex1: %s in %s"%(repr(self),repr(self.default_catalog)))
self.reindex_object()	self.reindex_object()
logging.debug("manage_add: fob_data=%s"%fob.getData())	#logging.debug("manage_add: fob_data=%s"%fob.getData())
logging.debug("reindex2: %s in %s"%(repr(fob), repr(fob.default_catalog)))	logging.debug("reindex2: %s in %s"%(repr(fob), repr(fob.default_catalog)))
fob.index_object()	fob.index_object()

	self.CDLIRoot.updateOrAddToFileBTree(ob)
if REQUEST is not None:	if REQUEST is not None:
REQUEST['RESPONSE'].redirect(self.absolute_url()+'/manage_main')	REQUEST['RESPONSE'].redirect(self.absolute_url()+'/manage_main')

Line 1584 class CDLIFile(extVersionedFile,CatalogA	Line 1875 class CDLIFile(extVersionedFile,CatalogA
content_meta_type = ["CDLI File Object"]	content_meta_type = ["CDLI File Object"]

default_catalog='CDLICatalog'	default_catalog='CDLICatalog'

security.declareProtected('manage','index_html')	security.declareProtected('manage','index_html')

def getLastVersionData(self):	def getLastVersionData(self):
Line 1594 class CDLIFile(extVersionedFile,CatalogA	Line 1886 class CDLIFile(extVersionedFile,CatalogA
"""get last version data"""	"""get last version data"""
return self.getContentObject().getFormattedData()	return self.getContentObject().getFormattedData()

	def getTextId(self):
	"""returns P-number of text"""
	# assuming that its the beginning of the title
	return self.title[:7]

#security.declarePublic('history')	#security.declarePublic('history')
def history(self):	def history(self):
"""history"""	"""history"""
Line 1641 class CDLIFile(extVersionedFile,CatalogA	Line 1938 class CDLIFile(extVersionedFile,CatalogA

def _newContentObject(self, id, title='', versionNumber=0, versionComment=None, time=None, author=None):	def _newContentObject(self, id, title='', versionNumber=0, versionComment=None, time=None, author=None):
"""factory for content objects. to be overridden in derived classes."""	"""factory for content objects. to be overridden in derived classes."""
	logging.debug("_newContentObject(CDLI)")
return CDLIFileObject(id,title,versionNumber=versionNumber,versionComment=versionComment,time=time,author=author)	return CDLIFileObject(id,title,versionNumber=versionNumber,versionComment=versionComment,time=time,author=author)


Line 1678 class CDLIFile(extVersionedFile,CatalogA	Line 1976 class CDLIFile(extVersionedFile,CatalogA
except:	except:
pass	pass

	#self.cdliRoot.updateOrAddToFileBTree(self)# now update the object in the cache


if RESPONSE:	if RESPONSE:
if ob.getSize()==0:	if ob.getSize()==0:
self.REQUEST.SESSION['objID']=ob.getId()	self.REQUEST.SESSION['objID']=ob.getId()
Line 1710 def manage_addCDLIFile(self,id,title,loc	Line 2011 def manage_addCDLIFile(self,id,title,loc
if RESPONSE is not None:	if RESPONSE is not None:
RESPONSE.redirect('manage_main')	RESPONSE.redirect('manage_main')


def checkUTF8(data):	def checkUTF8(data):
"""check utf 8"""	"""check utf 8"""
try:	try:
Line 1740 def splitatf(fh,dir=None,ext=None):	Line 2042 def splitatf(fh,dir=None,ext=None):
nf=None	nf=None
i=0	i=0

	#ROC: why split \n first and then \r???
if (type(fh) is StringType) or (type(fh) is UnicodeType):	if (type(fh) is StringType) or (type(fh) is UnicodeType):
iter=fh.split("\n")	iter=fh.split("\n")
else:	else:
Line 1796 class CDLIFileFolder(extVersionedFileFol	Line 2099 class CDLIFileFolder(extVersionedFileFol
file_meta_type=['CDLI file']	file_meta_type=['CDLI file']
folder_meta_type=['CDLI Folder']	folder_meta_type=['CDLI Folder']

default_catalog='CDLICatalog'	file_catalog='CDLICatalog'
defaultFileCatalog=default_catalog #wenn dieses definiert ist, wird beim hinzufuegen einer neuen version eines files dieser catalog neuindiziert
#downloadCounter=0 # counts how many download for all files currently run, be mehr als 5 wird verweigert.	#downloadCounter=0 # counts how many download for all files currently run, be mehr als 5 wird verweigert.
tmpStore2={}	tmpStore2={}

	def _newVersionedFile(self, id, title='', lockedBy=None, author=None):
	"""factory for versioned files. to be overridden in derived classes."""
	logging.debug("_newVersionedFile(CDLI)")
	return CDLIFile(id, title, lockedBy=lockedBy, author=author)

def setTemp(self,name,value):	def setTemp(self,name,value):
"""set tmp"""	"""set tmp"""

setattr(self,name,value)	setattr(self,name,value)

	deleteFileForm = PageTemplateFile("zpt/doDeleteFile", globals())

def delete(self,ids):	def delete(self,ids,REQUEST=None):
"""delete these files"""	"""delete these files"""
if type(ids) is not ListType:	if type(ids) is not ListType:
ids=[ids]	ids=[ids]

self.manage_delObjects(ids)	self.manage_delObjects(ids)

	if REQUEST is not None:
	return self.index_html()


def getVersionNumbersFromIds(self,ids):	def getVersionNumbersFromIds(self,ids):
"""get the numbers of the current versions of documents described by their ids"""	"""get the numbers of the current versions of documents described by their ids"""
Line 1834 class CDLIFileFolder(extVersionedFileFol	Line 2146 class CDLIFileFolder(extVersionedFileFol
logging.debug("getFile: %s"%repr(fn))	logging.debug("getFile: %s"%repr(fn))
if not self.hasObject(fn):	if not self.hasObject(fn):
# search deeper	# search deeper
founds=self.CDLICatalog.search({'title':fn})	founds=getattr(self, self.file_catalog).search({'textid':fn})
if founds:	if founds:
obj=founds[0].getObject().getContentObject()	obj=founds[0].getObject().getContentObject()
else:	else:
Line 1869 class CDLIFileFolder(extVersionedFileFol	Line 2181 class CDLIFileFolder(extVersionedFileFol

return ret	return ret

def findObjectsFromList(self,enterList=None,display=False,start=None,upload=None,list=None,basketName=None,numberOfObjects=None,RESPONSE=None):	def expandFile(self,fileId,fileTree):
	"""wildcard in fileID suche alle Treffer"""
	founds=self.CDLICatalog({'title':fileId})
	for found in founds:
	fileTree.add(found.getId)
	logging.debug("ADDD:"+found.getId)

	def findObjectsFromList(self,enterList=None,display=False,start=None,upload=None,list=None,basketName=None,numberOfObjects=None,RESPONSE=None,REQUEST=None,returnHash=False,hash=None):
"""findObjectsFromList (, TAB oder LINE separated)"""	"""findObjectsFromList (, TAB oder LINE separated)"""

	logging.debug("start: findObjectsFromList")
	#logging.debug("start: findObjectsFromList"+repr(list))


if upload: # list from file upload	if upload: # list from file upload
txt=upload.read()	txt=upload.read()
Line 1903 class CDLIFileFolder(extVersionedFileFol	Line 2225 class CDLIFileFolder(extVersionedFileFol
pt=getattr(self,'filelist.html')	pt=getattr(self,'filelist.html')
return pt(basketName=basketName,numberOfObjects=numberOfObjects)	return pt(basketName=basketName,numberOfObjects=numberOfObjects)


	result =self.CDLICache.retrieve(hash)
	if result:
	logging.debug("give result from storage2")
	return hash,result

if list is not None: # got already a list	if list is not None: # got already a list

	logging.debug(" ----List version")
ret=[]	ret=[]
	fileTree=Set()

for fileId in list:	for fileId in list:
if fileId.find("*"): #check for wildcards
fileId=fileId	if fileId.find("*")>-1: #check for wildcards
	self.expandFile(fileId,fileTree)

elif len(fileId.split("."))==1:	elif len(fileId.split("."))==1:
fileId=fileId+".atf"	fileId=fileId+".atf"
	fileTree.add(fileId)
	#logging.debug(" -----:"+fileId)
	#ret+=self.CDLICatalog({'title':fileId})
	#x =self.getFileObject(fileId)
	#if x is not None:
	# ret.append(x)



	ids = fileTree & self.v_file_ids
	#self.REQUEST.SESSION['fileIds']=ids#store fieldIds in session for further usage
	l=makelist(fileTree)[0:]
	#logging.debug("l-list:"+repr(l))
	self.REQUEST.SESSION['fileIds']=l#store fieldIds in session for further usage
	self.REQUEST.SESSION['searchList']=l
	#self.REQUEST.SESSION['searchList']=['P000001.atf']


	hash = md5.new(repr(makelist(fileTree))).hexdigest() # erzeuge hash als identification
	self.REQUEST.SESSION['hash']=hash
	#TODO: do I need garbage collection for v_tmpStore ?

	#logging.debug("Hash:"+repr(hash))
	#
	# if hasattr(self.cdliRoot,'v_tmpStore') and self.cdliRoot.v_tmpStore.has_key(hash):
	# logging.debug("asking for storage")
	# res=self.cdliRoot.v_tmpStore[hash]
	# if res:
	# if returnHash == True:
	# return hash,res
	# return res

ret+=self.CDLICatalog({'title':fileId})
#TODO: get rid of one of these..	#TODO: get rid of one of these..
ids=[x.getObject().getId() for x in ret]	#ids=[x.getObject().getId() for x in ret]
self.REQUEST.SESSION['fileIds']=ids#store fieldIds in session for further usage	ret=[(self.getFileObject(x),self.getFileObjectLastVersion(x)) for x in ids]
self.REQUEST.SESSION['searchList']=self.REQUEST.SESSION['fileIds']
	#self.REQUEST.SESSION['fileIds']=ids#store fieldIds in session for further usage
	#self.REQUEST.SESSION['searchList']=self.REQUEST.SESSION['fileIds']

if display:	if display:
pt=getattr(self,'filelist.html')	pt=getattr(self,'filelist.html')

return pt(search=ids)	return pt(search=ids)
else:	else:
	#self.REQUEST.SESSION['hash'] = ret # store in session

	#logging.debug("HHHHHHNEU:"+repr(self.makelist(ids)))
	#logging.debug("HHHHHHNEU:"+repr(hash))
	self.CDLICache.store(hash,ret)

	if returnHash == True:
	return hash,ret
return ret	return ret


Line 1929 class CDLIFileFolder(extVersionedFileFol	Line 2303 class CDLIFileFolder(extVersionedFileFol
if start:	if start:
RESPONSE.redirect("filelist.html?start:int="+str(start))	RESPONSE.redirect("filelist.html?start:int="+str(start))


security.declareProtected('Manage','createAllFilesAsSingleFile')	security.declareProtected('Manage','createAllFilesAsSingleFile')
def createAllFilesAsSingleFile(self,RESPONSE=None):	def createAllFilesAsSingleFile(self,RESPONSE=None):
"""download all files"""	"""download all files"""
Line 1937 class CDLIFileFolder(extVersionedFileFol	Line 2310 class CDLIFileFolder(extVersionedFileFol
def sortF(x,y):	def sortF(x,y):
return cmp(x[0],y[0])	return cmp(x[0],y[0])

catalog=getattr(self,self.default_catalog)	catalog=getattr(self,self.file_catalog)
#tf,tfilename=mkstemp()	#tf,tfilename=mkstemp()
if not hasattr(self.temp_folder,'downloadCounter'):	if not hasattr(self.temp_folder,'downloadCounter'):
self.temp_folder.downloadCounter=0	self.temp_folder.downloadCounter=0
Line 2057 class CDLIRoot(Folder):	Line 2430 class CDLIRoot(Folder):
'graphemes':cdliSplitter.graphemeSplitter()}	'graphemes':cdliSplitter.graphemeSplitter()}


	def unicodify(self,txt):
	return unicodify(txt)
	def invalidateOldCacheVersion(self):
	"""loescht die alte Version des Cache"""
	del self.v_tmpStore
	return "done"

	def viewATF(self,id,RESPONSE):
	"""view an Object"""
	ob = self.CDLICatalog({'title':id})
	logging.debug(ob[0].getObject().getLastVersion().absolute_url()+"/view")
	if len(ob)>0:
	RESPONSE.redirect(ob[0].getObject().getLastVersion().absolute_url()+"/view")
	return "not found"

	def history(self,id,RESPONSE):
	"""view an Object"""
	ob = self.CDLICatalog({'title':id})
	if len(ob)>0:
	RESPONSE.redirect(ob[0].absolute_url+"/history")
	return "not found"


	def downloadLocked(self,id,RESPONSE):
	"""view an Object"""
	ob = self.CDLICatalog({'title':id})
	if len(ob)>0:
	RESPONSE.redirect(ob[0].absolute_url+"/downloadLocked")
	return "not found"

	def download(self,id,RESPONSE):
	"""view an Object"""
	ob = self.CDLICatalog({'title':id})
	if len(ob)>0:
	RESPONSE.redirect(ob[0].getLastVersion().absolute_url())
	return "not found"
	def addCDLIFileObjectForm(self,id,RESPONSE):
	"""view an Object"""
	ob = self.CDLICatalog({'title':id})
	if len(ob)>0:
	RESPONSE.redirect(ob[0].absolute_url+"/addCDLIFileObjectForm")
	return "not found"

	def addVersionedFileObjectForm(self,id,RESPONSE):
	"""view an Object"""
	ob = self.CDLICatalog({'title':id})
	if len(ob)>0:
	RESPONSE.redirect(ob[0].absolute_url+"/addVersionedFileObjectForm")
	return "not found"

	def unlock(self,id,RESPONSE):
	"""view an Object"""
	ob = self.CDLICatalog({'title':id})
	if len(ob)>0:
	RESPONSE.redirect(ob[0].absolute_url+"/unlock")
	return "not found"

	def getFileObject(self,fileId):
	"""get an object"""
	x=self.v_files.get(fileId)
	#logging.debug(x)
	return x

	def getFileObjectLastVersion(self,fileId):
	"""get an object"""
	x=self.v_files_lastVersion.get(fileId)
	#logging.debug("lastVersion: "+repr(x))
	return x

	def showFileIds(self):
	"""showIds"""
	return self.v_file_ids

	def generateFileBTree(self):
	"""erzeuge einen Btree aus allen Files"""
	self.v_files = OOBTree()
	self.v_files_lastVersion = OOBTree()
	self.v_file_ids = Set()

	for x in self.CDLICatalog.searchResults():

	self.v_files.update({x.getId:x.getObject()})
	self.v_files_lastVersion.update({x.getId:x.getObject().getLastVersion()})
	self.v_file_ids.add(x.getId)
	logging.debug("add:"+x.getId+"XXX"+repr(x.getObject()))

	return True


	def updateOrAddToFileBTree(self,obj):
	"""update a BTree"""
	self.v_files.update({obj.getId():obj})
	self.v_files_lastVersion.update({obj.getId():obj.getLastVersion()})

	self.v_file_ids.add(obj.getId())
	logging.debug("update:"+obj.getId()+"XXX"+repr(obj))

	def deleteFromBTree(self,objId):
	"""delete an obj"""
	self.v_files.pop(objId)
	self.v_files_lastVersion.pop(objId)
	self.v_file_ids.remove(objId)



def deleteFiles(self,ids):	def deleteFiles(self,ids):
"""delete files"""	"""delete files"""
for id in ids:	for id in ids:
Line 2069 class CDLIRoot(Folder):	Line 2547 class CDLIRoot(Folder):



def searchText(self, query, index='words'):	def searchText(self, query, index='graphemes'):
"""searches query in the fulltext index and returns a list of file ids/P-numbers"""	"""searches query in the fulltext index and returns a list of file ids/P-numbers"""
	# see also: http://www.plope.com/Books/2_7Edition/SearchingZCatalog.stx#2-13
	logging.debug("searchtext for '%s' in index %s"%(query,index))
	#import Products.ZCTextIndex.QueryParser
	#qp = QueryParser.QueryParser()
	#logging.debug()
idxQuery = {index:{'query':query}}	idxQuery = {index:{'query':query}}
idx = getattr(self, self.file_catalog)	idx = getattr(self, self.file_catalog)
results = []
# do search	# do search
resultset = idx.search(idxQuery)	resultset = idx.search(query_request=idxQuery,sort_index='textid')
for res in resultset:
# put only the P-Number in the result	# put only the P-Number in the result
results.append(res.getId[:7])	results = [res.getId[:7] for res in resultset]
	logging.debug("searchtext: found %d texts"%len(results))
return results	return results

# from PluginINdexes.common.util.py:parseIndexRequest:
#
# The class understands the following type of parameters:
#
# - old-style parameters where the query for an index as value inside
# the request directory where the index name is the name of the key.
# Additional parameters for an index could be passed as index+"_usage" ...
#
#
# - dictionary-style parameters specify a query for an index as
# an entry in the request dictionary where the key corresponds to the
# name of the index and the key is a dictionary with the parameters
# passed to the index.
#
# Allowed keys of the parameter dictionary:
#
# 'query' - contains the query (either string, list or tuple) (required)
#
# other parameters depend on the the index
#
#
# - record-style parameters specify a query for an index as instance of the
# Record class. This happens usually when parameters from a web form use
# the "record" type e.g. <input type="text" name="path.query:record:string">.
# All restrictions of the dictionary-style parameters apply to the record-style
# parameters

	def getFile(self, pnum):
	"""get the translit file with the given pnum"""
	f = getattr(self, self.file_catalog).search({'textid':pnum})
	if not f:
	return ""

	return f[0].getObject().getData()


def showFile(self,fileId,wholePage=False):	def showFile(self,fileId,wholePage=False):
"""show a file	"""show a file
@param fileId: P-Number of the document to be displayed	@param fileId: P-Number of the document to be displayed
"""	"""
f=self.CDLICatalog({'title':fileId})	f=getattr(self, self.file_catalog).search({'textid':fileId})
if not f:	if not f:
return ""	return ""

Line 2125 class CDLIRoot(Folder):	Line 2588 class CDLIRoot(Folder):
return f[0].getObject().getLastVersionFormattedData()	return f[0].getObject().getLastVersionFormattedData()


def showWordInFile(self,fileId,word,lineList=None,regExp=False,indexName=""):	def showWordInFile(self,fileId,word,indexName='graphemes',regExp=False,):
"""get lines with word from FileId"""	"""get lines with word from FileId"""
	logging.debug("showwordinfile word='%s' index=%s file=%s"%(word,indexName,fileId))

file=self.showFile(fileId)	file = formatAtfFullLineNum(self.getFile(fileId))
logging.debug("show word regEXP %s"%regExp)
ret=[]	ret=[]
if regExp: # wenn regexp dann generiere alle worte aus der list die der regexp entsprechen
wordlist=self.findWordRegExp(indexName,word)
else:
wordlist=[word]

for line in file.split("\n"):	# add whitespace before and whitespace and line-end to splitter bounds expressions
line = formatAtfLineHtml(unicodify(line))	bounds = self.splitter[indexName].bounds
found=False	splitexp = "(%s\|\s)(%%s)(%s\|\s\|\Z)"%(bounds,bounds)
	# clean word expression
	# TODO: this should use QueryParser itself
	# take out double quotes
	word = word.replace('"','')
	# take out ignorable signs
	ignorable = self.splitter[indexName].ignorex
	word = ignorable.sub('', word)
	# compile into regexp objects and escape parens
	wordlist = [re.compile(splitexp%re.escape(w)) for w in word.split(' ')]

	for line in file.splitlines():
for word in wordlist:	for word in wordlist:
try: # just a hack because of possible unicode errors in line	#logging.debug("showwordinfile: searching for %s in %s"%(word.pattern,ignoreable.sub('',line)))
if line.find(word)>-1:	if word.search(ignorable.sub('',line)):
if lineList: #liste of moeglichen Zeilennummern	line = formatAtfLineHtml(line)
num=line.split(".")[0] #Zeilenummer ist alles vor dem . in der Zeile
if num in lineList:
ret.append(line)
else: # nimm alles ohne line check
ret.append(line)	ret.append(line)
break;	break
except:
pass
return ret	return ret


def tagWordInFile(self,fileId,word,indexName='words',regExp=False):	def showWordInFiles(self,fileIds,word,indexName='graphemes',regExp=False):
	"""
	get lines with word from all ids in list FileIds.
	returns dict with id:lines pairs.
	"""
	logging.debug("showwordinfiles word='%s' index=%s file=%s"%(word,indexName,fileIds))

	return dict([(id,self.showWordInFile(id, word, indexName, regExp)) for id in fileIds])


	def tagWordInFile(self,fileId,word,indexName='graphemes',regExp=False):
"""get text with word highlighted from FileId"""	"""get text with word highlighted from FileId"""
	logging.debug("tagwordinfile word='%s' index=%s file=%s"%(word,indexName,fileId))

file=self.showFile(fileId)	file=self.getFile(fileId)
tagStr=u'<span class="found">%s</span>'	tagStart=u'<span class="found">'
	tagEnd=u'</span>'
	tagStr=tagStart + u'%%s' + tagEnd
ret=[]	ret=[]
# search using lowercase
word = word.lower()

if regExp: # wenn regexp dann generiere alle worte aus der list die der regexp entsprechen	# add whitespace to splitter bounds expressions and compile into regexp object
wordlist=self.findWordRegExp(indexName,word)	bounds = self.splitter[indexName].bounds
else:	wordsplit = re.compile("(%s\|\s)"%bounds)
# split the search term into words according to the corresponding splitter	# clean word expression
#try:	# TODO: this should use QueryParser itself
wordlist = self.splitter[indexName].process([word])	word = word.replace('"','') # take out double quotes
#except:	# take out ignoreable signs
# wordlist=[word]	ignorable = self.splitter[indexName].ignorex
	word = ignorable.sub('', word)
for line in file.split("\n"):	# split search terms by blanks
line = formatAtfLineHtml(unicodify(line))	words = word.split(' ')
if not line:	# split search terms again (for grapheme search with words)
# formatAtf can produce empty lines	splitwords = dict(((w,self.splitter[indexName].process([w])) for w in words))

	for line in file.splitlines():
	line = unicodify(line)
	# ignore lemma and other lines
	if line.lstrip().startswith('#lem:'):
	continue
	# ignore p-num line
	if line.startswith('&P'):
	continue
	# ignore version lines
	if line.startswith('#version'):
continue	continue
	# ignore atf type lines
	if line.startswith('#atf:'):
	continue

	# first scan
	hitwords = []
	for w in words:
	if ignorable.sub('',line).find(w) > -1:
	# word is in line
	# append split word for grapheme search with words
	hitwords.extend(splitwords[w])
	#hitwords.extend(wordsplit.split(w))

	# examine hits closer
	if hitwords:
	# split line into words
	parts = wordsplit.split(line)
	line = ""
	for p in parts:
	#logging.debug("tagwordinfile: searching for %s in %s"%(p,hitwords))
	# reassemble line
	if ignorable.sub('', p) in hitwords:
	#logging.debug("tagwordinfile: found %s in %s"%(p,hitwords))
	# this part was found
	line += tagStart + formatAtfHtml(p) + tagEnd
	else:
	line += formatAtfHtml(p)

for w in wordlist:	else:
if line.lower().find(w)>-1:	# no hits
#word ist gefunden dann makiere	line = formatAtfHtml(line)
line = line.replace(w,tagStr%w)

ret.append(line)	ret.append(line)

return u'<br>\n'.join(ret)	return u'<br>\n'.join(ret)



	def tagWordInFiles(self,fileIds,word,indexName='graphemes',regExp=False):
	"""
	get texts with highlighted word from all ids in list FileIds.
	returns dict with id:text pairs.
	"""
	logging.debug("tagwordinfiles word='%s' index=%s file=%s"%(word,indexName,fileIds))
	return dict([(id,self.tagWordInFile(id, word, indexName, regExp)) for id in fileIds])


	def getFileVersionList(self, pnum):
	"""get the version history as a list for the translit file with the given pnum"""
	f = getattr(self, self.file_catalog).search({'textid':pnum})
	if not f:
	return []

	return f[0].getObject().getVersionList()


def URLquote(self,str):	def URLquote(self,str):
"""quote url"""	"""quote url"""
return urllib.quote(str)	return urllib.quote(str)
Line 2259 class CDLIRoot(Folder):	Line 2792 class CDLIRoot(Folder):

return """ 2;url=%s?repeat=%s """%(self.absolute_url()+txt,threadName)	return """ 2;url=%s?repeat=%s """%(self.absolute_url()+txt,threadName)

	def refreshTxtBasket(self,txt="",threadName=None):
	"""txt fuer refresh"""

	return """ 2;url=%s?repeat=%s """%(txt,threadName)


def getResult(self,threadName=None):	def getResult(self,threadName=None):
"""result of thread"""	"""result of thread"""
Line 2435 class CDLIRoot(Folder):	Line 2973 class CDLIRoot(Folder):
pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self)	pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self)
return pt(txt='/uploadATFfinally',threadName=threadName)	return pt(txt='/uploadATFfinally',threadName=threadName)
else:	else:


	idTmp=self.REQUEST.SESSION['idTmp']
	stObj=getattr(self.temp_folder,idTmp)
self.REQUEST.SESSION['idTmp']=None	self.REQUEST.SESSION['idTmp']=None

	#update changed
	logging.debug("dir:"+repr(stObj.returnValue['changed']))
	for x in stObj.returnValue['changed']:
	ob=self.CDLICatalog.search({'title':x[0]})

	self.cdliRoot.updateOrAddToFileBTree(ob[0].getObject())
if RESPONSE is not None:	if RESPONSE is not None:
RESPONSE.redirect(self.absolute_url())	RESPONSE.redirect(self.absolute_url())

def importFiles(self,comment="",author="" ,folderName="/Users/dwinter/atf", files=None,ext=None):	def importFiles(self,comment="",author="" ,folderName="/Users/dwinter/atf", files=None,ext=None):
"""import files"""	"""import files"""
	logging.debug("importFiles folderName=%s files=%s ext=%s"%(folderName,files,ext))
root=self.cdli_main	root=self.cdli_main
count=0	count=0
if not files:	if not files:
Line 2449 class CDLIRoot(Folder):	Line 2999 class CDLIRoot(Folder):
for f in files:	for f in files:
folder=f[0:3]	folder=f[0:3]
f2=f[0:5]	f2=f[0:5]

	#check if main folder PXX already exists
obj=self.ZopeFind(root,obj_ids=[folder])	obj=self.ZopeFind(root,obj_ids=[folder])
	logging.debug("importFiles: folder=%s f2=%s obj=%s"%(folder,f2,obj))
if ext:	if ext:

ext.result="<p>adding: %s </p>"%f+ext.result	ext.result="<p>adding: %s </p>"%f+ext.result
if not obj:

	if not obj: # if not create it
manage_addCDLIFileFolder(root,folder,folder)	manage_addCDLIFileFolder(root,folder,folder)
fobj=getattr(root,folder)	fobj=getattr(root,folder)
#transaction.get().commit()	#transaction.get().commit()

else:	else:
fobj=obj[0][1]	fobj=obj[0][1]

	#check IF PYYYYY already exist
obj2=fobj.ZopeFind(fobj,obj_ids=[f2])	obj2=fobj.ZopeFind(fobj,obj_ids=[f2])
	logging.debug("importFiles: fobj=%s obj2=%s"%(fobj,obj2))

if not obj2:	if not obj2:# if not create it
manage_addCDLIFileFolder(fobj,f2,f2)	manage_addCDLIFileFolder(fobj,f2,f2)
fobj2=getattr(fobj,f2)	fobj2=getattr(fobj,f2)

else:	else:
fobj2=obj2[0][1]	fobj2=obj2[0][1]

	# not add the file
file2=os.path.join(folderName,f)	file2=os.path.join(folderName,f)
id=f	id=f
manage_addCDLIFile(fobj2,f,'','')	logging.debug("importFiles: addCDLIFile fobj2=%s, f=%s file2=%s"%(fobj2,repr(f),repr(file2)))
id=f	fobj2.addFile(vC='',file=file(file2),author=author,newName=f)
ob=fobj2._getOb(f)
ob.title=id

manage_addCDLIFileObject(ob,id,comment,author,file2,content_type='',from_tmp=True)
self.CDLICatalog.catalog_object(ob)
#self.CDLICatalog.manage_catalogFoundItems(obj_ids=[id],search_sub=1)
#self.CDLICatalog.manage_catalogObject(self.REQUEST, self.REQUEST.RESPONSE, 'CDLICatalog', urlparse.urlparse(ob.absolute_url())[1])
count+=1	count+=1

if count > 1000:	#now add the file to the storage
print "committing"	ob = getattr(fobj2,f)
	self.cdliRoot.updateOrAddToFileBTree(ob)

	if count%100==0:
	logging.debug("importfiles: committing")
transaction.get().commit()	transaction.get().commit()
count=0
transaction.get().commit()	transaction.get().commit()
return "ok"	return "ok"

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>

Removed from v.1.80.2.4
changed lines
	Added in v.1.103