cdli/cdli_files.py - diff

Return to cdli_files.py CVS log

Up to [Repository] / cdli

Diff for /cdli/cdli_files.py between versions 1.72 and 1.79

version 1.72, 2007/03/22 19:56:22	version 1.79, 2007/08/31 14:22:52
Line 26 import logging	Line 26 import logging
import transaction	import transaction
import copy	import copy
import codecs	import codecs
	import sys

	def unicodify(s):
	"""decode str (utf-8 or latin-1 representation) into unicode object"""
	if not s:
	return u""
	if isinstance(s, str):
	try:
	return s.decode('utf-8')
	except:
	return s.decode('latin-1')
	else:
	return s

	def utf8ify(s):
	"""encode unicode object or string into byte string in utf-8 representation.
	assumes string objects to be utf-8"""
	if not s:
	return ""
	if isinstance(s, str):
	return s
	else:
	return s.encode('utf-8')


	def generateXMLReturn(hash):
	"""erzeugt das xml file als returnwert fuer uploadATFRPC"""

	ret="<return>"

	ret+="<errors>"
	for error in hash['errors']:
	ret+="""<error atf="%s">%s</error>"""%error

	ret+="</errors>"

	ret+="<changes>"
	for changed in hash['changed']:
	ret+="""<change atf="%s">%s</change>"""%changed
	ret+="</changes>"

	ret+="<newPs>"
	for new in hash['newPs']:
	ret+="""<new atf="%s"/>"""%new
	ret+="</newPs>"

	ret+="</return>"
	return ret










def unique(s):	def unique(s):
"""Return a list of the elements in s, but without duplicates.	"""Return a list of the elements in s, but without duplicates.
Line 196 class uploadATFfinallyThread(Thread):	Line 253 class uploadATFfinallyThread(Thread):
self.result+="<h2>Start processing</h2>"	self.result+="<h2>Start processing</h2>"

#shall I only upload the changed files?	#shall I only upload the changed files?
	logging.info("uploadATFfinally procedure: %s"%procedure)
if procedure=="uploadchanged":	if procedure=="uploadchanged":
changed=[x[0] for x in SESSION.get('changed',[])]	changed=[x[0] for x in SESSION.get('changed',[])]
uploadFns=changed+SESSION.get('newPs',[])	uploadFns=changed+SESSION.get('newPs',[])
Line 341 class uploadATFThread(Thread):	Line 399 class uploadATFThread(Thread):
ctx = self.getContext(app,serverport=self.serverport)	ctx = self.getContext(app,serverport=self.serverport)
logging.info("run intern")	logging.info("run intern")
try:	try:
	logging.info("created: %s"%idTmp)
ctx.temp_folder._setObject(idTmp,tmpStore(idTmp))	ctx.temp_folder._setObject(idTmp,tmpStore(idTmp))
except:	except:
logging.error("thread upload: %s %s"%sys.exc_info()[0:2])	logging.error("thread upload: %s %s"%sys.exc_info()[0:2])
Line 355 class uploadATFThread(Thread):	Line 414 class uploadATFThread(Thread):

conn.close()	conn.close()

	return getattr(ctx.temp_folder,idTmp)

def getResult(self):	def getResult(self):
"""method for accessing result"""	"""method for accessing result"""
Line 479 class CDLIBasketContainer(OrderedFolder)	Line 539 class CDLIBasketContainer(OrderedFolder)
security=ClassSecurityInfo()	security=ClassSecurityInfo()
meta_type="CDLIBasketContainer"	meta_type="CDLIBasketContainer"

	def getPNumbersOfBasket(self,basketName):
	"""get all pnumbers of a basket as a list, returns an empty list if basket not found
	@param basketName: name of the basket
	"""
	ret=[]
	basketId=self.getBasketIdfromName(basketName)
	if not basketId:
	return []

	ob=getattr(self,basketId).getContent()

	ret=[x[0].split(".")[0] for x in ob]

	return ret

	security.declareProtected('manage','getBasketAsOneFile')
	def getBasketAsOneFile(self,basketName,current="no"):
	"""returns all files of the basket combined in one file
	@param basketName: Name of the basket
	@param current: (optional) if current is set to "yes" then the most current version of
	all files are downloaded and not the versions of the files as stored in the basket
	"""
	ret=""
	basketId=self.getBasketIdfromName(basketName)
	if not basketId:
	return ""

	ob=getattr(self,basketId).getLastVersion()
	for object in ob.getContent():
	if current=="no": #version as they are in the basket
	ret+=str(object[0].getData())+"\n"
	elif current=="yes":
	#search current object
	logging.info("crrent: %s"%object[1].getId().split(".")[0])
	founds=self.CDLICatalog.search({'title':object[1].getId().split(".")[0]})
	if len(founds)>0:
	ret+=str(founds[0].getObject().getLastVersion().getData())+"\n"
	return ret

	security.declareProtected('manage','upDateBaskets')
def upDateBaskets(self):	def upDateBaskets(self):
"""update content in to objects"""	"""update content in to objects"""

Line 506 class CDLIBasketContainer(OrderedFolder)	Line 606 class CDLIBasketContainer(OrderedFolder)
trash.manage_pasteObjects(cut)	trash.manage_pasteObjects(cut)

security.declareProtected('manage','manageBaskets')	security.declareProtected('manage','manageBaskets')
def manageBaskets(self,submit,ids=None,basket1="",basket2="",REQUEST=None,RESPONSE=None):	def manageBaskets(self,submit,ids=None,basket1="",basket2="",joinBasket="",subtractBasket="",REQUEST=None,RESPONSE=None):
"""manage baskets, delete or copy"""	"""manage baskets, delete or copy"""
if submit=="delete":	if submit=="delete":
self.deleteBaskets(ids)	self.deleteBaskets(ids)

elif submit=="join":	elif submit=="join":
flag,msg=self.joinBasket("joinedBasket", ids)	flag,msg=self.joinBasket(joinBasket, ids)
logging.info("joining %s %s"%(flag,msg))	logging.info("joining %s %s"%(flag,msg))

elif submit=="subtract":	elif submit=="subtract":
logging.info("BBBb %s %s"%(basket1,basket2))	logging.info("BBBb %s %s"%(basket1,basket2))
flag,msg=self.subtractBasket("subtractedBasket", basket1,basket2)	flag,msg=self.subtractBasket(subtractBasket, basket1,basket2)
logging.info("subtract %s %s"%(flag,msg))	logging.info("subtract %s %s"%(flag,msg))

if RESPONSE:	if RESPONSE:
Line 549 class CDLIBasketContainer(OrderedFolder)	Line 649 class CDLIBasketContainer(OrderedFolder)
return pt(basketId=basketId,basketName=basketName)	return pt(basketId=basketId,basketName=basketName)


security.declareProtected('View','index_html')	security.declareProtected('manage','index_html')
def index_html(self):	def index_html(self):
"""stanadard ansicht"""	"""stanadard ansicht"""

Line 753 class CDLIBasketContainer(OrderedFolder)	Line 853 class CDLIBasketContainer(OrderedFolder)
"""get name of the actualuser"""	"""get name of the actualuser"""
return str(self.REQUEST['AUTHENTICATED_USER'])	return str(self.REQUEST['AUTHENTICATED_USER'])

	security.declareProtected('manage','addBasket')
def addBasket(self,newBasketName):	def addBasket(self,newBasketName):
"""add a new basket"""	"""add a new basket"""

Line 809 class CDLIBasket(Folder,CatalogAware):	Line 909 class CDLIBasket(Folder,CatalogAware):
meta_type="CDLIBasket"	meta_type="CDLIBasket"
default_catalog="CDLIBasketCatalog"	default_catalog="CDLIBasketCatalog"

	def searchInBasket(self,indexName,searchStr,regExp=False):
	"""searchInBasket"""

	lst=self.searchInLineIndexDocs(indexName,searchStr,uniq=True,regExp=regExp)
	ret={}

	lv=self.getLastVersion()


	for obj in lv.content.getContent():
	id=obj[1].getId().split(".")[0]
	if id in lst:

	ret[id]=self.showWordInFile(id,searchStr,lineList=self.getLinesFromIndex(indexName,searchStr,id,regExp=regExp),regExp=regExp,indexName=indexName)


	pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','searchResultsInBasket')).__of__(self)
	return pt(result=ret,indexName=indexName,regExp=regExp,word=searchStr)




	def searchInBasket_v1(self,searchStr):
	"""search occurences of searchStr in files im basket"""
	ret=[]
	lv=self.getLastVersion()
	logging.info("searching")
	for obj in lv.content.getContent():
	txt=obj[0].getData()
	for x in txt.split("\n"):
	logging.info("search %s"%x)
	if re.match(searchStr,x):
	ret.append(x)

	return "\n".join(ret)


def getFile(self,obj):	def getFile(self,obj):
return obj[1]	return obj[1]
Line 1130 class CDLIBasketVersion(Implicit,Persist	Line 1266 class CDLIBasketVersion(Implicit,Persist
self.REQUEST.RESPONSE.setHeader("Content-Length",length)	self.REQUEST.RESPONSE.setHeader("Content-Length",length)
self.REQUEST.RESPONSE.write(ret)	self.REQUEST.RESPONSE.write(ret)

security.declareProtected('View','downloadObjectsAsOneFile')	security.declareProtected('manage','downloadObjectsAsOneFile')
def downloadObjectsAsOneFile(self,lock=None,procedure=None,REQUEST=None,check="yes",current="no"):	def downloadObjectsAsOneFile(self,lock=None,procedure=None,REQUEST=None,check="yes",current="no"):
"""download all selected files in one file"""	"""download all selected files in one file"""

Line 1202 class CDLIBasketVersion(Implicit,Persist	Line 1338 class CDLIBasketVersion(Implicit,Persist
ret+=str(object[0].getData())+"\n"	ret+=str(object[0].getData())+"\n"
elif current=="yes":	elif current=="yes":
#search current object	#search current object
founds=self.CDLICatalog.search({'title':object[0].getId()})	founds=self.CDLICatalog.search({'title':object[1].getId().split(".")[0]})
if len(founds)>0:	if len(founds)>0:
ret+=str(founds[0].getObject().getLastVersion().getData())+"\n"	ret+=str(founds[0].getObject().getLastVersion().getData())+"\n"

Line 1248 class CDLIBasketVersion(Implicit,Persist	Line 1384 class CDLIBasketVersion(Implicit,Persist
def __init__(self,id,user,comment="",basketContent=[]):	def __init__(self,id,user,comment="",basketContent=[]):
""" init a basket version"""	""" init a basket version"""
self.id=id	self.id=id
self.coment=comment	self.comment=comment
self._setObject('content',BasketContent(basketContent))	self._setObject('content',BasketContent(basketContent))
#self.basketContent=basketContent[0:]a	#self.basketContent=basketContent[0:]a
self.user=user	self.user=user
Line 1262 class CDLIBasketVersion(Implicit,Persist	Line 1398 class CDLIBasketVersion(Implicit,Persist
"""get Comment"""	"""get Comment"""
return self.comment	return self.comment

security.declareProtected('View','index_html')	security.declareProtected('manage','index_html')
def index_html(self):	def index_html(self):
"""view the basket"""	"""view the basket"""

Line 1317 class CDLIFileObject(CatalogAware,extVer	Line 1453 class CDLIFileObject(CatalogAware,extVer

security.declarePublic('makeThisVersionCurrent')	security.declarePublic('makeThisVersionCurrent')

	security.declareProtected('manage','index_html')
def PrincipiaSearchSource(self):	def PrincipiaSearchSource(self):
"""Return cataloguable key for ourselves."""	"""Return cataloguable key for ourselves."""
return str(self)	return str(self)
Line 1425 def manage_addCDLIFileObject(self,id,vC=	Line 1562 def manage_addCDLIFileObject(self,id,vC=
class CDLIFile(extVersionedFile,CatalogAware):	class CDLIFile(extVersionedFile,CatalogAware):
"""CDLI file"""	"""CDLI file"""

	security=ClassSecurityInfo()
meta_type="CDLI file"	meta_type="CDLI file"
default_catalog='CDLICatalog'	default_catalog='CDLICatalog'
	security.declareProtected('manage','index_html')
#security.declarePublic('history')	#security.declarePublic('history')
def getLastVersionData(self):	def getLastVersionData(self):
"""get last version data"""	"""get last version data"""
Line 1613 def splitatf(fh,dir=None,ext=None):	Line 1751 def splitatf(fh,dir=None,ext=None):
nf=None	nf=None
i=0	i=0

for lineTmp in fh.readlines():	if (type(fh) is StringType) or (type(fh) is UnicodeType):
	iter=fh.split("\n")
	else:
	iter=fh.readlines()

	for lineTmp in iter:
lineTmp=lineTmp.replace(codecs.BOM_UTF8,'') # make sure that all BOM are removed..	lineTmp=lineTmp.replace(codecs.BOM_UTF8,'') # make sure that all BOM are removed..
for line in lineTmp.split("\r"):	for line in lineTmp.split("\r"):
#logging.log("Deal with: %s"%line)	#logging.log("Deal with: %s"%line)
Line 1650 def splitatf(fh,dir=None,ext=None):	Line 1793 def splitatf(fh,dir=None,ext=None):
nf.close()	nf.close()
except:	except:
pass	pass

	if not((type(fh) is StringType) or (type(fh) is UnicodeType)):
fh.close()	fh.close()
return ret,len(os.listdir(dir))	return ret,len(os.listdir(dir))

Line 1705 class CDLIFileFolder(extVersionedFileFol	Line 1850 class CDLIFileFolder(extVersionedFileFol
"""get the content of the file fn"""	"""get the content of the file fn"""
founds=self.CDLICatalog.search({'title':fn})	founds=self.CDLICatalog.search({'title':fn})
if not founds:	if not founds:
return []	return ""
else:	else:
obj=founds[0].getObject().getLastVersion()	obj=founds[0].getObject().getLastVersion()

Line 1872 class CDLIFileFolder(extVersionedFileFol	Line 2017 class CDLIFileFolder(extVersionedFileFol
return ret	return ret


security.declareProtected('View','index_html')	security.declareProtected('manage','index_html')
def index_html(self):	def index_html(self):
"""main"""	"""main"""
ext=self.ZopeFind(self,obj_ids=["index.html"])	ext=self.ZopeFind(self,obj_ids=["index.html"])
Line 1921 class CDLIRoot(Folder):	Line 2066 class CDLIRoot(Folder):
meta_type="CDLIRoot"	meta_type="CDLIRoot"
downloadCounterBaskets=0# counts the current basket downloads if counter > 10 no downloads are possible	downloadCounterBaskets=0# counts the current basket downloads if counter > 10 no downloads are possible

	def deleteFiles(self,ids):
	"""delete files (resp. move into .trash folder)"""
	# find or generete trash folder

	found=self.ZopeFind(self,obj_ids=['.trash'])

	if len(found)<1:
	manage_addCDLIFileFolder(self, '.trash',title="Trash")
	trash=self._getOb('.trash')
	else:
	logging.info(found)
	trash=found[0][1]


	for id in ids:
	founds=self.CDLICatalog.search({'title':id.split(".")[0]})
	if founds:
	logging.info(founds)
	folder=founds[0].getObject().aq_parent #get the parent folder of the object
	logging.info(folder)
	cut=folder.manage_cutObjects([founds[0].getId]) #cut it out
	trash.manage_pasteObjects(cut) #paste it in the trash


def findWordRegExp(self,indexName,searchTerm):	def findWordRegExp(self,indexName,searchTerm):
"""find all words in index which match regexp in SearchTerm	"""find all words in index which match regexp in SearchTerm
@param indexName: name of the index to be searched in	@param indexName: name of the index to be searched in
Line 1943 class CDLIRoot(Folder):	Line 2112 class CDLIRoot(Folder):
words=self.findWordRegExp(indexName,searchTerm) # suche nach allen Treffern	words=self.findWordRegExp(indexName,searchTerm) # suche nach allen Treffern
logging.info("wd:%s"%words)	logging.info("wd:%s"%words)
for word in words:	for word in words:

ret+=self.searchInLineIndexDocs(indexName,word)	ret+=self.searchInLineIndexDocs(indexName,word)

return unique(ret)
	x= unique(ret)
	logging.info("words_done")
	return x

def showInLineIndex(self):	def showInLineIndex(self):
"""get the index for debug purposes"""	"""get the index for debug purposes"""
Line 2050 class CDLIRoot(Folder):	Line 2223 class CDLIRoot(Folder):
transaction.get().commit()	transaction.get().commit()


def showFile(self,fileId):	def showFile(self,fileId,wholePage=False):
"""show a file	"""show a file
@param fileId: P-Number of the document to be displayed	@param fileId: P-Number of the document to be displayed
"""	"""
Line 2058 class CDLIRoot(Folder):	Line 2231 class CDLIRoot(Folder):
if not f:	if not f:
return ""	return ""

	if wholePage:
	logging.info("whole")
	return f[0].getObject().getLastVersion().view()
	else:
return f[0].getObject().getLastVersionFormattedData()	return f[0].getObject().getLastVersionFormattedData()


Line 2065 class CDLIRoot(Folder):	Line 2242 class CDLIRoot(Folder):
"""get lines with word fromFileId"""	"""get lines with word fromFileId"""

file=self.showFile(fileId)	file=self.showFile(fileId)
	logging.info("regEXP %s"%regExp)
ret=[]	ret=[]
if regExp: # wenn regexp dann generiere alle worte aus der list die der regexp entsprechen	if regExp: # wenn regexp dann generiere alle worte aus der list die der regexp entsprechen
wordlist=self.findWordRegExp(indexName,word)	wordlist=self.findWordRegExp(indexName,word)
Line 2075 class CDLIRoot(Folder):	Line 2252 class CDLIRoot(Folder):
for line in file.split("\n"):	for line in file.split("\n"):
found=False	found=False
for word in wordlist:	for word in wordlist:
	try: # just a hack because of possible unicode errors in line
if line.find(word)>-1:	if line.find(word)>-1:
if lineList: #liste of moeglichen Zeilennummern	if lineList: #liste of moeglichen Zeilennummern
num=line.split(".")[0] #Zeilenummer ist alles vor dem . in der Zeile	num=line.split(".")[0] #Zeilenummer ist alles vor dem . in der Zeile
Line 2086 class CDLIRoot(Folder):	Line 2264 class CDLIRoot(Folder):
ret.append(line)	ret.append(line)

break;	break;
	except:
	pass
return ret	return ret

def tagWordInFile(self,fileId,word,lineList=None,regExp=True,indexName=""):	def tagWordInFile(self,fileId,word,lineList=None,regExp=True,indexName=""):
"""get lines with word fromFileId"""	"""get lines with word fromFileId"""

file=self.showFile(fileId)	file=self.showFile(fileId)
tagStr="""<span class="found">%s</span>"""	tagStr=u'<span class="found">%s</span>'
ret=[]	ret=[]

if regExp: # wenn regexp dann generiere alle worte aus der list die der regexp entsprechen	if regExp: # wenn regexp dann generiere alle worte aus der list die der regexp entsprechen
Line 2101 class CDLIRoot(Folder):	Line 2281 class CDLIRoot(Folder):
wordlist=[word]	wordlist=[word]

for line in file.split("\n"):	for line in file.split("\n"):
	line = unicodify(line)
found=False	found=False
for word in wordlist:	for word in wordlist:
if line.find(word)>-1: #word ist gefunden dann makiere und breche die Schleife ab	if line.find(word)>-1: #word ist gefunden dann makiere und breche die Schleife ab
Line 2118 class CDLIRoot(Folder):	Line 2299 class CDLIRoot(Folder):
if not found: #word wurde nicht gefunden keine makierung	if not found: #word wurde nicht gefunden keine makierung
ret.append(line)	ret.append(line)

return "<br>\n".join(ret)	return u'<br>\n'.join(ret)

def URLquote(self,str):	def URLquote(self,str):
"""quote url"""	"""quote url"""
Line 2210 class CDLIRoot(Folder):	Line 2391 class CDLIRoot(Folder):
return ret	return ret


	def uploadATFRPC(self,data,username):
	"""upload an atffile via xml-rpc"""
	uploader=uploadATFThread()

	#generate an random id for the upload object
	from random import randint
	if (not self.REQUEST.SESSION.get('idTmp',None)):

	idTmp=str(randint(0,1000000000))
	self.REQUEST.SESSION['idTmp']=idTmp
	else:
	idTmp=self.REQUEST.SESSION.get('idTmp',None)


	uploader.set(data,0,username,idTmp)

	stObj=uploader.run()

	processor=uploadATFfinallyThread()

	basketname=stObj.returnValue['basketNameFromFile']

	processor.set("uploadchanged",basketname=basketname,SESSION=stObj.returnValue,username=username,serverport=self.REQUEST['SERVER_PORT'])

	processor.run()


	return generateXMLReturn(stObj.returnValue)

def uploadATF(self,repeat=None,upload=None,basketId=0,RESPONSE=None):	def uploadATF(self,repeat=None,upload=None,basketId=0,RESPONSE=None):
"""upload an atf file / basket file"""	"""upload an atf file / basket file"""

FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>

Removed from v.1.72
changed lines
	Added in v.1.79