![]() ![]() | ![]() |
more fixes for full text search and display
1: """CDLI extensions of the filearchive""" 2: from Products.versionedFile.extVersionedFile import * 3: from Products.ZCatalog.CatalogPathAwareness import CatalogAware 4: from tempfile import mkstemp,mkdtemp 5: import os.path 6: import os 7: from types import * 8: import urlparse 9: import urllib 10: import cgi 11: from OFS.OrderedFolder import OrderedFolder 12: from OFS.SimpleItem import SimpleItem 13: import time 14: from OFS.Folder import manage_addFolder 15: import re 16: from AccessControl import ClassSecurityInfo 17: from Acquisition import Implicit 18: from Globals import Persistent 19: from threading import Thread 20: from ZPublisher.HTTPRequest import HTTPRequest 21: from ZPublisher.HTTPResponse import HTTPResponse 22: from ZPublisher.BaseRequest import RequestContainer 23: import threading 24: from BTrees.OOBTree import OOBTree 25: import logging 26: import transaction 27: import copy 28: import codecs 29: import sys 30: 31: import cdliSplitter 32: 33: 34: def unicodify(s): 35: """decode str (utf-8 or latin-1 representation) into unicode object""" 36: if not s: 37: return u"" 38: if isinstance(s, str): 39: try: 40: return s.decode('utf-8') 41: except: 42: return s.decode('latin-1') 43: else: 44: return s 45: 46: def utf8ify(s): 47: """encode unicode object or string into byte string in utf-8 representation. 48: assumes string objects to be utf-8""" 49: if not s: 50: return "" 51: if isinstance(s, str): 52: return s 53: else: 54: return s.encode('utf-8') 55: 56: def formatAtfHtml(l): 57: """escape special ATF characters for HTML""" 58: if not l: 59: return "" 60: 61: # replace & 62: l = l.replace('&','&') 63: # replace angular brackets 64: l = l.replace('<','<') 65: l = l.replace('>','>') 66: return l 67: 68: def formatAtfLineHtml(l, nolemma=True): 69: """format ATF line for HTML""" 70: if not l: 71: return "" 72: 73: if nolemma: 74: # ignore lemma lines 75: if l.lstrip().startswith('#lem:'): 76: return "" 77: 78: return formatAtfHtml(l) 79: 80: 81: 82: def formatAtfFullLineNum(txt, nolemma=True): 83: """format full line numbers in ATF text""" 84: # surface codes 85: surfaces = {'@obverse':'obv', 86: '@reverse':'rev', 87: '@surface':'surface', 88: '@edge':'edge', 89: '@left':'left', 90: '@right':'right', 91: '@top':'top', 92: '@bottom':'bottom', 93: '@face':'face', 94: '@seal':'seal'} 95: 96: if not txt: 97: return "" 98: 99: ret = [] 100: surf = "" 101: col = "" 102: for line in txt.split("\n"): 103: line = unicodify(line) 104: if line and line[0] == '@': 105: # surface or column 106: words = line.split(' ') 107: if words[0] in surfaces: 108: surf = line.replace(words[0],surfaces[words[0]]).strip() 109: 110: elif words[0] == '@column': 111: col = words[1] 112: 113: elif line and line[0] in '123456789': 114: # ordinary line -> add line number 115: line = "%s:%s:%s"%(surf,col,line) 116: 117: ret.append(line) 118: 119: return '\n'.join(ret) 120: 121: 122: def generateXMLReturn(hash): 123: """erzeugt das xml file als returnwert fuer uploadATFRPC""" 124: 125: ret="<return>" 126: 127: ret+="<errors>" 128: for error in hash['errors']: 129: ret+="""<error atf="%s">%s</error>"""%error 130: 131: ret+="</errors>" 132: 133: ret+="<changes>" 134: for changed in hash['changed']: 135: ret+="""<change atf="%s">%s</change>"""%changed 136: ret+="</changes>" 137: 138: ret+="<newPs>" 139: for new in hash['newPs']: 140: ret+="""<new atf="%s"/>"""%new 141: ret+="</newPs>" 142: 143: ret+="</return>" 144: return ret 145: 146: 147: 148: 149: 150: 151: 152: 153: 154: 155: def unique(s): 156: """Return a list of the elements in s, but without duplicates. 157: 158: For example, unique([1,2,3,1,2,3]) is some permutation of [1,2,3], 159: unique("abcabc") some permutation of ["a", "b", "c"], and 160: unique(([1, 2], [2, 3], [1, 2])) some permutation of 161: [[2, 3], [1, 2]]. 162: 163: For best speed, all sequence elements should be hashable. Then 164: unique() will usually work in linear time. 165: 166: If not possible, the sequence elements should enjoy a total 167: ordering, and if list(s).sort() doesn't raise TypeError it's 168: assumed that they do enjoy a total ordering. Then unique() will 169: usually work in O(N*log2(N)) time. 170: 171: If that's not possible either, the sequence elements must support 172: equality-testing. Then unique() will usually work in quadratic 173: time. 174: (from the python cookbook) 175: """ 176: 177: n = len(s) 178: if n == 0: 179: return [] 180: 181: # Try using a dict first, as that's the fastest and will usually 182: # work. If it doesn't work, it will usually fail quickly, so it 183: # usually doesn't cost much to *try* it. It requires that all the 184: # sequence elements be hashable, and support equality comparison. 185: u = {} 186: try: 187: for x in s: 188: u[x] = 1 189: except TypeError: 190: del u # move on to the next method 191: else: 192: return u.keys() 193: 194: # We can't hash all the elements. Second fastest is to sort, 195: # which brings the equal elements together; then duplicates are 196: # easy to weed out in a single pass. 197: # NOTE: Python's list.sort() was designed to be efficient in the 198: # presence of many duplicate elements. This isn't true of all 199: # sort functions in all languages or libraries, so this approach 200: # is more effective in Python than it may be elsewhere. 201: try: 202: t = list(s) 203: t.sort() 204: except TypeError: 205: del t # move on to the next method 206: else: 207: assert n > 0 208: last = t[0] 209: lasti = i = 1 210: while i < n: 211: if t[i] != last: 212: t[lasti] = last = t[i] 213: lasti += 1 214: i += 1 215: return t[:lasti] 216: 217: # Brute force is all that's left. 218: u = [] 219: for x in s: 220: if x not in u: 221: u.append(x) 222: return u 223: 224: 225: class BasketContent(SimpleItem): 226: """classe fuer den Inhalt eines Baskets""" 227: 228: def __init__(self,content=[]): 229: """content""" 230: self.contentList=content[0:] 231: 232: def getContent(self): 233: """get content""" 234: 235: return self.contentList 236: 237: def setContent(self,content): 238: self.contentList=content[0:] 239: 240: def numberOfItems(self): 241: """number""" 242: 243: return len(self.getContent()) 244: 245: 246: class uploadATFfinallyThread(Thread): 247: """class for adding uploaded filed (temporarily stored in the staging area at /tmp""" 248: 249: def __init__(self): 250: """init for uploadATFfinallyThread""" 251: self.continueVar=True 252: self.returnValue=None 253: self.end=False 254: Thread.__init__(self) 255: 256: def set(self,procedure,comment="",basketname='',unlock=None,SESSION=None,username=None,serverport="8080"): 257: """set start values for the thread""" 258: self.procedure=procedure 259: self.comment=comment 260: self.basketname=basketname 261: self.unlock=unlock 262: self.SESSION=SESSION 263: self.username=username 264: self.serverport=serverport 265: 266: 267: def __call__(self): 268: """call of the thread (equals run)""" 269: self.run() 270: return True 271: 272: def getContext(self, app,serverport="8080"): 273: """get the context within the ZODB""" 274: 275: resp = HTTPResponse(stdout=None) 276: env = { 277: 'SERVER_NAME':'localhost', 278: 'SERVER_PORT':serverport, 279: 'REQUEST_METHOD':'GET' 280: } 281: req = HTTPRequest(None, env, resp) 282: return app.__of__(RequestContainer(REQUEST = req)) 283: 284: 285: def run(self): 286: """run""" 287: 288: self.result="" 289: #find context within ZODB 290: from Zope import DB 291: conn = DB.open() 292: root = conn.root() 293: app = root['Application'] 294: ctx = self.getContext(app,serverport=self.serverport) 295: 296: #add the files 297: self.uploadATFfinallyThread(ctx,self.procedure,comment=self.comment,basketname=self.basketname,unlock=self.unlock,SESSION=self.SESSION,username=self.username) 298: #commit the transactions 299: transaction.get().commit() 300: conn.close() 301: #set flag for end of this method 302: self.end=True 303: logging.info("ended") 304: return True 305: 306: def __del__(self): 307: """delete""" 308: 309: 310: 311: def getResult(self): 312: """method for accessing result""" 313: 314: return self.result 315: 316: def uploadATFfinallyThread(self,ctx,procedure,comment="",basketname='',unlock=None,RESPONSE=None,SESSION=None,username=None): 317: """upload the files""" 318: #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot 319: ctx2=ctx.cdliRoot 320: 321: self.result+="<h2>Start processing</h2>" 322: 323: #shall I only upload the changed files? 324: logging.info("uploadATFfinally procedure: %s"%procedure) 325: if procedure=="uploadchanged": 326: changed=[x[0] for x in SESSION.get('changed',[])] 327: uploadFns=changed+SESSION.get('newPs',[]) 328: 329: #or all 330: elif procedure=="uploadAll": 331: uploadFns=[] 332: for x in os.listdir(SESSION['tmpdir']): 333: if not x in SESSION['lockerrors']: 334: uploadFns.append(x) 335: 336: #or maybe nothing 337: elif procedure=="noupload": 338: return True 339: else: 340: uploadFns=[] 341: 342: #do first the changed files 343: i=0 344: for fn in uploadFns: 345: i+=1 346: founds=ctx2.CDLICatalog.search({'title':fn}) 347: if len(founds)>0: 348: SESSION['author']=str(username) 349: self.result="<p>Changing : %s"%fn+self.result 350: founds[0].getObject().manage_addCDLIFileObject('',comment,SESSION['author'],file=os.path.join(SESSION['tmpdir'],fn),from_tmp=True) 351: if i==200: 352: i=0 353: transaction.get().commit() 354: logging.info("changing: do commit") 355: 356: transaction.get().commit() 357: logging.info("changing: last commit") 358: 359: #now add the new files 360: newPs=SESSION['newPs'] 361: if len(newPs)>0: 362: tmpDir=SESSION['tmpdir'] 363: logging.info("adding start") 364: self.result="<p>Adding files</p>"+self.result 365: #TODO: make this configurable, at the moment base folder for the files has to be cdli_main 366: 367: ctx2.importFiles(comment=comment,author=str(username) ,folderName=tmpDir, files=newPs,ext=self) 368: logging.info("adding finished") 369: 370: 371: #unlock locked files? 372: if unlock: 373: logging.info("unlocking start") 374: self.result="<p>Unlock files</p>"+self.result 375: unlockFns=[] 376: for x in os.listdir(SESSION['tmpdir']): 377: if not x in SESSION['errors']: 378: unlockFns.append(x) 379: logging.info("unlocking have now what to unlock") 380: 381: for fn in unlockFns: 382: #logging.info("will unlock: %s"%fn) 383: founds=ctx2.CDLICatalog.search({'title':fn}) 384: #logging.info("found it: %s"%repr(founds)) 385: if len(founds)>0: 386: #logging.info("unlock: %s"%founds[0].getObject().getId()) 387: SESSION['author']=str(username) 388: 389: founds[0].getObject().lockedBy="" 390: logging.info("unlocking done") 391: 392: #if a basketname is given, add files to the basket 393: if not (basketname ==''): 394: logging.info("add to basket %s"%basketname) 395: self.result="<p>Add to basket</p>"+self.result 396: basketId=ctx2.basketContainer.getBasketIdfromName(basketname) 397: 398: if not basketId: # create new basket 399: logging.info("create basket %s"%basketname) 400: self.result="<p>Create a new basket</p>"+self.result 401: ob=ctx2.basketContainer.addBasket(basketname) 402: basketId=ob.getId() 403: basket=getattr(ctx2.basketContainer,str(basketId)) 404: ids=os.listdir(SESSION['tmpdir']) 405: basket.addObjects(ids,deleteOld=True,username=str(username)) 406: 407: if RESPONSE is not None: 408: RESPONSE.redirect(self.aq_parent.absolute_url()) 409: 410: 411: logging.info("uploadfinally done") 412: return True 413: 414: class tmpStore(SimpleItem): 415: """simple item""" 416: meta_type="cdli_upload" 417: 418: def __init__(self,id): 419: """init tmp""" 420: self.id=id 421: 422: class uploadATFThread(Thread): 423: """class for checking the files befor uploading""" 424: 425: def __init__(self): 426: """initialise""" 427: 428: self.continueVar=True 429: self.returnValue=None 430: 431: Thread.__init__(self) 432: 433: 434: def set(self,upload,basketId,username,idTmp,serverport="8080"): 435: """set start values for the thread""" 436: self.result="" 437: self.upload=upload 438: self.basketId=basketId 439: self.username=username 440: self.serverport=serverport 441: self.idTmp=idTmp 442: 443: def __call__(self): 444: """call method """ 445: self.run() 446: return True 447: 448: def getContext(self, app,serverport="8080"): 449: """get the context within the ZODB""" 450: resp = HTTPResponse(stdout=None) 451: env = { 452: 'SERVER_NAME':'localhost', 453: 'SERVER_PORT':serverport, 454: 'REQUEST_METHOD':'GET' 455: } 456: req = HTTPRequest(None, env, resp) 457: return app.__of__(RequestContainer(REQUEST = req)) 458: 459: def run(self): 460: idTmp=self.idTmp 461: self.result="" 462: #find context within ZODB 463: from Zope import DB 464: conn = DB.open() 465: root = conn.root() 466: app = root['Application'] 467: ctx = self.getContext(app,serverport=self.serverport) 468: logging.info("run intern") 469: try: 470: logging.info("created: %s"%idTmp) 471: ctx.temp_folder._setObject(idTmp,tmpStore(idTmp)) 472: except: 473: logging.error("thread upload: %s %s"%sys.exc_info()[0:2]) 474: 475: logging.info("call thread intern") 476: self.uploadATFThread(ctx,self.upload,idTmp,self.basketId) 477: 478: #ctx.cdliRoot.cdli_main.tmpStore2[self.getName()[0:]]=self.returnValue 479: 480: 481: transaction.get().commit() 482: 483: conn.close() 484: 485: return getattr(ctx.temp_folder,idTmp) 486: 487: def getResult(self): 488: """method for accessing result""" 489: return self.result 490: 491: def uploadATFThread(self,ctx,upload,idTmp,basketId=0): 492: """upload an atf file""" 493: #TODO: add comments 494: #TODO: finish uploadATF 495: 496: stObj=getattr(ctx.temp_folder,idTmp) 497: logging.info("start, upload thread") 498: self.result="<html><body><h2>I got your file, start now to split it into single atf-files!</h2><p>" 499: 500: #make sure that id is a string and not an integer 501: basketId=str(basketId) 502: 503: #TODO: make this configurable, at the moment, rootFolder for cdli has to be cdliRoot 504: ctx2=ctx.cdliRoot 505: 506: #get temporary file for staging the downloaded and splitted files 507: dir=mkdtemp() 508: 509: 510: changed=[] # changed files 511: errors=[] # files with errors 512: lockerrors=[] # files with errors 513: 514: newPs=[] # new p filed 515: psNotInCatalog=[] # files not in the catalog 516: 517: #split the uploadedd atf file 518: basketNameFromFile, numberOfFiles=splitatf(upload,dir,ext=self) 519: 520: #find basketId if not set 521: 522: #get active abaket 523: if basketId == '0': 524: basketObj=ctx2.basketContainer.getActiveBasket() 525: if basketObj: 526: basketId=basketObj.getId() 527: 528: #if there is no active basket and no basketid given, id is empty, else get besketname and length 529: if basketId == '0': 530: basketNameFromId="" 531: basketLen=0 532: else: 533: basketNameFromId=getattr(ctx2.basketContainer,basketId).title 534: basketLen=getattr(ctx2.basketContainer,basketId).getLastVersion().numberOfItems() 535: 536: logging.info("got the file, upload thread") 537: self.result+="""<html><body><h2>I got the files</h2>< 538: p>I am computing the differences to the exisiting files</p>""" 539: 540: #start to check the files 541: for fn in os.listdir(dir): 542: 543: self.result="<p>process:%s</p>"%fn+self.result 544: 545: # check if file is in the catalog 546: #TODO: checkCatalog is not implemented yet 547: if ctx2.cdli_main.checkCatalog(fn): 548: psNotInCatalog.append(fn) 549: 550: #check if p-file already at the server 551: founds=ctx2.CDLICatalog.search({'title':fn}) 552: 553: #if not than add filename to the list of newfiles 554: 555: data=file(os.path.join(dir,fn)).read() 556: status,msg=checkFile(fn,data,dir) 557: #status=True 558: 559: 560: if not status: # error 561: errors.append((fn,msg)) 562: 563: else: 564: if len(founds)==0: 565: newPs.append(fn) 566: 567: #if p file alread at the server 568: for found in founds: 569: #analyse the differences to the actual file 570: obj=found.getObject() 571: 572: if (not (str(obj.lockedBy))=='') and (not (str(obj.lockedBy)==str(self.username))): 573: lockerrors.append((fn,str(obj.lockedBy))) 574: else: 575: 576: diffs=obj.diff(data) 577: if diffs[0]>0: 578: changed.append((obj,diffs)) #hochladen 579: 580: #ready, set the returnValues 581: self.result+="<h3>Done</h3></body></html>" 582: 583: stObj.returnValue={} 584: 585: stObj.returnValue['errors']=errors 586: 587: stObj.returnValue['newPs']=newPs 588: stObj.returnValue['tmpdir']=dir 589: stObj.returnValue['basketLen']=basketLen 590: stObj.returnValue['numberOfFiles']=numberOfFiles 591: stObj.returnValue['basketNameFromId']=basketNameFromId 592: stObj.returnValue['basketNameFromFile']=basketNameFromFile 593: stObj.returnValue['basketId']=basketId 594: stObj.returnValue['dir']=dir 595: #stObj.returnValue['changed']=copy.copy(changed) 596: stObj.returnValue['changed']=[(x[0].getId(),x[1][0]) for x in changed] 597: #stObj.returnValue['lockerrors']=[x[0].getId() for x in lockerrors] 598: stObj.returnValue['lockerrors']=[x for x in lockerrors] 599: self.returnValue=True 600: #ctx2.cdli_main.setTemp('v_uploadATF_returnValue',True) 601: 602: 603: class CDLIBasketContainer(OrderedFolder): 604: """contains the baskets""" 605: 606: 607: security=ClassSecurityInfo() 608: meta_type="CDLIBasketContainer" 609: 610: def getPNumbersOfBasket(self,basketName): 611: """get all pnumbers of a basket as a list, returns an empty list if basket not found 612: @param basketName: name of the basket 613: """ 614: ret=[] 615: basketId=self.getBasketIdfromName(basketName) 616: if not basketId: 617: return [] 618: 619: ob=getattr(self,basketId).getContent() 620: 621: ret=[x[0].split(".")[0] for x in ob] 622: 623: return ret 624: 625: security.declareProtected('manage','getBasketAsOneFile') 626: def getBasketAsOneFile(self,basketName,current="no"): 627: """returns all files of the basket combined in one file 628: @param basketName: Name of the basket 629: @param current: (optional) if current is set to "yes" then the most current version of 630: all files are downloaded and not the versions of the files as stored in the basket 631: """ 632: ret="" 633: basketId=self.getBasketIdfromName(basketName) 634: if not basketId: 635: return "" 636: 637: ob=getattr(self,basketId).getLastVersion() 638: for object in ob.getContent(): 639: if current=="no": #version as they are in the basket 640: ret+=str(object[0].getData())+"\n" 641: elif current=="yes": 642: #search current object 643: logging.info("crrent: %s"%object[1].getId().split(".")[0]) 644: founds=self.CDLICatalog.search({'title':object[1].getId().split(".")[0]}) 645: if len(founds)>0: 646: ret+=str(founds[0].getObject().getLastVersion().getData())+"\n" 647: return ret 648: 649: security.declareProtected('manage','upDateBaskets') 650: def upDateBaskets(self): 651: """update content in to objects""" 652: 653: founds=self.ZopeFind(self,obj_metatypes=['CDLIBasketVersion'],search_sub=1) 654: 655: for found in founds: 656: found[1].updateBasket() 657: 658: security.declareProtected('manage','deleteBaskets') 659: def deleteBaskets(self,ids=None): 660: """delete baskets, i.e. move them into trash folder""" 661: 662: 663: found=self.ZopeFind(self,obj_ids=['trash']) 664: 665: if len(found)<1: 666: manage_addFolder(self, 'trash') 667: trash=self._getOb('trash') 668: else: 669: trash=found[0][1] 670: 671: if type(ids) is not ListType: 672: ids=[ids] 673: cut=self.manage_cutObjects(ids) 674: trash.manage_pasteObjects(cut) 675: 676: security.declareProtected('manage','manageBaskets') 677: def manageBaskets(self,submit,ids=None,basket1="",basket2="",joinBasket="",subtractBasket="",REQUEST=None,RESPONSE=None): 678: """manage baskets, delete or copy""" 679: if submit=="delete": 680: self.deleteBaskets(ids) 681: 682: elif submit=="join": 683: flag,msg=self.joinBasket(joinBasket, ids) 684: logging.info("joining %s %s"%(flag,msg)) 685: 686: elif submit=="subtract": 687: logging.info("BBBb %s %s"%(basket1,basket2)) 688: flag,msg=self.subtractBasket(subtractBasket, basket1,basket2) 689: logging.info("subtract %s %s"%(flag,msg)) 690: 691: if RESPONSE: 692: RESPONSE.redirect(self.absolute_url()) 693: 694: security.declareProtected('View','getBasketIdfromName') 695: def getBasketIdfromName(self,basketname): 696: """get id from name""" 697: 698: for basket in self.ZopeFind(self,obj_metatypes=["CDLIBasket"]): 699: if basket[1].title==basketname: 700: return basket[0] 701: else: 702: None 703: 704: security.declareProtected('manage','uploadBasket_html') 705: 706: def uploadBasket_html(self,basketId='0'): 707: """upload an atf file, html form""" 708: 709: 710: basketId=str(basketId) 711: if not basketId=='0': 712: basketName=getattr(self.basketContainer,basketId).title 713: else: 714: basketName="" 715: 716: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadBasket_html.zpt')).__of__(self) 717: return pt(basketId=basketId,basketName=basketName) 718: 719: 720: security.declareProtected('manage','index_html') 721: def index_html(self): 722: """stanadard ansicht""" 723: 724: 725: 726: ext=self.ZopeFind(self,obj_ids=["index.html"]) 727: if ext: 728: return ext[0][1]() 729: 730: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketContainerMain')).__of__(self) 731: return pt() 732: 733: def getStorageFolderRoot(self): 734: """root des storage folders""" 735: return self.cdli_main 736: 737: def __init__(self,id,title): 738: """ init basket container""" 739: self.id=id 740: self.title=title 741: 742: 743: def getBasketsId(self): 744: """get all baskets als klartext""" 745: 746: ret="" 747: baskets=self.ZopeFind(self,obj_metatypes=['CDLIBasket']) 748: for basket in baskets: 749: com,user,time,values = basket[1].getContentIds() 750: ret+= "BASKET:"+com+"\t"+user+"\t"+time+"\n" 751: for x in values: 752: ret+= x[0]+"\t"+x[1]+"\n" 753: return ret 754: 755: def getBaskets(self,sortField='title'): 756: """get all baskets files""" 757: 758: def sortName(x,y): 759: return cmp(x[1].title.lower(),y[1].title.lower()) 760: 761: def sortDate(x,y): 762: return cmp(y[1].getLastVersion().getTime(),x[1].getLastVersion().getTime()) 763: 764: 765: def sortComment(x,y): 766: 767: 768: 769: try: 770: xc=getattr(x[1],'comment','ZZZZZZZZZZZZZ').lower() 771: except: 772: xc='ZZZZZZZZZZZZZ'.lower() 773: try: 774: yc=getattr(y[1],'comment','ZZZZZZZZZZZZZ').lower() 775: except: 776: yc='ZZZZZZZZZZZZZ'.lower() 777: 778: 779: if (xc=='') or (xc=='ZZZZZZZZZZZZZ'.lower()): 780: 781: try: 782: xc=x[1].getLastVersion().getComment().lower() 783: except: 784: xc='ZZZZZZZZZZZZZ'.lower() 785: 786: if (yc=='') or (yc=='ZZZZZZZZZZZZZ'.lower()): 787: try: 788: yc=y[1].getLastVersion().getComment().lower() 789: except: 790: yc='ZZZZZZZZZZZZZ'.lower() 791: 792: 793: return cmp(xc,yc) 794: 795: def sortAuthor(x,y): 796: 797: return cmp(x[1].getLastVersion().getUser().lower(),y[1].getLastVersion().getUser().lower()) 798: 799: baskets=self.ZopeFind(self,obj_metatypes=['CDLIBasket']) 800: 801: 802: if sortField=='title': 803: baskets.sort(sortName) 804: elif sortField=='date': 805: baskets.sort(sortDate) 806: elif sortField=='author': 807: baskets.sort(sortAuthor) 808: elif sortField=='comment': 809: baskets.sort(sortComment) 810: 811: return baskets 812: 813: 814: def subtractBasket(self,newBasket,basket1,basket2): 815: """subtract basket2 from basket1 816: (i.e. newbasket will contain alle elements of basket1 which are not in basket2), 817: if basket2 contains files which are not in basket1, then theses files fill be ignored 818: 819: @param newbasket: name of the new basket 820: @param basket1: basket where basket2 will be subtracted from 821: @param basket2: see above 822: 823: """ 824: logging.info("CCCCC %s %s"%(basket1,basket2)) 825: 826: try: 827: newB=self.addBasket(newBasket) 828: except: 829: return False, "cannot create the new basket" 830: 831: 832: 833: 834: 835: bas2= getattr(self,basket2) 836: bas2content=bas2.getContent() 837: bas2ids=[x[0] for x in bas2content] 838: 839: 840: 841: bas1= getattr(self,basket1) 842: bas1content=bas1.getContent() 843: 844: 845: newBasketContent={} 846: 847: for id,version in bas1content: 848: if not (id in bas2ids): 849: newBasketContent[id]=version 850: 851: username=self.getActualUserName() 852: 853: logging.info("sbc %s"%newBasketContent) 854: newB.addObjectsWithVersion(newBasketContent,username=username,catalog=self.CDLICatalog) 855: 856: return True, "" 857: 858: 859: def joinBasket(self,newBasket,oldBaskets): 860: """join two baskets 861: @param newbasket: name of the new basket 862: @param oldbaskets: list of baskets to be joined 863: """ 864: try: 865: newB=self.addBasket(newBasket) 866: except: 867: return False, "cannot create the new basket" 868: 869: newBasketContent={} 870: for ob in oldBaskets: 871: x= getattr(self,ob,None) 872: if x is None: 873: return False, "cannot find basket: %s"%ob 874: 875: ids=x.getContent() # hole den Inhalt 876: 877: for id,version in ids: 878: if newBasketContent.has_key(id): # p number gibt's schon 879: newBasketContent[id]=max(newBasketContent[id],version) # speichere die groessere Versionsnumber 880: else: 881: newBasketContent[id]=version 882: username=self.getActualUserName() 883: 884: logging.info("nbc %s"%newBasketContent) 885: newB.addObjectsWithVersion(newBasketContent,username=username,catalog=self.CDLICatalog) 886: 887: return True, "" 888: 889: def getNewId(self): 890: """createIds""" 891: last=getattr(self,'last',0) 892: last +=1 893: while len(self.ZopeFind(self,obj_ids=[str(last)]))>0: 894: last+=1 895: 896: self.last=last 897: return last 898: 899: def setActiveBasket(self,basketId,REQUEST=None): 900: """store active basketId in a cookie""" 901: self.REQUEST.RESPONSE.setCookie("CDLIActiveBasket",basketId,path="/") 902: try: 903: qs=cgi.parse_qs(REQUEST['QUERY_STRING']) 904: del(qs['basketId']) 905: except: 906: qs={} 907: if REQUEST: 908: REQUEST.RESPONSE.redirect(REQUEST['URL1']+'?'+urllib.urlencode(qs)) 909: 910: def getActiveBasket(self): 911: """get active basket from cookie""" 912: 913: id= self.REQUEST.cookies.get('CDLIActiveBasket',None) 914: if id: 915: obj=getattr(self,str(id),None) 916: else: 917: obj=None 918: return obj 919: 920: def getActualUserName(self): 921: """get name of the actualuser""" 922: return str(self.REQUEST['AUTHENTICATED_USER']) 923: 924: security.declareProtected('manage','addBasket') 925: def addBasket(self,newBasketName): 926: """add a new basket""" 927: 928: ob=manage_addCDLIBasket(self,newBasketName) 929: return ob 930: 931: def storeInBasket(self,submit,ids=None,newBasketName=None,fromFileList=None,RESPONSE=None,REQUEST=None): 932: """store it""" 933: if not ids: 934: ids=self.REQUEST.SESSION['fileIds'] 935: 936: if type(ids) is not ListType: 937: ids=[ids] 938: 939: if (submit.lower()=="store in new basket") or (submit.lower()=="new basket"): 940: basketRet=self.addBasket(newBasketName) 941: self.setActiveBasket(basketRet.getId()) 942: basket=getattr(self,basketRet.getId()) 943: elif (submit.lower()=="store in active basket") or (submit.lower()=="active basket"): 944: basket=self.getActiveBasket() 945: 946: added=basket.addObjects(ids) 947: back=self.REQUEST['HTTP_REFERER'].split("?")[0]+"?basketName="+basket.title+"&numberOfObjects="+str(added) 948: 949: 950: if fromFileList: 951: 952: return self.cdli_main.findObjectsFromList(list=self.REQUEST.SESSION['fileIds'],basketName=basket.title,numberOfObjects=added) 953: 954: if RESPONSE: 955: 956: RESPONSE.redirect(back) 957: 958: return True 959: 960: def manage_addCDLIBasketContainerForm(self): 961: """add the CDLIBasketContainer form""" 962: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIBasketContainer.zpt')).__of__(self) 963: return pt() 964: 965: def manage_addCDLIBasketContainer(self,id,title,RESPONSE=None): 966: """add the basket""" 967: ob=CDLIBasketContainer(id,title) 968: 969: self._setObject(id, ob) 970: 971: if RESPONSE is not None: 972: RESPONSE.redirect('manage_main') 973: 974: class CDLIBasket(Folder,CatalogAware): 975: """basket""" 976: 977: meta_type="CDLIBasket" 978: default_catalog="CDLIBasketCatalog" 979: 980: def searchInBasket(self,indexName,searchStr,regExp=False): 981: """searchInBasket""" 982: 983: lst=self.searchInLineIndexDocs(indexName,searchStr,uniq=True,regExp=regExp) #TODO: fix this 984: ret={} 985: 986: lv=self.getLastVersion() 987: 988: 989: for obj in lv.content.getContent(): 990: id=obj[1].getId().split(".")[0] 991: if id in lst: 992: 993: ret[id]=self.showWordInFile(id,searchStr,lineList=self.getLinesFromIndex(indexName,searchStr,id,regExp=regExp),regExp=regExp,indexName=indexName) 994: 995: 996: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','searchResultsInBasket')).__of__(self) 997: return pt(result=ret,indexName=indexName,regExp=regExp,word=searchStr) 998: 999: 1000: 1001: 1002: def searchInBasket_v1(self,searchStr): 1003: """search occurences of searchStr in files im basket""" 1004: ret=[] 1005: lv=self.getLastVersion() 1006: logging.info("searching") 1007: for obj in lv.content.getContent(): 1008: txt=obj[0].getData() 1009: for x in txt.split("\n"): 1010: logging.info("search %s"%x) 1011: if re.match(searchStr,x): 1012: ret.append(x) 1013: 1014: return "\n".join(ret) 1015: 1016: 1017: def getFile(self,obj): 1018: return obj[1] 1019: 1020: def getFileLastVersion(self,obj): 1021: return obj[0] 1022: 1023: def getFileNamesInLastVersion(self): 1024: """get content of the last version as list""" 1025: 1026: return [x[1].getId() for x in self.getLastVersion().getContent()] 1027: 1028: 1029: def isActual(self,obj): 1030: """teste ob im basket die aktuelle version ist""" 1031: actualNo=obj[1].getLastVersion().getVersionNumber() 1032: storedNo=obj[0].getVersionNumber() 1033: 1034: founds=self.CDLICatalog.search({'title':obj[0].getId()}) 1035: if len(founds)>0: 1036: actualNo=founds[0].getObject().getLastVersion().getVersionNumber() 1037: 1038: if len(founds)>0 and founds[0].getObject().aq_parent.getId()==".trash": 1039: return False, -1 1040: 1041: if actualNo==storedNo: 1042: return True , 0 1043: else: 1044: return False, actualNo 1045: 1046: def history(self): 1047: """history""" 1048: 1049: ext=self.ZopeFind(self.aq_parent,obj_ids=["history_template.html"]) 1050: if ext: 1051: return getattr(self,ext[0][1].getId())() 1052: 1053: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketHistory')).__of__(self) 1054: return pt() 1055: 1056: def getStorageFolderRoot(self): 1057: """root des storage folders""" 1058: return self.aq_parent.cdli_main 1059: 1060: def __init__(self,id,title,shortDescription="",comment=""): 1061: """init a CDLIBasket""" 1062: 1063: self.id=id 1064: self.title=title 1065: self.shortDescription=shortDescription 1066: self.comment=comment 1067: 1068: def getActualUserName(self): 1069: """get name of the actualuser""" 1070: 1071: return str(self.REQUEST['AUTHENTICATED_USER']) 1072: 1073: 1074: def getLastVersion(self): 1075: """hole letzte version""" 1076: 1077: ids=[] 1078: idsTmp= self.objectIds() 1079: for x in idsTmp: 1080: try: 1081: ids.append(int(x)) 1082: except: 1083: pass 1084: ids.sort() 1085: 1086: if len(ids)==0: 1087: return None 1088: else: 1089: ob=getattr(self,str(ids[-1])) 1090: 1091: 1092: return ob 1093: 1094: def getVersions(self): 1095: """get versions""" 1096: versions=self.ZopeFind(self,obj_metatypes=["CDLIBasketVersion"]) 1097: return versions 1098: 1099: 1100: def updateObjects(self,ids,RESPONSE=None,REQUEST=None): 1101: """update ids, ids not in the basket the add""" 1102: if type(ids) is not ListType: 1103: ids=[ids] 1104: 1105: lastVersion=self.getLastVersion() 1106: oldContent=lastVersion.content.getContent() 1107: newContent=[] 1108: 1109: #first copy the old 1110: for obj in oldContent: 1111: if obj[1].getId() not in ids: 1112: newContent.append(obj) 1113: #now add the new 1114: 1115: for id in ids: 1116: founds=self.CDLICatalog.search({'title':id}) 1117: 1118: for found in founds: 1119: if found.getObject() not in oldContent: 1120: #TODO: was passiert wenn, man eine Object dazufŸgt, das schon da ist aber eine neuere version 1121: newContent.append((found.getObject().getLastVersion(),found.getObject())) 1122: 1123: 1124: content=newContent 1125: user=self.getActualUserName() 1126: 1127: ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=newContent) 1128: 1129: obj=self._getOb(ob.getId()) 1130: if RESPONSE: 1131: 1132: RESPONSE.redirect(obj.absolute_url()) 1133: 1134: return obj 1135: 1136: def addObjectsWithVersion(self,ids,deleteOld=None,username=None,catalog=None): 1137: """generate a new version of the basket with objects added, 1138: hier wird jedoch nicht die letzte Version jedes Files hinzugefuegt, s 1139: ondern ids is ein Tupel mit der Id (d.h. der p-number) und der Versionsnummer. 1140: """ 1141: logging.info("add to basket (%s)"%(self.getId())) 1142: lastVersion=self.getLastVersion() 1143: 1144: if not catalog: 1145: catalog=self.CDLICatalog 1146: 1147: if lastVersion is None: 1148: oldContent=[] 1149: else: 1150: oldContent=lastVersion.content.getContent() 1151: 1152: if deleteOld: 1153: oldContent=[] 1154: 1155: newContent=[] 1156: added=0 1157: 1158: for id,version in ids.iteritems(): 1159: logging.info("adding %s %s"%(id,version)) 1160: id=id.split(".")[0] # title nur die pnumber ohne atf 1161: 1162: try: 1163: founds=catalog.search({'title':id}) 1164: except: 1165: founds=[] 1166: logging.info(" found %s "%(founds)) 1167: for found in founds: 1168: if found.getObject() not in oldContent: 1169: 1170: #TODO: was passiert wenn, man eine Object dazufŸgt, das schon da ist aber eine neuere version 1171: newContent.append((found.getObject().getVersions()[version-1][1],found.getObject())) 1172: added+=1 1173: 1174: content=oldContent+newContent 1175: if not username: 1176: logging.error("XXXXXXXXXXX %s"%repr(self)) 1177: user=self.getActualUserName() 1178: else: 1179: user = username 1180: 1181: ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=content) 1182: logging.info("add to basket (%s) done"%(self.getId())) 1183: return added 1184: 1185: 1186: def addObjects(self,ids,deleteOld=None,username=None): 1187: """generate a new version of the basket with objects added""" 1188: logging.info("add to basket (%s)"%(self.getId())) 1189: lastVersion=self.getLastVersion() 1190: 1191: if lastVersion is None: 1192: oldContent=[] 1193: else: 1194: oldContent=lastVersion.content.getContent() 1195: 1196: if deleteOld: 1197: oldContent=[] 1198: 1199: newContent=[] 1200: added=0 1201: for id in ids: 1202: try: 1203: founds=self.CDLICatalog.search({'title':id}) 1204: except: 1205: founds=[] 1206: 1207: for found in founds: 1208: if found.getObject() not in oldContent: 1209: #TODO: was passiert wenn, man eine Object dazufŸgt, das schon da ist aber eine neuere version 1210: newContent.append((found.getObject().getLastVersion(),found.getObject())) 1211: added+=1 1212: 1213: content=oldContent+newContent 1214: if not username: 1215: user=self.getActualUserName() 1216: else: 1217: user = username 1218: 1219: ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=content) 1220: logging.info("add to basket (%s) done"%(self.getId())) 1221: return added 1222: 1223: 1224: 1225: def getContent(self): 1226: """print content""" 1227: ret=[] 1228: 1229: lv=self.getLastVersion() 1230: for obj in lv.content.getContent(): 1231: logging.info("XXXXXXXXXX %s"%repr(obj)) 1232: ret.append((obj[1].getId(),obj[0].versionNumber)) 1233: 1234: return ret 1235: 1236: def getContentIds(self): 1237: """print basket content""" 1238: ret=[] 1239: lv=self.getLastVersion() 1240: for obj in lv.content.getContent(): 1241: ret.append((obj[0].getId(),obj[1].getId())) 1242: 1243: 1244: return lv.getComment(),lv.getUser(),lv.getTime(),ret 1245: 1246: def changeBasket(self,ids,submit,RESPONSE=None,REQUEST=None): 1247: """change a basket""" 1248: if submit=="update": 1249: return self.updateObjects(ids,RESPONSE=RESPONSE,REQUEST=REQUEST) 1250: elif submit=="delete": 1251: return self.deleteObjects(ids,RESPONSE=RESPONSE,REQUEST=REQUEST) 1252: 1253: def deleteObjects(self,ids,RESPONSE=None,REQUEST=None): 1254: """delete objects""" 1255: 1256: if type(ids) is not ListType: 1257: ids=[ids] 1258: 1259: lastVersion=self.getLastVersion() 1260: oldContent=lastVersion.content.getContent() 1261: newContent=[] 1262: for obj in oldContent: 1263: if obj[1].getId() not in ids: 1264: newContent.append(obj) 1265: 1266: 1267: user=self.getActualUserName() 1268: 1269: ob=manage_addCDLIBasketVersion(self,user,comment="",basketContent=newContent) 1270: 1271: if RESPONSE: 1272: obj=self._getOb(ob.getId()) 1273: RESPONSE.redirect(obj.absolute_url()) 1274: 1275: def manage_addCDLIBasketForm(self): 1276: """add the CDLIBasketContainer form""" 1277: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIBasket.zpt')).__of__(self) 1278: return pt() 1279: 1280: def manage_addCDLIBasket(self,title,shortDescription="",comment="",RESPONSE=None): 1281: """add the basket""" 1282: 1283: id=str(self.getNewId()) 1284: 1285: ob=CDLIBasket(id,title,shortDescription,comment) 1286: 1287: self._setObject(id, ob) 1288: 1289: if RESPONSE is not None: 1290: RESPONSE.redirect('manage_main') 1291: else: 1292: return ob 1293: 1294: class CDLIBasketVersion(Implicit,Persistent,Folder): 1295: """version of a basket""" 1296: 1297: meta_type="CDLIBasketVersion" 1298: security=ClassSecurityInfo() 1299: 1300: def updateBasket(self): 1301: """update""" 1302: try: 1303: self._setObject('content',BasketContent(self.basketContent)) 1304: except: 1305: try: 1306: if len(self.basketContent)>0: 1307: self.content.setContent(self.basketContent) 1308: except: 1309: print "error",self.getId(),self.aq_parent.getId() 1310: self.basketContent=[] 1311: 1312: 1313: def containsNonActualFiles(self): 1314: """returns True if basket contains one or more non current files""" 1315: 1316: objs=self.getContent() 1317: for obj in objs: 1318: if not self.isActual(obj)[0]: 1319: return True 1320: return False 1321: 1322: def downloadListOfPnumbers(self): 1323: """download pnumbers of the basket as list""" 1324: 1325: basket_name=self.aq_parent.title 1326: 1327: ids=self.getContent() # get the list of objects 1328: logging.error(ids) 1329: ret="\n".join([x[1].getId().split(".")[0] for x in ids]) 1330: 1331: self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.txt" """%basket_name) 1332: self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream") 1333: length=len(ret) 1334: self.REQUEST.RESPONSE.setHeader("Content-Length",length) 1335: self.REQUEST.RESPONSE.write(ret) 1336: 1337: security.declareProtected('manage','downloadObjectsAsOneFile') 1338: def downloadObjectsAsOneFile(self,lock=None,procedure=None,REQUEST=None,check="yes",current="no"): 1339: """download all selected files in one file""" 1340: 1341: if self.temp_folder.downloadCounterBaskets > 10000: 1342: return """I am sorry, currently the server has to many requests for downloads, please come back later!""" 1343: 1344: 1345: if (check=="yes") and self.containsNonActualFiles(): 1346: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','downloadObjectAsOneFile_check.zpt')).__of__(self) 1347: 1348: return pt(lock=lock) 1349: 1350: else: 1351: 1352: return self.downloadObjectsAsOneFileFinally(lock=lock,procedure=procedure,REQUEST=REQUEST,current="no") 1353: 1354: def downloadObjectsAsOneFileFinally(self,lock=None,procedure=None,REQUEST=None,current="no"): 1355: """print do the download""" 1356: 1357: ret="" 1358: lockedObjects={} 1359: 1360: self.temp_folder.downloadCounterBaskets+=1 1361: self._p_changed=1 1362: transaction.get().commit() 1363: 1364: if lock: 1365: 1366: if str(self.REQUEST['AUTHENTICATED_USER'])=='Anonymous User': 1367: self.temp_folder.downloadCounterBaskets-=1 1368: self._p_changed=1 1369: transaction.get().commit() 1370: self.temp_folder.downloadCounterBaskets-=1 1371: self._p_changed=1 1372: transaction.get().commit() 1373: return "please login first" 1374: 1375: #check if a locked object exist in the basket. 1376: lockedObjects={} 1377: for object in self.content.getContent(): 1378: 1379: if (not str(object[1].lockedBy)=="") and (not (str(object[1].lockedBy)==str(self.REQUEST['AUTHENTICATED_USER']))): 1380: lockedObjects[object[1].title]=repr(object[1].lockedBy) 1381: 1382: 1383: keys=lockedObjects.keys() 1384: 1385: 1386: if len(keys)>0 and (not procedure): 1387: self.REQUEST.SESSION['lockedObjects']=lockedObjects 1388: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','lockedObjects.zpt')).__of__(self) 1389: 1390: self.temp_folder.downloadCounterBaskets-=1 1391: self._p_changed=1 1392: transaction.get().commit() 1393: 1394: return pt() 1395: 1396: elif not procedure: #keine fails gesperrt dann alle donwloaden 1397: procedure="downloadAll" 1398: 1399: 1400: 1401: 1402: for object in self.content.getContent(): 1403: 1404: if (procedure=="downloadAll") or (object[1].lockedBy=='') or (object[1].lockedBy==self.REQUEST['AUTHENTICATED_USER']): 1405: if current=="no": #version as they are in the basket 1406: ret+=str(object[0].getData())+"\n" 1407: elif current=="yes": 1408: #search current object 1409: founds=self.CDLICatalog.search({'title':object[1].getId().split(".")[0]}) 1410: if len(founds)>0: 1411: ret+=str(founds[0].getObject().getLastVersion().getData())+"\n" 1412: 1413: if lock and object[1].lockedBy=='': 1414: object[1].lockedBy=self.REQUEST['AUTHENTICATED_USER'] 1415: basket_name=self.aq_parent.title+"_V"+self.getId() 1416: 1417: #write basketname to header of atf file 1418: ret="#basket: %s\n"%basket_name+ret 1419: 1420: self.temp_folder.downloadCounterBaskets-=1 1421: self._p_changed=1 1422: transaction.get().commit() 1423: 1424: self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename="%s.atf" """%basket_name) 1425: self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream") 1426: length=len(ret) 1427: self.REQUEST.RESPONSE.setHeader("Content-Length",length) 1428: self.REQUEST.RESPONSE.write(ret) 1429: return True 1430: 1431: def numberOfItems(self): 1432: """return anzahl der elemente im basket""" 1433: return self.content.numberOfItems() 1434: 1435: def getTime(self): 1436: """getTime""" 1437: #return self.bobobase_modification_time().ISO() 1438: 1439: if hasattr(self,'time'): 1440: return time.strftime("%Y-%m-%d %H:%M:%S",self.time) 1441: elif hasattr(self,'timefixed'): 1442: return self.timefixed 1443: else: 1444: setattr(self,'timefixed',self.bobobase_modification_time().ISO()) 1445: return self.bobobase_modification_time().ISO() 1446: 1447: def getContent(self): 1448: """get Basket Content""" 1449: return self.content.getContent() 1450: 1451: 1452: def __init__(self,id,user,comment="",basketContent=[]): 1453: """ init a basket version""" 1454: self.id=id 1455: self.comment=comment 1456: self._setObject('content',BasketContent(basketContent)) 1457: #self.basketContent=basketContent[0:]a 1458: self.user=user 1459: self.time=time.localtime() 1460: 1461: def getUser(self): 1462: """get user""" 1463: return self.user 1464: 1465: def getComment(self): 1466: """get Comment""" 1467: return self.comment 1468: 1469: security.declareProtected('manage','index_html') 1470: def index_html(self): 1471: """view the basket""" 1472: 1473: if self.REQUEST.get('change',False): 1474: ob=self.aq_parent.updateObjects(self.REQUEST['change']) 1475: 1476: self.REQUEST.RESPONSE.redirect(ob.absolute_url())#go to new basket, because changing generates a new basket 1477: 1478: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','BasketVersionMain.zpt')).__of__(self) 1479: return pt() 1480: 1481: def getObjUrl(self,result): 1482: """getUrl of the version of the object""" 1483: objId=result[1].getTitle() 1484: founds=self.CDLICatalog.search({'title':objId}) 1485: if len(founds)>0: 1486: return founds[0].getObject().getLastVersion().absolute_url() 1487: 1488: else: #assume version number 1489: splitted=objId.split("_") 1490: founds=self.CDLICatalog.search({'title':splitted[1]}) 1491: return founds[0].getObject().getLastVersion().absolute_url()+'/'+objId 1492: 1493: def manage_addCDLIBasketVersion(self,user,comment="",basketContent=[],RESPONSE=None): 1494: """add a version""" 1495: 1496: #check for already existing versions 1497: 1498: lastVersion=self.getLastVersion() 1499: if lastVersion is None: 1500: newId=str(1) 1501: else: 1502: newId=str(int(lastVersion.getId())+1) 1503: 1504: ob=CDLIBasketVersion(newId,user,comment,basketContent) 1505: 1506: self._setObject(newId, ob) 1507: 1508: if RESPONSE is not None: 1509: RESPONSE.redirect('manage_main') 1510: else: 1511: return ob 1512: 1513: class CDLIFileObject(CatalogAware,extVersionedFileObject): 1514: """CDLI file object""" 1515: 1516: meta_type="CDLI File Object" 1517: default_catalog='CDLIObjectsCatalog' 1518: 1519: security=ClassSecurityInfo() 1520: 1521: security.declareProtected('manage','index_html') 1522: 1523: def PrincipiaSearchSource(self): 1524: """Return cataloguable key for ourselves.""" 1525: return str(self) 1526: 1527: def makeThisVersionCurrent_html(self): 1528: """form for making this version current""" 1529: 1530: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','makeThisVersionCurrent.zpt')).__of__(self) 1531: return pt() 1532: 1533: security.declarePublic('makeThisVersionCurrent') 1534: def makeThisVersionCurrent(self,comment,author,RESPONSE=None): 1535: """copy this version to current""" 1536: parent=self.aq_parent 1537: newversion=parent.manage_addCDLIFileObject('',comment,author) 1538: newversion.manage_upload(self.getData()) 1539: 1540: if RESPONSE is not None: 1541: RESPONSE.redirect(self.aq_parent.absolute_url()+'/history') 1542: 1543: return True 1544: 1545: def getFormattedData(self): 1546: """fromat text""" 1547: data=self.getData() 1548: # return re.sub("\s\#lem"," #lem",data) #remove return vor #lem 1549: return re.sub("#lem"," #lem",data) #remove return vor #lem 1550: 1551: security.declarePublic('view') 1552: def view(self): 1553: """view file""" 1554: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','viewCDLIFile.zpt')).__of__(self) 1555: return pt() 1556: 1557: security.declarePublic('getPNumber') 1558: def getPNumber(self): 1559: """get the pnumber""" 1560: try: 1561: txt=re.match("&[Pp](\d*)\s*=([^\r\n]*)",self.getData()[0:]) 1562: except: 1563: txt=self.getData()[0:] 1564: 1565: return "ERROR" 1566: try: 1567: return "P"+txt.group(1) 1568: except: 1569: return "ERROR" 1570: 1571: security.declarePublic('getDesignation') 1572: def getDesignation(self): 1573: """get the designation out of the file""" 1574: try: 1575: txt=re.match("&[Pp](\d*)\s*=([^\r\n]*)",self.getData()[0:]) 1576: except: 1577: txt=self.getData()[0:] 1578: 1579: return "ERROR" 1580: try: 1581: return txt.group(2) 1582: except: 1583: return "ERROR" 1584: 1585: manage_addCDLIFileObjectForm=DTMLFile('dtml/fileAdd', globals(),Kind='CDLIFileObject',kind='CDLIFileObject', version='1') 1586: 1587: def manage_addCDLIFileObject(self,id,vC='',author='', file='',title='',versionNumber=0, 1588: precondition='', content_type='', 1589: from_tmp=False,REQUEST=None): 1590: """Add a new File object. 1591: Creates a new File object 'id' with the contents of 'file'""" 1592: 1593: id=str(id) 1594: title=str(title) 1595: content_type=str(content_type) 1596: precondition=str(precondition) 1597: 1598: id, title = cookId(id, title, file) 1599: 1600: self=self.this() 1601: 1602: # First, we create the file without data: 1603: self._setObject(id, CDLIFileObject(id,title,versionNumber=versionNumber,versionComment=vC,time=time.localtime(),author=author)) 1604: fob = self._getOb(id) 1605: 1606: # Now we "upload" the data. By doing this in two steps, we 1607: # can use a database trick to make the upload more efficient. 1608: 1609: if file and not from_tmp: 1610: fob.manage_upload(file) 1611: elif file and from_tmp: 1612: fob.manage_file_upload(file) # manage_upload_from_tmp doesn't exist in ExtFile2 1613: # fob.manage_upload_from_tmp(file) # manage_upload_from_tmp doesn't exist in ExtFile2 1614: if content_type: 1615: fob.content_type=content_type 1616: 1617: logging.debug("manage_add: lastversion=%s"%self.getData()) 1618: logging.debug("reindex1: %s in %s"%(repr(self),repr(self.default_catalog))) 1619: self.reindex_object() 1620: logging.debug("manage_add: fob_data=%s"%fob.getData()) 1621: logging.debug("reindex2: %s in %s"%(repr(fob), repr(fob.default_catalog))) 1622: fob.index_object() 1623: 1624: if REQUEST is not None: 1625: REQUEST['RESPONSE'].redirect(self.absolute_url()+'/manage_main') 1626: 1627: 1628: class CDLIFile(extVersionedFile,CatalogAware): 1629: """CDLI file""" 1630: 1631: security=ClassSecurityInfo() 1632: meta_type="CDLI file" 1633: content_meta_type = ["CDLI File Object"] 1634: 1635: default_catalog='CDLICatalog' 1636: 1637: security.declareProtected('manage','index_html') 1638: 1639: def getLastVersionData(self): 1640: """get last version data""" 1641: return self.getData() 1642: 1643: def getLastVersionFormattedData(self): 1644: """get last version data""" 1645: return self.getContentObject().getFormattedData() 1646: 1647: def getTextId(self): 1648: """returns P-number of text""" 1649: # assuming that its the beginning of the title 1650: return self.title[:7] 1651: 1652: #security.declarePublic('history') 1653: def history(self): 1654: """history""" 1655: 1656: ext=self.ZopeFind(self.aq_parent,obj_ids=["history_template.html"]) 1657: if ext: 1658: return getattr(self,ext[0][1].getId())() 1659: 1660: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','versionHistory')).__of__(self) 1661: return pt() 1662: 1663: 1664: def getBasketFromId(self,basketid, context=None): 1665: """get basket from id""" 1666: 1667: if not context: 1668: context=self 1669: 1670: for basket in self.ZopeFind(context,obj_metatypes=["CDLIBasket"]): 1671: if basket[0]==basketid: 1672: return basket[1] 1673: else: 1674: None 1675: 1676: 1677: def isContainedInBaskets(self,context=None): 1678: """check is this file is part of any basket 1679: @param context: (optional) necessessary if CDLIBasketCatalog is not an (inherited) attribute of self, context.CDLIBasketCatalog 1680: has to exist. 1681: """ 1682: 1683: if not context: 1684: context=self 1685: 1686: ret=[] 1687: for x in context.CDLIBasketCatalog.search({'getFileNamesInLastVersion':self.getId()}): 1688: #if the basket x is deleted it seemes to be that x is sometimes still in the Catalog, why? 1689: try: 1690: ret.append(x.getObject()) 1691: except: 1692: pass 1693: return ret 1694: #return [x.getObject() for x in context.CDLIBasketCatalog.search({'getFileNamesInLastVersion':self.getId()})] 1695: 1696: 1697: def _newContentObject(self, id, title='', versionNumber=0, versionComment=None, time=None, author=None): 1698: """factory for content objects. to be overridden in derived classes.""" 1699: return CDLIFileObject(id,title,versionNumber=versionNumber,versionComment=versionComment,time=time,author=author) 1700: 1701: 1702: def addCDLIFileObjectForm(self): 1703: """add a new version""" 1704: 1705: if str(self.REQUEST['AUTHENTICATED_USER']) in ["Anonymous User"]: 1706: return "please login first" 1707: if (self.lockedBy==self.REQUEST['AUTHENTICATED_USER']) or (self.lockedBy==''): 1708: out=DTMLFile('dtml/fileAdd', globals(),Kind='CDLIFileObject',kind='CDLIFileObject',version=self.getVersion()).__of__(self) 1709: return out() 1710: else: 1711: return "Sorry file is locked by somebody else" 1712: 1713: def manage_addCDLIFileObject(self,id,vC,author, 1714: file='',title='', 1715: precondition='', 1716: content_type='', 1717: changeName='no',newName='', 1718: come_from=None, 1719: from_tmp=False,RESPONSE=None): 1720: """add""" 1721: 1722: try: #TODO: der ganze vC unsinn muss ueberarbeitet werden 1723: vC=self.REQUEST['vC'] 1724: except: 1725: pass 1726: 1727: ob = self.addContentObject(id, vC, author, file, title, changeName=changeName, newName=newName, from_tmp=from_tmp, 1728: precondition=precondition, content_type=content_type) 1729: 1730: try: 1731: #FIXME: wozu ist das gut? 1732: self.REQUEST.SESSION['objID_parent']=self.getId() 1733: except: 1734: pass 1735: 1736: if RESPONSE: 1737: if ob.getSize()==0: 1738: self.REQUEST.SESSION['objID']=ob.getId() 1739: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','errorUploadFile')).__of__(self) 1740: return pt() 1741: else: 1742: if come_from and (come_from!=""): 1743: RESPONSE.redirect(come_from+"?change="+self.getId()) 1744: else: 1745: RESPONSE.redirect(self.REQUEST['URL2']+'?uploaded=%s'%self.title) 1746: else: 1747: return ob 1748: 1749: 1750: def manage_addCDLIFileForm(self): 1751: """interface for adding the OSAS_root""" 1752: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','addCDLIFile.zpt')).__of__(self) 1753: return pt() 1754: 1755: def manage_addCDLIFile(self,id,title,lockedBy, author=None, RESPONSE=None): 1756: """add the OSAS_root""" 1757: newObj=CDLIFile(id,title,lockedBy,author) 1758: 1759: tryToggle=True 1760: tryCount=0 1761: 1762: self._setObject(id,newObj) 1763: getattr(self,id).reindex_object() 1764: 1765: if RESPONSE is not None: 1766: RESPONSE.redirect('manage_main') 1767: 1768: def checkUTF8(data): 1769: """check utf 8""" 1770: try: 1771: data.encode('utf-8') 1772: return True 1773: except: 1774: return False 1775: 1776: 1777: def checkFile(filename,data,folder): 1778: """check the files""" 1779: # first check the file name 1780: fn=filename.split(".") # no extension 1781: 1782: if not fn[0][0]=="P": 1783: return False,"P missing in the filename" 1784: elif len(fn[0])!=7: 1785: return False,"P number has not the right length 6" 1786: elif not checkUTF8(data): 1787: return False,"not utf-8" 1788: else: 1789: return True,"" 1790: 1791: 1792: def splitatf(fh,dir=None,ext=None): 1793: """split it""" 1794: ret=None 1795: nf=None 1796: i=0 1797: 1798: if (type(fh) is StringType) or (type(fh) is UnicodeType): 1799: iter=fh.split("\n") 1800: else: 1801: iter=fh.readlines() 1802: 1803: for lineTmp in iter: 1804: lineTmp=lineTmp.replace(codecs.BOM_UTF8,'') # make sure that all BOM are removed.. 1805: for line in lineTmp.split("\r"): 1806: #logging.log("Deal with: %s"%line) 1807: if ext: 1808: i+=1 1809: if (i%100)==0: 1810: ext.result+="." 1811: if i==10000: 1812: i=0 1813: ext.result+="<br>" 1814: #check if basket name is in the first line 1815: if line.find("#atf basket")>=0: #old convention 1816: ret=line.replace('#atf basket ','') 1817: ret=ret.split('_')[0] 1818: elif line.find("#basket:")>=0: #new convention 1819: ret=line.replace('#basket: ','') 1820: ret=ret.split('_')[0] 1821: 1822: else: 1823: if (len(line.lstrip())>0) and (line.lstrip()[0]=="&"): #newfile 1824: if nf: 1825: nf.close() #close last file 1826: 1827: 1828: filename=line[1:].split("=")[0].rstrip()+".atf" 1829: if dir: 1830: filename=os.path.join(dir,filename) 1831: nf=file(filename,"w") 1832: logging.info("open %s"%filename) 1833: if nf: 1834: nf.write(line.replace("\n","")+"\n") 1835: 1836: try: 1837: nf.close() 1838: except: 1839: pass 1840: 1841: if not((type(fh) is StringType) or (type(fh) is UnicodeType)): 1842: fh.close() 1843: return ret,len(os.listdir(dir)) 1844: 1845: 1846: class CDLIFileFolder(extVersionedFileFolder): 1847: """CDLI File Folder""" 1848: 1849: security=ClassSecurityInfo() 1850: meta_type="CDLI Folder" 1851: file_meta_type=['CDLI file'] 1852: folder_meta_type=['CDLI Folder'] 1853: 1854: file_catalog='CDLICatalog' 1855: 1856: #downloadCounter=0 # counts how many download for all files currently run, be mehr als 5 wird verweigert. 1857: tmpStore2={} 1858: 1859: def setTemp(self,name,value): 1860: """set tmp""" 1861: 1862: setattr(self,name,value) 1863: 1864: 1865: def delete(self,ids): 1866: """delete these files""" 1867: if type(ids) is not ListType: 1868: ids=[ids] 1869: 1870: self.manage_delObjects(ids) 1871: 1872: 1873: def getVersionNumbersFromIds(self,ids): 1874: """get the numbers of the current versions of documents described by their ids""" 1875: 1876: ret=[] 1877: searchStr=" OR ".join(ids) 1878: 1879: founds=self.CDLICatalog.search({'title':searchStr}) 1880: 1881: for found in founds: 1882: lastVersion=found.getObject().getContentObject() 1883: ret.append((found.getId,lastVersion)) 1884: 1885: return ret 1886: 1887: def getFile(self,fn): 1888: """get the content of the file fn""" 1889: logging.debug("getFile: %s"%repr(fn)) 1890: if not self.hasObject(fn): 1891: # search deeper 1892: founds=getattr(self, self.file_catalog).search({'textid':fn}) 1893: if founds: 1894: obj=founds[0].getObject().getContentObject() 1895: else: 1896: return "" 1897: else: 1898: obj = self[fn].getContentObject() 1899: 1900: return obj.getData()[0:] 1901: 1902: 1903: def checkCatalog(self,fn): 1904: """check if fn is in the catalog""" 1905: #TODO add checkCatalog 1906: 1907: 1908: def findObjectsFromListWithVersion(self,list,author=None): 1909: """find objects from a list with versions 1910: @param list: list of tuples (cdliFile,version) 1911: """ 1912: #self.REQUEST.SESSION['fileIds']=list#store fieldIds in session for further usage 1913: #self.REQUEST.SESSION['searchList']=self.REQUEST.SESSION['fileIds'] 1914: 1915: pt=getattr(self,'filelistVersioned.html') 1916: 1917: return pt(search=list,author=author) 1918: 1919: 1920: def getAllPNumbers(self): 1921: """get a list of all files (resp their p-numbers) stored""" 1922: 1923: ret=[x.getId for x in self.CDLICatalog()] 1924: 1925: return ret 1926: 1927: def findObjectsFromList(self,enterList=None,display=False,start=None,upload=None,list=None,basketName=None,numberOfObjects=None,RESPONSE=None): 1928: """findObjectsFromList (, TAB oder LINE separated)""" 1929: 1930: 1931: if upload: # list from file upload 1932: txt=upload.read() 1933: 1934: if enterList: 1935: txt=enterList 1936: 1937: if upload or enterList: 1938: txt=txt.replace(",","\n") 1939: txt=txt.replace("\t","\n") 1940: txt=txt.replace("\r","\n") 1941: idsTmp=txt.split("\n") 1942: ids=[] 1943: for id in idsTmp: # make sure that no empty lines 1944: idTmp=id.lstrip().rstrip() 1945: if len(idTmp)>0: 1946: 1947: ids.append(idTmp) 1948: 1949: #self.REQUEST.SESSION['ids']=" OR ".join(ids) 1950: 1951: pt=getattr(self,'filelist.html') 1952: self.REQUEST.SESSION['searchList']=ids 1953: return pt(search=ids) 1954: 1955: if basketName: 1956: #TODO: get rid of one of these.. 1957: 1958: pt=getattr(self,'filelist.html') 1959: return pt(basketName=basketName,numberOfObjects=numberOfObjects) 1960: 1961: if list is not None: # got already a list 1962: ret=[] 1963: for fileId in list: 1964: if fileId.find("*"): #check for wildcards 1965: fileId=fileId 1966: elif len(fileId.split("."))==1: 1967: fileId=fileId+".atf" 1968: 1969: ret+=self.CDLICatalog({'title':fileId}) 1970: #TODO: get rid of one of these.. 1971: ids=[x.getObject().getId() for x in ret] 1972: self.REQUEST.SESSION['fileIds']=ids#store fieldIds in session for further usage 1973: self.REQUEST.SESSION['searchList']=self.REQUEST.SESSION['fileIds'] 1974: 1975: if display: 1976: pt=getattr(self,'filelist.html') 1977: 1978: return pt(search=ids) 1979: else: 1980: return ret 1981: 1982: 1983: 1984: if start: 1985: RESPONSE.redirect("filelist.html?start:int="+str(start)) 1986: 1987: 1988: security.declareProtected('Manage','createAllFilesAsSingleFile') 1989: def createAllFilesAsSingleFile(self,RESPONSE=None): 1990: """download all files""" 1991: 1992: def sortF(x,y): 1993: return cmp(x[0],y[0]) 1994: 1995: catalog=getattr(self,self.file_catalog) 1996: #tf,tfilename=mkstemp() 1997: if not hasattr(self.temp_folder,'downloadCounter'): 1998: self.temp_folder.downloadCounter=0 1999: 2000: if getattr(self.temp_folder,'downloadCounter',0) > 5: 2001: return """I am sorry, currently the server has to many requests for downloads, please come back later!""" 2002: 2003: self.temp_folder.downloadCounter+=1 2004: self._p_changed=1 2005: transaction.get().commit() 2006: 2007: list=[(x.getId,x) for x in catalog()] 2008: list.sort(sortF) 2009: 2010: 2011: 2012: RESPONSE.setHeader("Content-Disposition","""attachement; filename=%s"""%"all.atf") 2013: RESPONSE.setHeader("Content-Type","application/octet-stream") 2014: tmp="" 2015: for l in list: 2016: obj=l[1].getObject() 2017: 2018: if obj.meta_type=="CDLI file": 2019: 2020: #os.write(tf,obj.getLastVersion().data) 2021: if RESPONSE: 2022: RESPONSE.write(obj.getData()[0:]) 2023: RESPONSE.write("\n") 2024: self.temp_folder.downloadCounter-=1 2025: self._p_changed=1 2026: transaction.get().commit() 2027: #os.close(tf) 2028: #RESPONSE.redirect(self.absolute_url()+"/downloadFile?fn="%tfilename) 2029: return True 2030: 2031: def downloadFile(self,fn): 2032: """download fn - not used yet""" 2033: self.REQUEST.RESPONSE.setHeader("Content-Disposition","""attachement; filename=%s"""%self.getLastVersion().getId()) 2034: self.REQUEST.RESPONSE.setHeader("Content-Type","application/octet-stream") 2035: self.REQUEST.RESPONSE.write(file(fn).read()) 2036: 2037: 2038: 2039: def hasParent(self): 2040: """returns true falls subfolder""" 2041: 2042: if self.aq_parent.meta_type in self.folder_meta_type: 2043: return True 2044: else: 2045: return False 2046: 2047: def getFolders(self): 2048: """get all subfolders""" 2049: ret=[] 2050: folders=self.ZopeFind(self,obj_metatypes=self.folder_meta_type) 2051: for folder in folders: 2052: ret.append((folder[1], 2053: len(self.ZopeFind(folder[1],obj_metatypes=self.folder_meta_type)), 2054: len(self.ZopeFind(folder[1],obj_metatypes=self.file_meta_type)) 2055: )) 2056: return ret 2057: 2058: 2059: security.declareProtected('manage','index_html') 2060: def index_html(self): 2061: """main""" 2062: ext=self.ZopeFind(self,obj_ids=["index.html"]) 2063: if ext: 2064: return ext[0][1]() 2065: 2066: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','CDLIFileFolderMain')).__of__(self) 2067: return pt() 2068: 2069: 2070: manage_addCDLIFileFolderForm=DTMLFile('dtml/folderAdd', globals()) 2071: 2072: 2073: def manage_addCDLIFileFolder(self, id, title='', 2074: createPublic=0, 2075: createUserF=0, 2076: REQUEST=None): 2077: """Add a new Folder object with id *id*. 2078: 2079: If the 'createPublic' and 'createUserF' parameters are set to any true 2080: value, an 'index_html' and a 'UserFolder' objects are created respectively 2081: in the new folder. 2082: """ 2083: ob=CDLIFileFolder() 2084: ob.id=str(id) 2085: ob.title=title 2086: self._setObject(id, ob) 2087: ob=self._getOb(id) 2088: 2089: checkPermission=getSecurityManager().checkPermission 2090: 2091: if createUserF: 2092: if not checkPermission('Add User Folders', ob): 2093: raise Unauthorized, ( 2094: 'You are not authorized to add User Folders.' 2095: ) 2096: ob.manage_addUserFolder() 2097: 2098: 2099: if REQUEST is not None: 2100: return self.manage_main(self, REQUEST, update_menu=1) 2101: 2102: class CDLIRoot(Folder): 2103: """main folder for cdli""" 2104: 2105: meta_type="CDLIRoot" 2106: downloadCounterBaskets=0 # counts the current basket downloads if counter > 10 no downloads are possible 2107: 2108: file_catalog = 'CDLICatalog' 2109: 2110: # word splitter for search 2111: splitter = {'words':cdliSplitter.wordSplitter(), 2112: 'graphemes':cdliSplitter.graphemeSplitter()} 2113: 2114: 2115: def deleteFiles(self,ids): 2116: """delete files""" 2117: for id in ids: 2118: founds=self.CDLICatalog.search({'title':id.split(".")[0]}) 2119: if founds: 2120: logging.debug("deleting %s"%founds) 2121: folder=founds[0].getObject().aq_parent #get the parent folder of the object 2122: logging.debug("deleting from %s"%folder) 2123: cut=folder.delete([founds[0].getId]) #cut it out 2124: 2125: 2126: 2127: def searchText(self, query, index='graphemes'): 2128: """searches query in the fulltext index and returns a list of file ids/P-numbers""" 2129: # see also: http://www.plope.com/Books/2_7Edition/SearchingZCatalog.stx#2-13 2130: idxQuery = {index:{'query':query}} 2131: idx = getattr(self, self.file_catalog) 2132: # do search 2133: resultset = idx.search(query_request=idxQuery,sort_index='textid') 2134: # put only the P-Number in the result 2135: results = [res.getId[:7] for res in resultset] 2136: return results 2137: 2138: 2139: def getFile(self, pnum): 2140: """get the translit file with the given pnum""" 2141: f = getattr(self, self.file_catalog).search({'textid':pnum}) 2142: if not f: 2143: return "" 2144: 2145: return f[0].getObject().getData() 2146: 2147: 2148: 2149: def showFile(self,fileId,wholePage=False): 2150: """show a file 2151: @param fileId: P-Number of the document to be displayed 2152: """ 2153: f=getattr(self, self.file_catalog).search({'textid':fileId}) 2154: if not f: 2155: return "" 2156: 2157: if wholePage: 2158: logging.debug("show whole page") 2159: return f[0].getObject().getContentObject().view() 2160: else: 2161: return f[0].getObject().getLastVersionFormattedData() 2162: 2163: 2164: def showWordInFile(self,fileId,word,indexName='graphemes',regExp=False,): 2165: """get lines with word from FileId""" 2166: 2167: file = formatAtfFullLineNum(self.getFile(fileId)) 2168: ret=[] 2169: 2170: # add whitespace before and whitespace and line-end to splitter bounds expressions 2171: bounds = self.splitter[indexName].bounds 2172: splitexp = "(%s|\s)(%%s)(%s|\s|\Z)"%(bounds,bounds) 2173: # compile into regexp objects 2174: wordlist = [re.compile(splitexp%w,re.IGNORECASE) for w in word.split(' ')] 2175: 2176: for line in file.split("\n"): 2177: for word in wordlist: 2178: #logging.debug("showwordinfile: searching for %s in %s"%(word.pattern,line)) 2179: if word.search(line): 2180: line = formatAtfLineHtml(line) 2181: ret.append(line) 2182: break 2183: 2184: return ret 2185: 2186: 2187: def tagWordInFile(self,fileId,word,indexName='graphemes',regExp=False): 2188: """get text with word highlighted from FileId""" 2189: 2190: file=self.getFile(fileId) 2191: tagStart=u'<span class="found">' 2192: tagEnd=u'</span>' 2193: tagStr=tagStart + u'%%s' + tagEnd 2194: ret=[] 2195: 2196: # add whitespace to splitter bounds expressions and compile into regexp object 2197: bounds = self.splitter[indexName].bounds 2198: wordsplit = re.compile("(%s|\s)"%bounds) 2199: # split search terms by blanks 2200: words = word.split(' ') 2201: 2202: for line in file.split("\n"): 2203: line = unicodify(line) 2204: # ignore lemma lines 2205: if line.lstrip().startswith('#lem:'): 2206: continue 2207: 2208: # first scan 2209: hitwords = [] 2210: for w in words: 2211: if line.find(w) > -1: 2212: # word is in line 2213: hitwords.append(w) 2214: 2215: # examine hits closer 2216: if hitwords: 2217: # split line into words 2218: parts = wordsplit.split(line) 2219: line = "" 2220: for p in parts: 2221: # reassemble line 2222: if p in hitwords: 2223: # this part was found 2224: line += tagStart + formatAtfHtml(p) + tagEnd 2225: else: 2226: line += formatAtfHtml(p) 2227: 2228: else: 2229: # no hits 2230: line = formatAtfHtml(line) 2231: 2232: ret.append(line) 2233: 2234: return u'<br>\n'.join(ret) 2235: 2236: 2237: def URLquote(self,str): 2238: """quote url""" 2239: return urllib.quote(str) 2240: 2241: def URLunquote(self,str): 2242: """unquote url""" 2243: return urllib.unquote(str) 2244: 2245: def URLquote_plus(self,str): 2246: """quote url""" 2247: return urllib.quote_plus(str) 2248: 2249: def URLunquote_plus(self,str): 2250: """unquote url""" 2251: return urllib.unquote_plus(str) 2252: 2253: 2254: def forceunlock(self): 2255: "break all locks" 2256: ret=[] 2257: for f in self.ZopeFind(self,obj_metatypes="CDLI file",search_sub=1): 2258: un=f[1].forceunlock() 2259: 2260: if un and un !="": 2261: ret.append((f[0],un)) 2262: 2263: return ret 2264: 2265: 2266: def getChangesByAuthor(self,author,n=100): 2267: """getChangesByAuthor""" 2268: zcat=self.CDLIObjectsCatalog 2269: res=zcat({'lastEditor':author, 2270: 'sort_on':'getTime', 2271: 'sort_order':'descending', 2272: 'sort_limit':n})[:n ] 2273: 2274: return res 2275: 2276: def getChangesByAuthor_html(self,author,n=100): 2277: """html output for changes by author""" 2278: tmp={} 2279: list=[] 2280: for x in self.getChangesByAuthor(author): 2281: nr=x.getObject().getVersionNumber() 2282: id=x.getObject().aq_parent.getId() 2283: #hinzufuegen, wenn Version neuer als die 2284: if tmp.get(id,(0,0))[1] < nr: 2285: tmp[id]=(x.getObject().aq_parent,nr) 2286: 2287: 2288: return self.cdli_main.findObjectsFromListWithVersion(list=tmp.values(),author=author) 2289: 2290: def getLastChanges(self,n=100): 2291: """get the last n changes""" 2292: n=int(n) 2293: zcat=self.CDLICatalog 2294: return zcat({'sort_on':'getLastChangeDate', 2295: 'sort_order':'descending', 2296: 'sort_limit':n})[:n ] 2297: 2298: 2299: def getLastChanges_html(self,n=100): 2300: """get the last n changes""" 2301: list = [x.getId for x in self.getLastChanges(n)] 2302: return self.cdli_main.findObjectsFromList(list=list,display=True) 2303: 2304: def refreshTxt(self,txt="",threadName=None): 2305: """txt fuer refresh""" 2306: 2307: return """ 2;url=%s?repeat=%s """%(self.absolute_url()+txt,threadName) 2308: 2309: 2310: def getResult(self,threadName=None): 2311: """result of thread""" 2312: try: 2313: return self._v_uploadATF[threadName].getResult() 2314: except: 2315: return "One moment, please" 2316: 2317: 2318: def checkThreads(self): 2319: """check threads""" 2320: ret="<html><body>" 2321: for thread in threading.enumerate(): 2322: ret+="<p>%s (%s): %s</p>"%(repr(thread),thread.getName(),thread.isAlive()) 2323: 2324: return ret 2325: 2326: 2327: def uploadATFRPC(self,data,username): 2328: """upload an atffile via xml-rpc""" 2329: uploader=uploadATFThread() 2330: 2331: #generate an random id for the upload object 2332: from random import randint 2333: if (not self.REQUEST.SESSION.get('idTmp',None)): 2334: 2335: idTmp=str(randint(0,1000000000)) 2336: self.REQUEST.SESSION['idTmp']=idTmp 2337: else: 2338: idTmp=self.REQUEST.SESSION.get('idTmp',None) 2339: 2340: 2341: uploader.set(data,0,username,idTmp) 2342: 2343: stObj=uploader.run() 2344: 2345: processor=uploadATFfinallyThread() 2346: 2347: basketname=stObj.returnValue['basketNameFromFile'] 2348: 2349: processor.set("uploadchanged",basketname=basketname,SESSION=stObj.returnValue,username=username,serverport=self.REQUEST['SERVER_PORT']) 2350: 2351: processor.run() 2352: 2353: 2354: return generateXMLReturn(stObj.returnValue) 2355: 2356: def uploadATF(self,repeat=None,upload=None,basketId=0,RESPONSE=None): 2357: """upload an atf file / basket file""" 2358: #self._v_uploadATF.returnValue=None 2359: 2360: #generate an random id for the upload thread 2361: from random import randint 2362: if (not self.REQUEST.SESSION.get('idTmp',None)): 2363: 2364: idTmp=str(randint(0,1000000000)) 2365: self.REQUEST.SESSION['idTmp']=idTmp 2366: else: 2367: idTmp=self.REQUEST.SESSION.get('idTmp',None) 2368: 2369: 2370: threadName=repeat 2371: if not threadName or threadName=="": 2372: #new thread not called from the waiting page 2373: tmpVar=False 2374: 2375: thread=uploadATFThread() 2376: threadName=thread.getName()[0:] 2377: if (not hasattr(self,'_v_uploadATF')): 2378: self._v_uploadATF={} 2379: 2380: self._v_uploadATF[threadName]=thread 2381: #self._xmltrans.start() 2382: #thread=Thread(target=self._v_uploadATF) 2383: logging.info("set thread. extern") 2384: self._v_uploadATF[threadName].set(upload,basketId,self.REQUEST['AUTHENTICATED_USER'],idTmp,serverport=self.REQUEST['SERVER_PORT']) 2385: #thread.start() 2386: logging.info("start thread. extern") 2387: self._v_uploadATF[threadName].start() 2388: 2389: 2390: self.threadName=self._v_uploadATF[threadName].getName()[0:] 2391: wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template']) 2392: 2393: if wait_template: 2394: return wait_template[0][1]() 2395: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self) 2396: return pt(txt='/uploadATF',threadName=threadName) 2397: #_v_xmltrans.run() 2398: 2399: else: 2400: #recover thread, if lost 2401: if (not hasattr(self,'_v_uploadATF')): 2402: self._v_uploadATF={} 2403: if not self._v_uploadATF.get(threadName,None): 2404: for thread in threading.enumerate(): 2405: if threadName == thread.getName(): 2406: self._v_uploadATF[threadName]=thread 2407: 2408: if self._v_uploadATF.get(threadName,None) and (not self._v_uploadATF[threadName].returnValue): 2409: 2410: 2411: wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template']) 2412: if wait_template: 2413: return wait_template[0][1]() 2414: 2415: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self) 2416: 2417: return pt(txt='/uploadATF',threadName=threadName) 2418: 2419: else: 2420: tmp=getattr(self.temp_folder,idTmp).returnValue 2421: 2422: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadCheck.zpt')).__of__(self) 2423: 2424: return pt(changed=tmp['changed'],lockerrors=tmp['lockerrors'],errors=tmp['errors'],dir=tmp['dir'],newPs=tmp['newPs'],basketLen=tmp['basketLen'],numberOfFiles=tmp['numberOfFiles'], 2425: basketNameFromId=tmp['basketNameFromId'],basketNameFromFile=tmp['basketNameFromFile'],basketId=tmp['basketId']) 2426: 2427: def redoUpload(self,threadName): 2428: """redo the upload""" 2429: tmp=self.cdli_main.tmpStore2[threadName] 2430: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadCheck.zpt')).__of__(self) 2431: return pt(changed=tmp['changed'],lockerrors=tmp['lockerrors'],errors=tmp['errors'],dir=tmp['dir'],newPs=tmp['newPs'],basketLen=tmp['basketLen'],numberOfFiles=tmp['numberOfFiles'], 2432: basketNameFromId=tmp['basketNameFromId'],basketNameFromFile=tmp['basketNameFromFile'],basketId=tmp['basketId']) 2433: 2434: def uploadATFfinally(self,procedure='',comment="",basketname='',unlock=None,repeat=None,RESPONSE=None): 2435: """nowupload the files""" 2436: 2437: 2438: 2439: threadName=repeat 2440: if not threadName or threadName=="": 2441: thread=uploadATFfinallyThread() 2442: threadName=thread.getName()[0:] 2443: 2444: if (not hasattr(self,'_v_uploadATF')): 2445: self._v_uploadATF={} 2446: 2447: 2448: self._v_uploadATF[threadName]=thread 2449: 2450: idTmp=self.REQUEST.SESSION['idTmp'] 2451: stObj=getattr(self.temp_folder,idTmp) 2452: self._v_uploadATF[threadName].set(procedure,comment=comment,basketname=basketname,unlock=unlock,SESSION=stObj.returnValue,username=self.REQUEST['AUTHENTICATED_USER'],serverport=self.REQUEST['SERVER_PORT']) 2453: 2454: self._v_uploadATF[threadName].start() 2455: 2456: 2457: 2458: wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template']) 2459: 2460: if wait_template: 2461: return wait_template[0][1]() 2462: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self) 2463: 2464: return pt(txt='/uploadATFfinally',threadName=threadName) 2465: #_v_xmltrans.run() 2466: 2467: else: 2468: #recover thread, if lost 2469: if not hasattr(self,'_v_uploadATF'): 2470: self._v_uploadATF={} 2471: if not self._v_uploadATF.get(threadName,None): 2472: for thread in threading.enumerate(): 2473: if threadName == thread.getName(): 2474: self._v_uploadATF[threadName]=thread 2475: 2476: if self._v_uploadATF.get(threadName,None) and (self._v_uploadATF[threadName] is not None) and (not self._v_uploadATF[threadName].end) : 2477: 2478: wait_template=self.aq_parent.ZopeFind(self.aq_parent,obj_ids=['wait_template']) 2479: if wait_template: 2480: return wait_template[0][1]() 2481: 2482: pt=PageTemplateFile(os.path.join(package_home(globals()),'zpt','uploadATFWait.zpt')).__of__(self) 2483: return pt(txt='/uploadATFfinally',threadName=threadName) 2484: else: 2485: self.REQUEST.SESSION['idTmp']=None 2486: if RESPONSE is not None: 2487: RESPONSE.redirect(self.absolute_url()) 2488: 2489: def importFiles(self,comment="",author="" ,folderName="/Users/dwinter/atf", files=None,ext=None): 2490: """import files""" 2491: root=self.cdli_main 2492: count=0 2493: if not files: 2494: files=os.listdir(folderName) 2495: 2496: for f in files: 2497: folder=f[0:3] 2498: f2=f[0:5] 2499: obj=self.ZopeFind(root,obj_ids=[folder]) 2500: if ext: 2501: 2502: ext.result="<p>adding: %s </p>"%f+ext.result 2503: if not obj: 2504: manage_addCDLIFileFolder(root,folder,folder) 2505: fobj=getattr(root,folder) 2506: #transaction.get().commit() 2507: else: 2508: fobj=obj[0][1] 2509: 2510: obj2=fobj.ZopeFind(fobj,obj_ids=[f2]) 2511: 2512: if not obj2: 2513: manage_addCDLIFileFolder(fobj,f2,f2) 2514: fobj2=getattr(fobj,f2) 2515: 2516: else: 2517: fobj2=obj2[0][1] 2518: 2519: file2=os.path.join(folderName,f) 2520: id=f 2521: manage_addCDLIFile(fobj2,f,'','') 2522: id=f 2523: ob=fobj2._getOb(f) 2524: ob.title=id 2525: 2526: manage_addCDLIFileObject(ob,id,comment,author,file2,content_type='',from_tmp=True) 2527: self.CDLICatalog.catalog_object(ob) 2528: #self.CDLICatalog.manage_catalogFoundItems(obj_ids=[id],search_sub=1) 2529: #self.CDLICatalog.manage_catalogObject(self.REQUEST, self.REQUEST.RESPONSE, 'CDLICatalog', urlparse.urlparse(ob.absolute_url())[1]) 2530: count+=1 2531: 2532: if count > 1000: 2533: print "committing" 2534: transaction.get().commit() 2535: count=0 2536: transaction.get().commit() 2537: return "ok" 2538: 2539: 2540: manage_addCDLIRootForm=DTMLFile('dtml/rootAdd', globals()) 2541: 2542: 2543: def manage_addCDLIRoot(self, id, title='', 2544: createPublic=0, 2545: createUserF=0, 2546: REQUEST=None): 2547: """Add a new Folder object with id *id*. 2548: 2549: If the 'createPublic' and 'createUserF' parameters are set to any true 2550: value, an 'index_html' and a 'UserFolder' objects are created respectively 2551: in the new folder. 2552: """ 2553: ob=CDLIRoot() 2554: ob.id=str(id) 2555: ob.title=title 2556: try: 2557: self._setObject(id, ob) 2558: except: 2559: pass 2560: ob=self._getOb(id) 2561: 2562: checkPermission=getSecurityManager().checkPermission 2563: 2564: if createUserF: 2565: if not checkPermission('Add User Folders', ob): 2566: raise Unauthorized, ( 2567: 'You are not authorized to add User Folders.' 2568: ) 2569: ob.manage_addUserFolder() 2570: 2571: 2572: if REQUEST is not None: 2573: return self.manage_main(self, REQUEST, update_menu=1) 2574: