Return to cdliSplitter.py CVS log

Up to [Repository] / cdli

fixed big in highlighting regexp

    1: """
    2: CDLI word and grapheme splitter
    3: """
    4: 
    5: from Products.ZCTextIndex.PipelineFactory import element_factory
    6: 
    7: import re
    8: import logging
    9: 
   10: def getSupportedEncoding(encodings):
   11:     for encoding in encodings:
   12:         try:
   13:             unicode('A', encoding)
   14:             return encoding
   15:         except:
   16:             pass
   17:     return 'utf-8'
   18: 
   19: 
   20: 
   21: """beta of a fulltext splitter for cdli
   22: 
   23: """
   24: ignoreLines=['$','@','#','&','>']
   25: separators=['']
   26: # kommas relevant for graphemes will not be deleted
   27: komma_exception="([^sStThH])," 
   28: # grapheme boundaries
   29: #graphemeBounds="\{|\}|<|>|\(|\)|-|_|\#|,|\||\]|\[|\!|\?"
   30: graphemeBounds="\{|\}|<|>|-|_|\#|,|\]|\[|\!|\?|\""
   31: # for words 
   32: #wordBounds="<|>|\(|\)|_|\#|,|\||\]|\[|\!|\?"
   33: wordBounds="<|>|_|\#|,|\]|\[|\!|\?|\""
   34: 
   35:            
   36: class cdliSplitter:
   37:     """base class for splitter. 
   38:     the difference between word and grapheme splitter 
   39:     is the word boundary list."""
   40:     
   41:     default_encoding = "utf-8"
   42:     bounds=graphemeBounds
   43:     indexName="cdliSplitter"
   44:     
   45:     
   46:     def process(self, lst):
   47:         """gets a list of strings and returns a list of words"""
   48:         
   49:         logging.debug("cdliSplitter: %s"%self.indexName) 
   50:         result = []
   51:         pNum=None
   52:         lineNum=None
   53:     
   54:         for t in lst:
   55:             # normalise line breaks
   56:             t.replace("\r","\n")
   57:             # split lines
   58:             for s in t.split("\n"):
   59:                 if isinstance(s, str): 
   60:                     # not unicode
   61:                     s = unicode(s, self.default_encoding, 'replace')
   62:          
   63:                 if (s!=''):
   64:                     if s[0]=='&': 
   65:                         # store pNum
   66:                         pNum=s[1:8]
   67:                         logging.debug("%s processing: %s"%(self.indexName,pNum))
   68:                         
   69:                     elif not (s[0] in ignoreLines):
   70:                         # regular line
   71:                         lineparts=s.split(".")
   72:                         if len(lineparts)==1: 
   73:                             # no line number
   74:                             txt=s
   75:                         else:
   76:                             #store line number
   77:                             txt=lineparts[1]
   78:                             lineNum=lineparts[0] 
   79:                             
   80:                         # delete kommata except kommata relevant for graphemes
   81:                         txt = re.sub(komma_exception,r"\1",txt)
   82:                         # replace word boundaries by spaces
   83:                         txt = re.sub(self.bounds,' ',txt)
   84:                         # split words
   85:                         words = txt.split(" ")
   86:                         for w in words:
   87:                             w=w.strip()
   88:                             if not (w==''):
   89:                                 result.append(w)
   90: 
   91:         logging.debug("split '%s' into %s"%(lst,repr(result)))
   92:         return result
   93: 
   94: 
   95: class graphemeSplitter(cdliSplitter):
   96:     bounds=graphemeBounds
   97:     indexName="graphemeSplitter"
   98:     
   99: class wordSplitter(cdliSplitter):
  100:     bounds=wordBounds
  101:     indexName="wordSplitter"
  102:       
  103: try:
  104:     element_factory.registerFactory('Word Splitter',
  105:           'CDLI grapheme splitter', graphemeSplitter)
  106: except:
  107:     # in case the splitter is already registered, ValueError is raised
  108:     pass
  109: 
  110: try:
  111:     element_factory.registerFactory('Word Splitter',
  112:           'CDLI word splitter', wordSplitter)
  113: except:
  114:     # in case the splitter is already registered, ValueError is raised
  115:     pass
  116: