--- cdli/cdliSplitter.py 2007/10/19 16:25:07 1.7.2.2 +++ cdli/cdliSplitter.py 2007/12/03 21:30:19 1.7.2.6 @@ -26,9 +26,11 @@ separators=[''] # kommas relevant for graphemes will not be deleted komma_exception="([^sStThH])," # grapheme boundaries -graphemeBounds="\{|\}|<|>|\(|\)|-|_|\#|,|\||\]|\[|\!|\?" +#graphemeBounds="\{|\}|<|>|\(|\)|-|_|\#|,|\||\]|\[|\!|\?" +graphemeBounds="\{|\}|<|>|-|_|\#|,|\]|\[|\!|\?|\"" # for words -wordBounds="<|>|\(|\)|_|\#|,|\||\]|\[|\!|\?" +#wordBounds="<|>|\(|\)|_|\#|,|\||\]|\[|\!|\?" +wordBounds="<|>|_|\#|,|\]|\[|\!|\?|\"" class cdliSplitter: @@ -44,7 +46,7 @@ class cdliSplitter: def process(self, lst): """gets a list of strings and returns a list of words""" - logging.debug("cdliSplitter") + logging.debug("cdliSplitter: %s"%self.indexName) result = [] pNum=None lineNum=None @@ -62,7 +64,7 @@ class cdliSplitter: if s[0]=='&': # store pNum pNum=s[1:8] - logging.debug("%s processing: %s"%(indexName,pNum)) + logging.debug("%s processing: %s"%(self.indexName,pNum)) elif not (s[0] in ignoreLines): # regular line @@ -86,6 +88,7 @@ class cdliSplitter: if not (w==''): result.append(w) + logging.debug("split '%s' into %s"%(lst,repr(result))) return result