--- cdli/cdliSplitter.py 2007/10/19 16:30:58 1.7.2.3 +++ cdli/cdliSplitter.py 2007/12/03 21:30:19 1.7.2.6 @@ -26,9 +26,11 @@ separators=[''] # kommas relevant for graphemes will not be deleted komma_exception="([^sStThH])," # grapheme boundaries -graphemeBounds="\{|\}|<|>|\(|\)|-|_|\#|,|\||\]|\[|\!|\?" +#graphemeBounds="\{|\}|<|>|\(|\)|-|_|\#|,|\||\]|\[|\!|\?" +graphemeBounds="\{|\}|<|>|-|_|\#|,|\]|\[|\!|\?|\"" # for words -wordBounds="<|>|\(|\)|_|\#|,|\||\]|\[|\!|\?" +#wordBounds="<|>|\(|\)|_|\#|,|\||\]|\[|\!|\?" +wordBounds="<|>|_|\#|,|\]|\[|\!|\?|\"" class cdliSplitter: @@ -44,7 +46,7 @@ class cdliSplitter: def process(self, lst): """gets a list of strings and returns a list of words""" - logging.debug("cdliSplitter") + logging.debug("cdliSplitter: %s"%self.indexName) result = [] pNum=None lineNum=None @@ -86,6 +88,7 @@ class cdliSplitter: if not (w==''): result.append(w) + logging.debug("split '%s' into %s"%(lst,repr(result))) return result