--- cdli/cdli_files.py 2007/12/03 21:30:19 1.80.2.10 +++ cdli/cdli_files.py 2007/12/13 19:20:45 1.80.2.11 @@ -2178,15 +2178,18 @@ class CDLIRoot(Folder): splitexp = "(%s|\s)(%%s)(%s|\s|\Z)"%(bounds,bounds) # clean word expression # TODO: this should use QueryParser itself - word = word.replace('"','') # take out double quotes - # escape parens for regexp too - # compile into regexp objects + # take out double quotes + word = word.replace('"','') + # take out ignorable signs + ignorable = self.splitter[indexName].ignorex + word = ignorable.sub('', word) + # compile into regexp objects and escape parens wordlist = [re.compile(splitexp%re.escape(w)) for w in word.split(' ')] for line in file.split("\n"): for word in wordlist: - #logging.debug("showwordinfile: searching for %s in %s"%(word.pattern,line)) - if word.search(line): + #logging.debug("showwordinfile: searching for %s in %s"%(word.pattern,ignoreable.sub('',line))) + if word.search(ignorable.sub('',line)): line = formatAtfLineHtml(line) ret.append(line) break @@ -2220,8 +2223,13 @@ class CDLIRoot(Folder): # clean word expression # TODO: this should use QueryParser itself word = word.replace('"','') # take out double quotes + # take out ignoreable signs + ignorable = self.splitter[indexName].ignorex + word = ignorable.sub('', word) # split search terms by blanks words = word.split(' ') + # split search terms again (for grapheme search with words) + splitwords = dict(((w,self.splitter[indexName].process([w])) for w in words)) for line in file.split("\n"): line = unicodify(line) @@ -2232,9 +2240,11 @@ class CDLIRoot(Folder): # first scan hitwords = [] for w in words: - if line.find(w) > -1: + if ignorable.sub('',line).find(w) > -1: # word is in line - hitwords.append(w) + # append split word for grapheme search with words + hitwords.extend(splitwords[w]) + #hitwords.extend(wordsplit.split(w)) # examine hits closer if hitwords: @@ -2242,8 +2252,10 @@ class CDLIRoot(Folder): parts = wordsplit.split(line) line = "" for p in parts: + #logging.debug("tagwordinfile: searching for %s in %s"%(p,hitwords)) # reassemble line - if p in hitwords: + if ignorable.sub('', p) in hitwords: + #logging.debug("tagwordinfile: found %s in %s"%(p,hitwords)) # this part was found line += tagStart + formatAtfHtml(p) + tagEnd else: