#!/usr/bin/env python
import sys, os, PyLucene, threading, time
import xmlrpclib
from datetime import datetime
"""
This class is loosely based on the Lucene (java implementation) demo class
org.apache.lucene.demo.IndexFiles. It will take a directory as an argument
and will index all of the files in that directory and downward recursively.
It will index on the file path, the file name and the file contents. The
resulting Lucene index will be placed in the current directory and called
'index'.
"""
class Ticker(object):
def __init__(self):
self.tick = True
def run(self):
while self.tick:
sys.stdout.write('.')
sys.stdout.flush()
time.sleep(1.0)
class IndexFiles(object):
"""Usage: python IndexFiles"""
def __init__(self, storeDir, analyzer):
if not os.path.exists(storeDir):
os.mkdir(storeDir)
store = PyLucene.FSDirectory.getDirectory(storeDir, True)
writer = PyLucene.IndexWriter(store, analyzer, True)
writer.setMaxFieldLength(1048576)
self.indexDocs(writer)
ticker = Ticker()
print 'optimizing index',
threading.Thread(target=ticker.run).start()
writer.optimize()
writer.close()
ticker.tick = False
print 'done'
def indexDocs(self, writer):
s=xmlrpclib.Server("http://127.0.0.1:8080/cdliRoot/cdli_main")
pns=s.getAllPNumbers()
for pn in pns:
txt=s.getFile(pn)
i=0
for line in txt.split("\n"):
doc = PyLucene.Document()
doc.add(PyLucene.Field("name", pn,
PyLucene.Field.Store.YES,
PyLucene.Field.Index.UN_TOKENIZED))
doc.add(PyLucene.Field("line", str(i),
PyLucene.Field.Store.YES,
PyLucene.Field.Index.UN_TOKENIZED))
doc.add(PyLucene.Field("contents", line,
PyLucene.Field.Store.YES,
PyLucene.Field.Index.TOKENIZED))
writer.addDocument(doc)
i+=1
print " %s"%i
print "indexed %s"%pn
if __name__ == '__main__':
print 'PyLucene', PyLucene.VERSION, 'Lucene', PyLucene.LUCENE_VERSION
start = datetime.now()
try:
IndexFiles("index", PyLucene.StandardAnalyzer())
end = datetime.now()
print end - start
except Exception, e:
print "Failed: ", e
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>