Integrate Full Text search functionality in to your applications with Lucene (indexing)

Part 1) Indexing your data.

Any Full Text search functionality involves indexing the data first. Lucene was no different in this approach. By indexing your data, it can perform high-performance full-text searching very fast. I did indexed 17,000 html files (my product documentation) in less than 5 minutes.

Creating Index writer & adding documents methods are key.
Rest f the methods for book keeping.
Following code indexes html, htm files in a folder. (It recursively iterates the nested folders & indexes each file)


////you data
public static final String dataDir = "D:\\webapps\\help";
//the directory that is used to store lucene index
private final String indexDir = "D:\\help_index";

public static String src1 = "";
public IndexWriter indexWriter;
public static int numF;
public static int numD;

public void openIndexWriter()throws IOException
{
Directory fsDirectory = FSDirectory.getDirectory(indexDir);
Analyzer analyzer = new StandardAnalyzer();
indexWriter = new IndexWriter(fsDirectory, true, analyzer);
indexWriter.setWriteLockTimeout(IndexWriter.WRITE_LOCK_TIMEOUT * 100 );
}

public void closeIndexWriter()throws IOException
{
indexWriter.optimize();
indexWriter.close();
}

public void indexFiles(String strPath) throws IOException
{
File src = new File(strPath);
if (src.isDirectory())
{
numD++;
String list[] = src.list();
try
{
for (int i = 0; i < list.length; i++)
{
src1 = src.getAbsolutePath() + File.separatorChar + list[i];
File file = new File(src1);
/*
* Try check like read/write access check etc.
*/
if ( file.isDirectory() )indexFiles(src1);
else
{
numF++;
if(src1.endsWith(".html") src1.endsWith(".htm")){
addDocument(src1, indexWriter);
}
}
}
}catch(java.lang.NullPointerException e){}
}
}
public boolean createIndex() throws IOException{
if(true == ifIndexExist()){
return true;
}
File dir = new File(dataDir);
if(!dir.exists()){
return false;
}
File[] htmls = dir.listFiles();
Directory fsDirectory = FSDirectory.getDirectory(indexDir);
Analyzer analyzer = new StandardAnalyzer();
IndexWriter indexWriter = new IndexWriter(fsDirectory, analyzer, true);
for(int i = 0; i < htmls.length; i++){
String htmlPath = htmls[i].getAbsolutePath();
if(htmlPath.endsWith(".html") htmlPath.endsWith(".htm")){
addDocument(htmlPath, indexWriter);
}
}
return true;
}
/**
* Add one document to the lucene index
*/
public void addDocument(String htmlPath, IndexWriter indexWriter){
//System.out.println("\n adding file to index "+htmlPath );
HTMLDocParser htmlParser = new HTMLDocParser(htmlPath);
String path = htmlParser.getPath();
String title = htmlParser.getTitle();
Reader content = htmlParser.getContent();
Document document = new Document();
document.add(new Field("path",path,Field.Store.YES,Field.Index.NO));
document.add(new Field("title",title,Field.Store.YES,Field.Index.TOKENIZED));
document.add(new Field("content",content));
try {
indexWriter.addDocument(document);
} catch (IOException e) {
e.printStackTrace();
}
}

im.openIndexWriter();
File src = new File(dataDir);
if(!src.exists()){
System.out.println("\n DATA DIR DOES NOT EXISTS" );
return;
}
long start = System.currentTimeMillis();
System.out.println("\n INDEXING STARTED" );
im.indexFiles(dataDir);
im.closeIndexWriter();
long end = System.currentTimeMillis();
long diff = (end-start)/1000;
System.out.println("\n Time consumed in Index the whole help=" +diff );
System.out.println("Number of files :\t"+numF);
System.out.println("Number of dirs :\t"+numD);
}