Hey, you should compare with the ThreadedIndexWriter too :). I'll attach the source from Lucene in action SE manual and you can just replace the new IntexWriter(... with new ThreadedIndexWriter(...
See if those results make a difference. Also I presume you don't have a single core cpu 2011/10/11 Marc Sturlese <marc.sturl...@gmail.com> > I'm doing some performance test doing bulk indexing with lucene 4.0 >
package org.ubervu.search.CustomLucene; import java.io.IOException; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; public class ThreadedIndexWriter extends IndexWriter { private ExecutorService threadPool; private class Job implements Runnable { //A Document doc; Analyzer analyzer; Term delTerm; public Job(Document doc, Term delTerm, Analyzer analyzer) { this.doc = doc; this.analyzer = analyzer; this.delTerm = delTerm; } public void run() { //B try { if (analyzer != null) { if (delTerm != null) { ThreadedIndexWriter.super.updateDocument(delTerm, doc, analyzer); } else ThreadedIndexWriter.super.addDocument(doc, analyzer); } else { if (delTerm != null) { ThreadedIndexWriter.super.updateDocument(delTerm, doc); } else ThreadedIndexWriter.super.addDocument(doc); } } catch (IOException ioe) { // TODO: you must log & escalate this error ioe.printStackTrace(System.err); } } } public ThreadedIndexWriter(Directory idx, IndexWriterConfig iConfig) throws CorruptIndexException, IOException { super(idx,iConfig); int numThreads=6; int maxQueueSize=numThreads*4; threadPool = new ThreadPoolExecutor( //C numThreads, numThreads, Long.MAX_VALUE, TimeUnit.NANOSECONDS, new ArrayBlockingQueue<Runnable>(maxQueueSize, false), new ThreadPoolExecutor.CallerRunsPolicy()); } public void addDocument(Document doc) { //D threadPool.execute(new Job(doc, null, null)); //D } //D //D public void addDocument(Document doc, Analyzer a) { //D threadPool.execute(new Job(doc, null, a)); //D } //D //D public void updateDocument(Term term, Document doc) { //D threadPool.execute(new Job(doc, term, null)); //D } //D //D public void updateDocument(Term term, Document doc, Analyzer a) { //D threadPool.execute(new Job(doc, term, a)); //D } //D public void close() throws CorruptIndexException, IOException { finish(); super.close(); } public void close(boolean doWait) throws CorruptIndexException, IOException { finish(); super.close(doWait); } public void rollback() throws CorruptIndexException, IOException { finish(); super.rollback(); } private void finish() { //E threadPool.shutdown(); while(true) { try { if (threadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS)) break; } catch (InterruptedException ie) { Thread.currentThread().interrupt(); } } } }
--------------------------------------------------------------------- To unsubscribe, e-mail: java-user-unsubscr...@lucene.apache.org For additional commands, e-mail: java-user-h...@lucene.apache.org