Hey, you should compare with the ThreadedIndexWriter too :). I'll attach the
source from Lucene in action SE manual and you can just replace the new
IntexWriter(... with new ThreadedIndexWriter(...
See if those results make a difference. Also I presume you don't have a
single core cpu
2011/10/11 Marc Sturlese <[email protected]>
> I'm doing some performance test doing bulk indexing with lucene 4.0
>
package org.ubervu.search.CustomLucene;
import java.io.IOException;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
public class ThreadedIndexWriter extends IndexWriter {
private ExecutorService threadPool;
private class Job implements Runnable { //A
Document doc;
Analyzer analyzer;
Term delTerm;
public Job(Document doc, Term delTerm, Analyzer analyzer) {
this.doc = doc;
this.analyzer = analyzer;
this.delTerm = delTerm;
}
public void run() { //B
try {
if (analyzer != null) {
if (delTerm != null) {
ThreadedIndexWriter.super.updateDocument(delTerm, doc,
analyzer);
} else
ThreadedIndexWriter.super.addDocument(doc, analyzer);
} else {
if (delTerm != null) {
ThreadedIndexWriter.super.updateDocument(delTerm, doc);
} else
ThreadedIndexWriter.super.addDocument(doc);
}
} catch (IOException ioe) {
// TODO: you must log & escalate this error
ioe.printStackTrace(System.err);
}
}
}
public ThreadedIndexWriter(Directory idx, IndexWriterConfig iConfig)
throws CorruptIndexException, IOException {
super(idx,iConfig);
int numThreads=6; int maxQueueSize=numThreads*4;
threadPool = new ThreadPoolExecutor( //C
numThreads, numThreads,
Long.MAX_VALUE, TimeUnit.NANOSECONDS,
new ArrayBlockingQueue<Runnable>(maxQueueSize, false),
new ThreadPoolExecutor.CallerRunsPolicy());
}
public void addDocument(Document doc) { //D
threadPool.execute(new Job(doc, null, null)); //D
} //D
//D
public void addDocument(Document doc, Analyzer a) { //D
threadPool.execute(new Job(doc, null, a)); //D
} //D
//D
public void updateDocument(Term term, Document doc) { //D
threadPool.execute(new Job(doc, term, null)); //D
} //D
//D
public void updateDocument(Term term, Document doc, Analyzer a) { //D
threadPool.execute(new Job(doc, term, a)); //D
} //D
public void close() throws CorruptIndexException, IOException {
finish();
super.close();
}
public void close(boolean doWait) throws CorruptIndexException,
IOException {
finish();
super.close(doWait);
}
public void rollback() throws CorruptIndexException, IOException {
finish();
super.rollback();
}
private void finish() { //E
threadPool.shutdown();
while(true) {
try {
if (threadPool.awaitTermination(Long.MAX_VALUE,
TimeUnit.SECONDS))
break;
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
}
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
