Hey, you should compare with the ThreadedIndexWriter too :). I'll attach the
source from Lucene in action SE manual and you can just replace the new
IntexWriter(... with new ThreadedIndexWriter(...

See if those results make a difference. Also I presume you don't have a
single core cpu

2011/10/11 Marc Sturlese <marc.sturl...@gmail.com>

> I'm doing some performance test doing bulk indexing with lucene 4.0
>
package org.ubervu.search.CustomLucene;

import java.io.IOException;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;

public class ThreadedIndexWriter extends IndexWriter { 
         
          private ExecutorService threadPool; 
         
          private class Job implements Runnable {                       //A 
            Document doc; 
            Analyzer analyzer; 
            Term delTerm; 
            public Job(Document doc, Term delTerm, Analyzer analyzer) { 
              this.doc = doc; 
              this.analyzer = analyzer; 
              this.delTerm = delTerm; 
            } 
            public void run() {                                         //B 
              try { 
                if (analyzer != null) { 
                  if (delTerm != null) { 
                    ThreadedIndexWriter.super.updateDocument(delTerm, doc, 
analyzer); 
                  } else 
                    ThreadedIndexWriter.super.addDocument(doc, analyzer); 
                } else { 
                  if (delTerm != null) { 
                    ThreadedIndexWriter.super.updateDocument(delTerm, doc); 
                  } else 
                      ThreadedIndexWriter.super.addDocument(doc); 
                } 
              } catch (IOException ioe) { 
                // TODO: you must log & escalate this error 
                ioe.printStackTrace(System.err); 
              }
            }
          }
          public ThreadedIndexWriter(Directory idx, IndexWriterConfig iConfig) 
               throws CorruptIndexException, IOException {
                  super(idx,iConfig); 
                  int numThreads=6; int maxQueueSize=numThreads*4;
            threadPool = new ThreadPoolExecutor(                        //C 
                  numThreads, numThreads, 
                  Long.MAX_VALUE, TimeUnit.NANOSECONDS, 
                  new ArrayBlockingQueue<Runnable>(maxQueueSize, false), 
                  new ThreadPoolExecutor.CallerRunsPolicy()); 
          }
         
          public void addDocument(Document doc) {                       //D 
            threadPool.execute(new Job(doc, null, null));               //D 
          }                                                             //D 
                                                                        //D 
          public void addDocument(Document doc, Analyzer a) {           //D 
            threadPool.execute(new Job(doc, null, a));                  //D 
          }                                                             //D 
                                                                        //D 
          public void updateDocument(Term term, Document doc) {         //D 
            threadPool.execute(new Job(doc, term, null));               //D 
          }                                                             //D 
                                                                        //D  
          public void updateDocument(Term term, Document doc, Analyzer a) { //D 
            threadPool.execute(new Job(doc, term, a));                  //D 
          }                                                             //D 
         
          public void close() throws CorruptIndexException, IOException { 
            finish(); 
            super.close(); 
          } 
         
          public void close(boolean doWait) throws CorruptIndexException, 
IOException { 
            finish(); 
            super.close(doWait); 
          } 
         
          public void rollback() throws CorruptIndexException, IOException { 
            finish(); 
            super.rollback(); 
          } 
         
          private void finish() {                                       //E 
            threadPool.shutdown(); 
            while(true) { 
              try { 
                if (threadPool.awaitTermination(Long.MAX_VALUE, 
TimeUnit.SECONDS)) 
                  break; 
              } catch (InterruptedException ie) { 
                Thread.currentThread().interrupt(); 
              } 
            } 
          } 
        }
---------------------------------------------------------------------
To unsubscribe, e-mail: java-user-unsubscr...@lucene.apache.org
For additional commands, e-mail: java-user-h...@lucene.apache.org

Reply via email to