NullPointerException issue with single IndexWriter instance shared among
multiple Thread
----------------------------------------------------------------------------------------
Key: LUCENE-1399
URL: https://issues.apache.org/jira/browse/LUCENE-1399
Project: Lucene - Java
Issue Type: Bug
Components: Index
Affects Versions: 2.3.2
Environment: Windows XP, JDK 1.6
Reporter: Rudi Quark
NullPointerException is thrown while indexing within multiple threads but using
one single IndexWriter instance. According to "Lucene in Action" this should
do: "...a single instance of either class can be shared among multiple threads,
and all calls to its index-modifying methods will be properly synchronized so
that index modifications are executed one after the other."
Following my exception trace:
java.lang.NullPointerException
at
org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:475)
at
org.apache.lucene.index.IndexWriter.commitTransaction(IndexWriter.java:1986)
at org.apache.lucene.index.IndexWriter.addIndexes(IndexWriter.java:2219)
at
com.shi.marketingspy.test.SHILuceneWritingFromThreadsTest$testthread.run(SHILuceneWritingFromThreadsTest.java:48)
at
java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:885)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:619)
Following my test program:
package com.shi.marketingspy.test;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.IndexWriter;
public class SHILuceneWritingFromThreadsTest
{
public static void log(String s, com.shi.jipmisc misc)
{
misc.__Log(s);
}
public static class testthread implements Runnable
{
public org.apache.lucene.index.IndexWriter indexwriter;
public int iThread;
public boolean bWithOptimize;
public com.shi.jipmisc misc;
public testthread(org.apache.lucene.index.IndexWriter
indexwriter, int iThread, boolean bWithOptimize, com.shi.jipmisc misc)
{
this.indexwriter = indexwriter;
this.iThread = iThread;
this.bWithOptimize = bWithOptimize;
this.misc = misc;
}
public void run()
{
try
{
Analyzer anal = new
com.shi.lucene.index.SHIAnalyser(null,null);
org.apache.lucene.store.Directory tmpidxdirectory
= new org.apache.lucene.store.RAMDirectory();
org.apache.lucene.index.IndexWriter tempindexwriter =
new IndexWriter(tmpidxdirectory, anal, true);
Document doc = new Document();
for(int ifld=0; ifld < 9; ifld++)
{
doc.add(new Field("field_"+ifld, "iThread:
"+iThread+" field: "+ifld+" "+System.currentTimeMillis()+"
"+misc.getDateString()+" "+misc.getDateString2()+"
"+com.shi.jipmisc.getTimeStamp()+" "+misc.getClass()+" "+misc.getDebug(),
Store.COMPRESS, Field.Index.UN_TOKENIZED));
}
tempindexwriter.addDocument(doc);
tempindexwriter.optimize();
org.apache.lucene.store.Directory[] idxs = new
org.apache.lucene.store.Directory[1];
idxs[0] = tmpidxdirectory;
//todo: del-call...
this.indexwriter.addIndexes(idxs);
if (this.bWithOptimize)
this.indexwriter.optimize();
}
catch(Throwable e)
{
log("testthread:run\tERR:2(bWithOpti:"+bWithOptimize+")(iThread:"+iThread+"):"+e.getMessage()+"\n\t"+com.shi.jipmisc.stack2string(e),
misc);
}
}
}
public SHILuceneWritingFromThreadsTest()
{
}
public static void main(String[] args)
{
testluceneconcurrency(false);
testluceneconcurrency(true );
}
public static java.util.concurrent.ExecutorService threadpool_optimized
= java.util.concurrent.Executors.newFixedThreadPool(9999999);
public static java.util.concurrent.ExecutorService
threadpool_not_optimized =
java.util.concurrent.Executors.newFixedThreadPool(9999999);
public static void testluceneconcurrency(boolean bWithOptimize)
{
com.shi.jipmisc.createDir("temp", "c:");
com.shi.jipmisc.createDir("testluceneconcurrency", "c:/temp");
com.shi.jipmisc.createDir(""+bWithOptimize,
"c:/temp/testluceneconcurrency");
com.shi.jipmisc misc = new com.shi.jipmisc();
misc.setMyPath("C:/temp/testluceneconcurrency");
misc.setDebug(true);
misc.emptyDir("c:/temp/testluceneconcurrency/"+bWithOptimize);
Analyzer anal = new com.shi.lucene.index.SHIAnalyser(null,null);
org.apache.lucene.index.IndexWriter indexwriter = null;
try
{
indexwriter = new
IndexWriter("c:/temp/testluceneconcurrency/"+bWithOptimize, anal, true);
}
catch(Throwable e)
{
log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tERR:0("+bWithOptimize+"):"+e.getMessage(),
misc);
return;
}
log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tINF:step0("+bWithOptimize+")(BEGIN)-------------------------------------",
misc);
for(int idoc=0; idoc < 9999; idoc++)
{
Document doc = new Document();
for(int ifld=0; ifld < 9; ifld++)
{
doc.add(new Field("field_"+ifld, "doc: "+idoc+"
field: "+ifld+" "+System.currentTimeMillis()+" "+misc.getDateString()+"
"+misc.getDateString2()+" "+com.shi.jipmisc.getTimeStamp()+"
"+misc.getClass()+" "+misc.getDebug(), Store.COMPRESS,
Field.Index.UN_TOKENIZED));
}
try
{
indexwriter.addDocument(doc);
}
catch(Throwable e)
{
log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tERR:1("+bWithOptimize+"):"+e.getMessage(),
misc);
return;
}
}
log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tINF:step1("+bWithOptimize+")",
misc);
try
{
indexwriter.optimize();
}
catch(Throwable e)
{
log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tERR:2("+bWithOptimize+"):"+e.getMessage(),
misc);
return;
}
log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tINF:step2("+bWithOptimize+")",
misc);
java.util.concurrent.ExecutorService threadpool = bWithOptimize
? threadpool_optimized : threadpool_not_optimized;
for(int it=0; it < 999; it++)
{
threadpool.execute(new testthread(indexwriter, it,
bWithOptimize, misc));
}
log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tINF:step3("+bWithOptimize+")",
misc);
threadpool.shutdown(); //=execute threads and wait til all
threads finish
com.shi.jipmisc.wait(999, misc);
log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tINF:step4("+bWithOptimize+")",
misc);
try
{
//indexwriter.optimize();
//indexwriter.close();
}
catch(Throwable e)
{
log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tERR:9("+bWithOptimize+"):"+e.getMessage(),
misc);
return;
}
log("SHILuceneWritingFromThreadsTest:testluceneconcurrency\tINF:step5("+bWithOptimize+")(END)-----------------------------",
misc);
}
}
--
This message is automatically generated by JIRA.
-
You can reply to this email to add a comment to the issue online.
---------------------------------------------------------------------
To unsubscribe, e-mail: [EMAIL PROTECTED]
For additional commands, e-mail: [EMAIL PROTECTED]
