It's could be due to CMS so I'll create a config with SMS, run 10 times and see if it fails.
On Wed, Aug 19, 2009 at 10:59 AM, Koji Sekiguchi<k...@r.email.ne.jp> wrote: > OK, now the problem is gone. > > Koji > > Noble Paul നോബിള് नोब्ळ् wrote: >> >> I'm looking into it. >> >> I guess I goofed up somewhere >> >> 2009/8/19 Rafał Kuć <ra...@alud.com.pl>: >> >>> >>> Hello! >>> >>> Same here, assertion fails on our hudson. >>> >>> >>> >>> >>>> >>>> On Wed, Aug 19, 2009 at 10:07 PM, Koji >>>> Sekiguchi<k...@r.email.ne.jp> wrote: >>>> >>>>> >>>>> Noble, >>>>> >>>>> testExpungeDeletes fails: >>>>> >>>> >>>> oh . It passed twice in my box.is any one else noticing this failure? >>>> >>>>> >>>>> <failure message="null" type="junit.framework.AssertionFailedError"> >>>>> junit.framework.AssertionFailedError: null >>>>> at >>>>> >>>>> org.apache.solr.update.DirectUpdateHandlerTest.testExpungeDeletes(DirectUpdateHandlerTest.java:303) >>>>> </failure> >>>>> >>>>> Koji >>>>> >>>>> no...@apache.org wrote: >>>>> >>>>>> >>>>>> Author: noble >>>>>> Date: Wed Aug 19 12:21:22 2009 >>>>>> New Revision: 805774 >>>>>> >>>>>> URL: http://svn.apache.org/viewvc?rev=805774&view=rev >>>>>> Log: >>>>>> SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 >>>>>> >>>>>> Modified: >>>>>> lucene/solr/trunk/CHANGES.txt >>>>>> >>>>>> >>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >>>>>> >>>>>> >>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >>>>>> >>>>>> >>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >>>>>> >>>>>> >>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >>>>>> >>>>>> >>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >>>>>> >>>>>> >>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >>>>>> >>>>>> >>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >>>>>> >>>>>> Modified: lucene/solr/trunk/CHANGES.txt >>>>>> URL: >>>>>> >>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=805774&r1=805773&r2=805774&view=diff >>>>>> >>>>>> >>>>>> ============================================================================== >>>>>> --- lucene/solr/trunk/CHANGES.txt (original) >>>>>> +++ lucene/solr/trunk/CHANGES.txt Wed Aug 19 12:21:22 2009 >>>>>> @@ -267,7 +267,9 @@ >>>>>> HTMLStripStandardTokenizerFactory deprecated. To strip HTML tags, >>>>>> HTMLStripCharFilter can be used >>>>>> with an arbitrary Tokenizer. (koji) >>>>>> -68. SOLR-1367: Added callback mechanism for converting DocList to >>>>>> SolrDocumentList in SolrPluginUtils (gsingers) +68. SOLR-1367: Added >>>>>> callback mechanism for converting DocList to SolrDocumentList in >>>>>> SolrPluginUtils (gsingers) >>>>>> + >>>>>> +68. SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 (noble) >>>>>> Optimizations >>>>>> >>>>>> Modified: >>>>>> >>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >>>>>> URL: >>>>>> >>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java?rev=805774&r1=805773&r2=805774&view=diff >>>>>> >>>>>> >>>>>> ============================================================================== >>>>>> --- >>>>>> >>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >>>>>> (original) >>>>>> +++ >>>>>> >>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >>>>>> Wed Aug 19 12:21:22 2009 >>>>>> @@ -49,4 +49,6 @@ >>>>>> * If optimizing, set the maximum number of segments left in the >>>>>> index >>>>>> after optimization. 1 is the default (and is equivalent to calling >>>>>> IndexWriter.optimize() in Lucene). >>>>>> */ >>>>>> public static final String MAX_OPTIMIZE_SEGMENTS = "maxSegments"; >>>>>> + >>>>>> + public static final String EXPUNGE_DELETES = "expungeDeletes"; >>>>>> } >>>>>> >>>>>> Modified: >>>>>> >>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >>>>>> URL: >>>>>> >>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java?rev=805774&r1=805773&r2=805774&view=diff >>>>>> >>>>>> >>>>>> ============================================================================== >>>>>> --- >>>>>> >>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >>>>>> (original) >>>>>> +++ >>>>>> >>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >>>>>> Wed Aug 19 12:21:22 2009 >>>>>> @@ -68,6 +68,7 @@ >>>>>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize ); >>>>>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH, >>>>>> cmd.waitFlush ); >>>>>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, >>>>>> cmd.waitSearcher ); >>>>>> + cmd.expungeDeletes = params.getBool( >>>>>> UpdateParams.EXPUNGE_DELETES, >>>>>> cmd.expungeDeletes); >>>>>> cmd.maxOptimizeSegments = >>>>>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, >>>>>> cmd.maxOptimizeSegments); >>>>>> req.getCore().getUpdateHandler().commit( cmd ); >>>>>> @@ -101,6 +102,7 @@ >>>>>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize ); >>>>>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH, >>>>>> cmd.waitFlush ); >>>>>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, >>>>>> cmd.waitSearcher ); >>>>>> + cmd.expungeDeletes = params.getBool( >>>>>> UpdateParams.EXPUNGE_DELETES, >>>>>> cmd.expungeDeletes); cmd.maxOptimizeSegments = >>>>>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, >>>>>> cmd.maxOptimizeSegments); >>>>>> processor.processCommit( cmd ); >>>>>> return true; >>>>>> >>>>>> Modified: >>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >>>>>> URL: >>>>>> >>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java?rev=805774&r1=805773&r2=805774&view=diff >>>>>> >>>>>> >>>>>> ============================================================================== >>>>>> --- lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >>>>>> (original) >>>>>> +++ lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >>>>>> Wed >>>>>> Aug 19 12:21:22 2009 >>>>>> @@ -155,6 +155,8 @@ >>>>>> sawWaitSearcher = true; >>>>>> } else if >>>>>> (UpdateParams.MAX_OPTIMIZE_SEGMENTS.equals(attrName)) { >>>>>> cmd.maxOptimizeSegments = Integer.parseInt(attrVal); >>>>>> + } else if >>>>>> (UpdateParams.EXPUNGE_DELETES.equals(attrName)) { >>>>>> + cmd.expungeDeletes = StrUtils.parseBoolean(attrVal); >>>>>> } else { >>>>>> XmlUpdateRequestHandler.log.warn("unexpected attribute >>>>>> commit/@" + attrName); >>>>>> } >>>>>> >>>>>> Modified: >>>>>> >>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >>>>>> URL: >>>>>> >>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java?rev=805774&r1=805773&r2=805774&view=diff >>>>>> >>>>>> >>>>>> ============================================================================== >>>>>> --- >>>>>> >>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >>>>>> (original) >>>>>> +++ >>>>>> >>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >>>>>> Wed Aug 19 12:21:22 2009 >>>>>> @@ -23,6 +23,7 @@ >>>>>> public boolean optimize; >>>>>> public boolean waitFlush; >>>>>> public boolean waitSearcher=true; >>>>>> + public boolean expungeDeletes = false; >>>>>> /** >>>>>> * During optimize, optimize down to <= this many segments. Must be >>>>>> >= >>>>>> 1 >>>>>> @@ -39,6 +40,7 @@ >>>>>> return "commit(optimize="+optimize >>>>>> +",waitFlush="+waitFlush >>>>>> +",waitSearcher="+waitSearcher >>>>>> + +",expungeDeletes="+expungeDeletes >>>>>> +')'; >>>>>> } >>>>>> } >>>>>> >>>>>> Modified: >>>>>> >>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >>>>>> URL: >>>>>> >>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java?rev=805774&r1=805773&r2=805774&view=diff >>>>>> >>>>>> >>>>>> ============================================================================== >>>>>> --- >>>>>> >>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >>>>>> (original) >>>>>> +++ >>>>>> >>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >>>>>> Wed Aug 19 12:21:22 2009 >>>>>> @@ -241,9 +241,10 @@ >>>>>> synchronized (this) { >>>>>> pset.clear(); >>>>>> closeSearcher(); // flush any deletes >>>>>> - if (cmd.optimize) { >>>>>> + if (cmd.optimize || cmd.expungeDeletes) { >>>>>> openWriter(); // writer needs to be open to optimize >>>>>> - writer.optimize(cmd.maxOptimizeSegments); >>>>>> + if(cmd.optimize) writer.optimize(cmd.maxOptimizeSegments); >>>>>> + if(cmd.expungeDeletes) >>>>>> writer.expungeDeletes(cmd.expungeDeletes); >>>>>> } >>>>>> closeWriter(); >>>>>> >>>>>> Modified: >>>>>> >>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >>>>>> URL: >>>>>> >>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=805774&r1=805773&r2=805774&view=diff >>>>>> >>>>>> >>>>>> ============================================================================== >>>>>> --- >>>>>> >>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >>>>>> (original) >>>>>> +++ >>>>>> >>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >>>>>> Wed Aug 19 12:21:22 2009 >>>>>> @@ -130,6 +130,7 @@ >>>>>> AtomicLong deleteByIdCommandsCumulative= new AtomicLong(); >>>>>> AtomicLong deleteByQueryCommands= new AtomicLong(); >>>>>> AtomicLong deleteByQueryCommandsCumulative= new AtomicLong(); >>>>>> + AtomicLong expungeDeleteCommands = new AtomicLong(); >>>>>> AtomicLong mergeIndexesCommands = new AtomicLong(); >>>>>> AtomicLong commitCommands= new AtomicLong(); >>>>>> AtomicLong optimizeCommands= new AtomicLong(); >>>>>> @@ -382,6 +383,8 @@ >>>>>> if (cmd.optimize) { >>>>>> optimizeCommands.incrementAndGet(); >>>>>> + } else if (cmd.expungeDeletes) { >>>>>> + expungeDeleteCommands.incrementAndGet(); >>>>>> } else { >>>>>> commitCommands.incrementAndGet(); >>>>>> } >>>>>> @@ -402,6 +405,10 @@ >>>>>> } >>>>>> closeWriter(); >>>>>> + if (!cmd.optimize && cmd.expungeDeletes) { >>>>>> + openWriter(); >>>>>> + writer.expungeDeletes(); >>>>>> + } >>>>>> callPostCommitCallbacks(); >>>>>> if (cmd.optimize) { >>>>>> >>>>>> Modified: >>>>>> >>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >>>>>> URL: >>>>>> >>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java?rev=805774&r1=805773&r2=805774&view=diff >>>>>> >>>>>> >>>>>> ============================================================================== >>>>>> --- >>>>>> >>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >>>>>> (original) >>>>>> +++ >>>>>> >>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >>>>>> Wed Aug 19 12:21:22 2009 >>>>>> @@ -153,6 +153,12 @@ >>>>>> return this; >>>>>> } >>>>>> + public UpdateRequest setAction(ACTION action, boolean waitFlush, >>>>>> boolean waitSearcher, int maxSegments , boolean expungeDeletes) { >>>>>> + setAction(action, waitFlush, waitSearcher,maxSegments) ; >>>>>> + params.set(UpdateParams.EXPUNGE_DELETES,""+expungeDeletes); >>>>>> + return this; >>>>>> + } >>>>>> + >>>>>> /** >>>>>> * @since Solr 1.4 >>>>>> */ >>>>>> >>>>>> Modified: >>>>>> >>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >>>>>> URL: >>>>>> >>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java?rev=805774&r1=805773&r2=805774&view=diff >>>>>> >>>>>> >>>>>> ============================================================================== >>>>>> --- >>>>>> >>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >>>>>> (original) >>>>>> +++ >>>>>> >>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >>>>>> Wed Aug 19 12:21:22 2009 >>>>>> @@ -17,20 +17,32 @@ >>>>>> package org.apache.solr.update; >>>>>> +import java.io.IOException; >>>>>> +import java.util.ArrayList; >>>>>> import java.util.HashMap; >>>>>> +import java.util.HashSet; >>>>>> +import java.util.List; >>>>>> import java.util.Map; >>>>>> +import java.util.Set; >>>>>> import org.apache.lucene.document.Document; >>>>>> import org.apache.lucene.document.Field; >>>>>> import org.apache.lucene.document.Field.Index; >>>>>> import org.apache.lucene.document.Field.Store; >>>>>> +import org.apache.lucene.index.IndexReader; >>>>>> +import org.apache.lucene.index.SegmentReader; >>>>>> +import org.apache.lucene.index.Term; >>>>>> +import org.apache.lucene.index.TermEnum; >>>>>> import org.apache.solr.common.SolrException; >>>>>> import org.apache.solr.common.params.CommonParams; >>>>>> import org.apache.solr.common.params.MapSolrParams; >>>>>> import org.apache.solr.core.SolrCore; >>>>>> import org.apache.solr.request.LocalSolrQueryRequest; >>>>>> import org.apache.solr.request.SolrQueryRequest; >>>>>> +import org.apache.solr.search.SolrIndexReader; >>>>>> +import org.apache.solr.search.SolrIndexSearcher; >>>>>> import org.apache.solr.util.AbstractSolrTestCase; >>>>>> +import org.apache.solr.util.RefCounted; >>>>>> /** >>>>>> * @@ -247,6 +259,90 @@ >>>>>> ); >>>>>> } >>>>>> + public void testExpungeDeletes() throws Exception { >>>>>> + for (int x = 0; x < 3000; x++) { >>>>>> + addSimpleDoc(x + ""); >>>>>> + } >>>>>> + SolrCore core = h.getCore(); >>>>>> + UpdateHandler updater = core.getUpdateHandler(); >>>>>> + CommitUpdateCommand cmtCmd = new CommitUpdateCommand(false); >>>>>> + cmtCmd.waitSearcher = true; >>>>>> + updater.commit(cmtCmd); >>>>>> + >>>>>> + List<String> todelete = new ArrayList<String>(); >>>>>> + >>>>>> + Set<String> segsdel = new HashSet<String>(); >>>>>> + >>>>>> + SegmentReader[] sirs = getSegmentReaders(core); >>>>>> + assertTrue(sirs.length > 6); >>>>>> + todelete.add(getNthIDTerm(2, sirs[0])); >>>>>> + segsdel.add(sirs[0].getSegmentName()); >>>>>> + + todelete.add(getNthIDTerm(7, sirs[2])); >>>>>> + segsdel.add(sirs[2].getSegmentName()); >>>>>> + + todelete.add(getNthIDTerm(4, sirs[5])); >>>>>> + segsdel.add(sirs[5].getSegmentName()); >>>>>> + + for (String id : todelete) { >>>>>> + deleteSimpleDoc(id); >>>>>> + } >>>>>> + // commit the deletes >>>>>> + cmtCmd = new CommitUpdateCommand(false); >>>>>> + cmtCmd.waitSearcher = true; >>>>>> + updater.commit(cmtCmd); >>>>>> + + // expunge deletes >>>>>> + cmtCmd = new CommitUpdateCommand(false); >>>>>> + cmtCmd.waitSearcher = true; >>>>>> + cmtCmd.expungeDeletes = true; >>>>>> + updater.commit(cmtCmd); >>>>>> + + // we'll have fewer segments >>>>>> + SegmentReader[] sirs2 = getSegmentReaders(core); >>>>>> + assertTrue(sirs.length > sirs2.length); >>>>>> + // check the actual segment names >>>>>> + for (SegmentReader sr : sirs2) { >>>>>> + assertTrue(!segsdel.contains(sr.getSegmentName())); >>>>>> + } >>>>>> + } >>>>>> + >>>>>> + SegmentReader[] getSegmentReaders(SolrCore core) throws IOException >>>>>> { >>>>>> + RefCounted<SolrIndexSearcher> ref = core.getSearcher(true, true, >>>>>> null); >>>>>> + SolrIndexSearcher is = ref.get(); >>>>>> + SegmentReader[] segmentReaders = null; >>>>>> + try { >>>>>> + SolrIndexReader reader = is.getReader(); >>>>>> + IndexReader[] subreaders = reader.getSequentialSubReaders(); >>>>>> + segmentReaders = new SegmentReader[subreaders.length]; >>>>>> + for (int x = 0; x < subreaders.length; x++) { >>>>>> + assert subreaders[x] instanceof SolrIndexReader; >>>>>> + SolrIndexReader sir = (SolrIndexReader) subreaders[x]; >>>>>> + SegmentReader sr = (SegmentReader) sir.getWrappedReader(); >>>>>> + segmentReaders[x] = sr; >>>>>> + } >>>>>> + } finally { >>>>>> + ref.decref(); >>>>>> + } >>>>>> + return segmentReaders; >>>>>> + } >>>>>> + >>>>>> + private String getNthIDTerm(int n, IndexReader r) throws >>>>>> IOException { >>>>>> + TermEnum te = r.terms(new Term("id", "")); >>>>>> + try { >>>>>> + int x = 0; >>>>>> + do { >>>>>> + if (x >= n) { >>>>>> + return te.term().text(); >>>>>> + } >>>>>> + x++; >>>>>> + } while (te.next()); >>>>>> + } finally { >>>>>> + te.close(); >>>>>> + } >>>>>> + return null; >>>>>> + } >>>>>> + private void addSimpleDoc(String id) throws Exception { >>>>>> SolrCore core = h.getCore(); >>>>>> >>>>>> >>>>>> >>>>>> >>>>>> >>>>> >>>>> >>> >>> >>> >>> >>> -- >>> Regards, >>> Rafał Kuć >>> >>> >>> >> >> >> >> > >
