Fails consistently with SMS so I'll post a new patch shortly.
On Wed, Aug 19, 2009 at 11:31 AM, Jason Rutherglen<jason.rutherg...@gmail.com> wrote: > It's could be due to CMS so I'll create a config with SMS, run 10 > times and see if it fails. > > On Wed, Aug 19, 2009 at 10:59 AM, Koji Sekiguchi<k...@r.email.ne.jp> wrote: >> OK, now the problem is gone. >> >> Koji >> >> Noble Paul നോബിള് नोब्ळ् wrote: >>> >>> I'm looking into it. >>> >>> I guess I goofed up somewhere >>> >>> 2009/8/19 Rafał Kuć <ra...@alud.com.pl>: >>> >>>> >>>> Hello! >>>> >>>> Same here, assertion fails on our hudson. >>>> >>>> >>>> >>>> >>>>> >>>>> On Wed, Aug 19, 2009 at 10:07 PM, Koji >>>>> Sekiguchi<k...@r.email.ne.jp> wrote: >>>>> >>>>>> >>>>>> Noble, >>>>>> >>>>>> testExpungeDeletes fails: >>>>>> >>>>> >>>>> oh . It passed twice in my box.is any one else noticing this failure? >>>>> >>>>>> >>>>>> <failure message="null" type="junit.framework.AssertionFailedError"> >>>>>> junit.framework.AssertionFailedError: null >>>>>> at >>>>>> >>>>>> org.apache.solr.update.DirectUpdateHandlerTest.testExpungeDeletes(DirectUpdateHandlerTest.java:303) >>>>>> </failure> >>>>>> >>>>>> Koji >>>>>> >>>>>> no...@apache.org wrote: >>>>>> >>>>>>> >>>>>>> Author: noble >>>>>>> Date: Wed Aug 19 12:21:22 2009 >>>>>>> New Revision: 805774 >>>>>>> >>>>>>> URL: http://svn.apache.org/viewvc?rev=805774&view=rev >>>>>>> Log: >>>>>>> SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 >>>>>>> >>>>>>> Modified: >>>>>>> lucene/solr/trunk/CHANGES.txt >>>>>>> >>>>>>> >>>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >>>>>>> >>>>>>> >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >>>>>>> >>>>>>> >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >>>>>>> >>>>>>> >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >>>>>>> >>>>>>> >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >>>>>>> >>>>>>> >>>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >>>>>>> >>>>>>> >>>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >>>>>>> >>>>>>> Modified: lucene/solr/trunk/CHANGES.txt >>>>>>> URL: >>>>>>> >>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=805774&r1=805773&r2=805774&view=diff >>>>>>> >>>>>>> >>>>>>> ============================================================================== >>>>>>> --- lucene/solr/trunk/CHANGES.txt (original) >>>>>>> +++ lucene/solr/trunk/CHANGES.txt Wed Aug 19 12:21:22 2009 >>>>>>> @@ -267,7 +267,9 @@ >>>>>>> HTMLStripStandardTokenizerFactory deprecated. To strip HTML tags, >>>>>>> HTMLStripCharFilter can be used >>>>>>> with an arbitrary Tokenizer. (koji) >>>>>>> -68. SOLR-1367: Added callback mechanism for converting DocList to >>>>>>> SolrDocumentList in SolrPluginUtils (gsingers) +68. SOLR-1367: Added >>>>>>> callback mechanism for converting DocList to SolrDocumentList in >>>>>>> SolrPluginUtils (gsingers) >>>>>>> + >>>>>>> +68. SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 (noble) >>>>>>> Optimizations >>>>>>> >>>>>>> Modified: >>>>>>> >>>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >>>>>>> URL: >>>>>>> >>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java?rev=805774&r1=805773&r2=805774&view=diff >>>>>>> >>>>>>> >>>>>>> ============================================================================== >>>>>>> --- >>>>>>> >>>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >>>>>>> (original) >>>>>>> +++ >>>>>>> >>>>>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >>>>>>> Wed Aug 19 12:21:22 2009 >>>>>>> @@ -49,4 +49,6 @@ >>>>>>> * If optimizing, set the maximum number of segments left in the >>>>>>> index >>>>>>> after optimization. 1 is the default (and is equivalent to calling >>>>>>> IndexWriter.optimize() in Lucene). >>>>>>> */ >>>>>>> public static final String MAX_OPTIMIZE_SEGMENTS = "maxSegments"; >>>>>>> + >>>>>>> + public static final String EXPUNGE_DELETES = "expungeDeletes"; >>>>>>> } >>>>>>> >>>>>>> Modified: >>>>>>> >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >>>>>>> URL: >>>>>>> >>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java?rev=805774&r1=805773&r2=805774&view=diff >>>>>>> >>>>>>> >>>>>>> ============================================================================== >>>>>>> --- >>>>>>> >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >>>>>>> (original) >>>>>>> +++ >>>>>>> >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >>>>>>> Wed Aug 19 12:21:22 2009 >>>>>>> @@ -68,6 +68,7 @@ >>>>>>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize ); >>>>>>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH, >>>>>>> cmd.waitFlush ); >>>>>>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, >>>>>>> cmd.waitSearcher ); >>>>>>> + cmd.expungeDeletes = params.getBool( >>>>>>> UpdateParams.EXPUNGE_DELETES, >>>>>>> cmd.expungeDeletes); >>>>>>> cmd.maxOptimizeSegments = >>>>>>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, >>>>>>> cmd.maxOptimizeSegments); >>>>>>> req.getCore().getUpdateHandler().commit( cmd ); >>>>>>> @@ -101,6 +102,7 @@ >>>>>>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize ); >>>>>>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH, >>>>>>> cmd.waitFlush ); >>>>>>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, >>>>>>> cmd.waitSearcher ); >>>>>>> + cmd.expungeDeletes = params.getBool( >>>>>>> UpdateParams.EXPUNGE_DELETES, >>>>>>> cmd.expungeDeletes); cmd.maxOptimizeSegments = >>>>>>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, >>>>>>> cmd.maxOptimizeSegments); >>>>>>> processor.processCommit( cmd ); >>>>>>> return true; >>>>>>> >>>>>>> Modified: >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >>>>>>> URL: >>>>>>> >>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java?rev=805774&r1=805773&r2=805774&view=diff >>>>>>> >>>>>>> >>>>>>> ============================================================================== >>>>>>> --- lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >>>>>>> (original) >>>>>>> +++ lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >>>>>>> Wed >>>>>>> Aug 19 12:21:22 2009 >>>>>>> @@ -155,6 +155,8 @@ >>>>>>> sawWaitSearcher = true; >>>>>>> } else if >>>>>>> (UpdateParams.MAX_OPTIMIZE_SEGMENTS.equals(attrName)) { >>>>>>> cmd.maxOptimizeSegments = Integer.parseInt(attrVal); >>>>>>> + } else if >>>>>>> (UpdateParams.EXPUNGE_DELETES.equals(attrName)) { >>>>>>> + cmd.expungeDeletes = StrUtils.parseBoolean(attrVal); >>>>>>> } else { >>>>>>> XmlUpdateRequestHandler.log.warn("unexpected attribute >>>>>>> commit/@" + attrName); >>>>>>> } >>>>>>> >>>>>>> Modified: >>>>>>> >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >>>>>>> URL: >>>>>>> >>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java?rev=805774&r1=805773&r2=805774&view=diff >>>>>>> >>>>>>> >>>>>>> ============================================================================== >>>>>>> --- >>>>>>> >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >>>>>>> (original) >>>>>>> +++ >>>>>>> >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >>>>>>> Wed Aug 19 12:21:22 2009 >>>>>>> @@ -23,6 +23,7 @@ >>>>>>> public boolean optimize; >>>>>>> public boolean waitFlush; >>>>>>> public boolean waitSearcher=true; >>>>>>> + public boolean expungeDeletes = false; >>>>>>> /** >>>>>>> * During optimize, optimize down to <= this many segments. Must be >>>>>>> >= >>>>>>> 1 >>>>>>> @@ -39,6 +40,7 @@ >>>>>>> return "commit(optimize="+optimize >>>>>>> +",waitFlush="+waitFlush >>>>>>> +",waitSearcher="+waitSearcher >>>>>>> + +",expungeDeletes="+expungeDeletes >>>>>>> +')'; >>>>>>> } >>>>>>> } >>>>>>> >>>>>>> Modified: >>>>>>> >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >>>>>>> URL: >>>>>>> >>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java?rev=805774&r1=805773&r2=805774&view=diff >>>>>>> >>>>>>> >>>>>>> ============================================================================== >>>>>>> --- >>>>>>> >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >>>>>>> (original) >>>>>>> +++ >>>>>>> >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >>>>>>> Wed Aug 19 12:21:22 2009 >>>>>>> @@ -241,9 +241,10 @@ >>>>>>> synchronized (this) { >>>>>>> pset.clear(); >>>>>>> closeSearcher(); // flush any deletes >>>>>>> - if (cmd.optimize) { >>>>>>> + if (cmd.optimize || cmd.expungeDeletes) { >>>>>>> openWriter(); // writer needs to be open to optimize >>>>>>> - writer.optimize(cmd.maxOptimizeSegments); >>>>>>> + if(cmd.optimize) writer.optimize(cmd.maxOptimizeSegments); >>>>>>> + if(cmd.expungeDeletes) >>>>>>> writer.expungeDeletes(cmd.expungeDeletes); >>>>>>> } >>>>>>> closeWriter(); >>>>>>> >>>>>>> Modified: >>>>>>> >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >>>>>>> URL: >>>>>>> >>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=805774&r1=805773&r2=805774&view=diff >>>>>>> >>>>>>> >>>>>>> ============================================================================== >>>>>>> --- >>>>>>> >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >>>>>>> (original) >>>>>>> +++ >>>>>>> >>>>>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >>>>>>> Wed Aug 19 12:21:22 2009 >>>>>>> @@ -130,6 +130,7 @@ >>>>>>> AtomicLong deleteByIdCommandsCumulative= new AtomicLong(); >>>>>>> AtomicLong deleteByQueryCommands= new AtomicLong(); >>>>>>> AtomicLong deleteByQueryCommandsCumulative= new AtomicLong(); >>>>>>> + AtomicLong expungeDeleteCommands = new AtomicLong(); >>>>>>> AtomicLong mergeIndexesCommands = new AtomicLong(); >>>>>>> AtomicLong commitCommands= new AtomicLong(); >>>>>>> AtomicLong optimizeCommands= new AtomicLong(); >>>>>>> @@ -382,6 +383,8 @@ >>>>>>> if (cmd.optimize) { >>>>>>> optimizeCommands.incrementAndGet(); >>>>>>> + } else if (cmd.expungeDeletes) { >>>>>>> + expungeDeleteCommands.incrementAndGet(); >>>>>>> } else { >>>>>>> commitCommands.incrementAndGet(); >>>>>>> } >>>>>>> @@ -402,6 +405,10 @@ >>>>>>> } >>>>>>> closeWriter(); >>>>>>> + if (!cmd.optimize && cmd.expungeDeletes) { >>>>>>> + openWriter(); >>>>>>> + writer.expungeDeletes(); >>>>>>> + } >>>>>>> callPostCommitCallbacks(); >>>>>>> if (cmd.optimize) { >>>>>>> >>>>>>> Modified: >>>>>>> >>>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >>>>>>> URL: >>>>>>> >>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java?rev=805774&r1=805773&r2=805774&view=diff >>>>>>> >>>>>>> >>>>>>> ============================================================================== >>>>>>> --- >>>>>>> >>>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >>>>>>> (original) >>>>>>> +++ >>>>>>> >>>>>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >>>>>>> Wed Aug 19 12:21:22 2009 >>>>>>> @@ -153,6 +153,12 @@ >>>>>>> return this; >>>>>>> } >>>>>>> + public UpdateRequest setAction(ACTION action, boolean waitFlush, >>>>>>> boolean waitSearcher, int maxSegments , boolean expungeDeletes) { >>>>>>> + setAction(action, waitFlush, waitSearcher,maxSegments) ; >>>>>>> + params.set(UpdateParams.EXPUNGE_DELETES,""+expungeDeletes); >>>>>>> + return this; >>>>>>> + } >>>>>>> + >>>>>>> /** >>>>>>> * @since Solr 1.4 >>>>>>> */ >>>>>>> >>>>>>> Modified: >>>>>>> >>>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >>>>>>> URL: >>>>>>> >>>>>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java?rev=805774&r1=805773&r2=805774&view=diff >>>>>>> >>>>>>> >>>>>>> ============================================================================== >>>>>>> --- >>>>>>> >>>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >>>>>>> (original) >>>>>>> +++ >>>>>>> >>>>>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >>>>>>> Wed Aug 19 12:21:22 2009 >>>>>>> @@ -17,20 +17,32 @@ >>>>>>> package org.apache.solr.update; >>>>>>> +import java.io.IOException; >>>>>>> +import java.util.ArrayList; >>>>>>> import java.util.HashMap; >>>>>>> +import java.util.HashSet; >>>>>>> +import java.util.List; >>>>>>> import java.util.Map; >>>>>>> +import java.util.Set; >>>>>>> import org.apache.lucene.document.Document; >>>>>>> import org.apache.lucene.document.Field; >>>>>>> import org.apache.lucene.document.Field.Index; >>>>>>> import org.apache.lucene.document.Field.Store; >>>>>>> +import org.apache.lucene.index.IndexReader; >>>>>>> +import org.apache.lucene.index.SegmentReader; >>>>>>> +import org.apache.lucene.index.Term; >>>>>>> +import org.apache.lucene.index.TermEnum; >>>>>>> import org.apache.solr.common.SolrException; >>>>>>> import org.apache.solr.common.params.CommonParams; >>>>>>> import org.apache.solr.common.params.MapSolrParams; >>>>>>> import org.apache.solr.core.SolrCore; >>>>>>> import org.apache.solr.request.LocalSolrQueryRequest; >>>>>>> import org.apache.solr.request.SolrQueryRequest; >>>>>>> +import org.apache.solr.search.SolrIndexReader; >>>>>>> +import org.apache.solr.search.SolrIndexSearcher; >>>>>>> import org.apache.solr.util.AbstractSolrTestCase; >>>>>>> +import org.apache.solr.util.RefCounted; >>>>>>> /** >>>>>>> * @@ -247,6 +259,90 @@ >>>>>>> ); >>>>>>> } >>>>>>> + public void testExpungeDeletes() throws Exception { >>>>>>> + for (int x = 0; x < 3000; x++) { >>>>>>> + addSimpleDoc(x + ""); >>>>>>> + } >>>>>>> + SolrCore core = h.getCore(); >>>>>>> + UpdateHandler updater = core.getUpdateHandler(); >>>>>>> + CommitUpdateCommand cmtCmd = new CommitUpdateCommand(false); >>>>>>> + cmtCmd.waitSearcher = true; >>>>>>> + updater.commit(cmtCmd); >>>>>>> + >>>>>>> + List<String> todelete = new ArrayList<String>(); >>>>>>> + >>>>>>> + Set<String> segsdel = new HashSet<String>(); >>>>>>> + >>>>>>> + SegmentReader[] sirs = getSegmentReaders(core); >>>>>>> + assertTrue(sirs.length > 6); >>>>>>> + todelete.add(getNthIDTerm(2, sirs[0])); >>>>>>> + segsdel.add(sirs[0].getSegmentName()); >>>>>>> + + todelete.add(getNthIDTerm(7, sirs[2])); >>>>>>> + segsdel.add(sirs[2].getSegmentName()); >>>>>>> + + todelete.add(getNthIDTerm(4, sirs[5])); >>>>>>> + segsdel.add(sirs[5].getSegmentName()); >>>>>>> + + for (String id : todelete) { >>>>>>> + deleteSimpleDoc(id); >>>>>>> + } >>>>>>> + // commit the deletes >>>>>>> + cmtCmd = new CommitUpdateCommand(false); >>>>>>> + cmtCmd.waitSearcher = true; >>>>>>> + updater.commit(cmtCmd); >>>>>>> + + // expunge deletes >>>>>>> + cmtCmd = new CommitUpdateCommand(false); >>>>>>> + cmtCmd.waitSearcher = true; >>>>>>> + cmtCmd.expungeDeletes = true; >>>>>>> + updater.commit(cmtCmd); >>>>>>> + + // we'll have fewer segments >>>>>>> + SegmentReader[] sirs2 = getSegmentReaders(core); >>>>>>> + assertTrue(sirs.length > sirs2.length); >>>>>>> + // check the actual segment names >>>>>>> + for (SegmentReader sr : sirs2) { >>>>>>> + assertTrue(!segsdel.contains(sr.getSegmentName())); >>>>>>> + } >>>>>>> + } >>>>>>> + >>>>>>> + SegmentReader[] getSegmentReaders(SolrCore core) throws IOException >>>>>>> { >>>>>>> + RefCounted<SolrIndexSearcher> ref = core.getSearcher(true, true, >>>>>>> null); >>>>>>> + SolrIndexSearcher is = ref.get(); >>>>>>> + SegmentReader[] segmentReaders = null; >>>>>>> + try { >>>>>>> + SolrIndexReader reader = is.getReader(); >>>>>>> + IndexReader[] subreaders = reader.getSequentialSubReaders(); >>>>>>> + segmentReaders = new SegmentReader[subreaders.length]; >>>>>>> + for (int x = 0; x < subreaders.length; x++) { >>>>>>> + assert subreaders[x] instanceof SolrIndexReader; >>>>>>> + SolrIndexReader sir = (SolrIndexReader) subreaders[x]; >>>>>>> + SegmentReader sr = (SegmentReader) sir.getWrappedReader(); >>>>>>> + segmentReaders[x] = sr; >>>>>>> + } >>>>>>> + } finally { >>>>>>> + ref.decref(); >>>>>>> + } >>>>>>> + return segmentReaders; >>>>>>> + } >>>>>>> + >>>>>>> + private String getNthIDTerm(int n, IndexReader r) throws >>>>>>> IOException { >>>>>>> + TermEnum te = r.terms(new Term("id", "")); >>>>>>> + try { >>>>>>> + int x = 0; >>>>>>> + do { >>>>>>> + if (x >= n) { >>>>>>> + return te.term().text(); >>>>>>> + } >>>>>>> + x++; >>>>>>> + } while (te.next()); >>>>>>> + } finally { >>>>>>> + te.close(); >>>>>>> + } >>>>>>> + return null; >>>>>>> + } >>>>>>> + private void addSimpleDoc(String id) throws Exception { >>>>>>> SolrCore core = h.getCore(); >>>>>>> >>>>>>> >>>>>>> >>>>>>> >>>>>>> >>>>>> >>>>>> >>>> >>>> >>>> >>>> >>>> -- >>>> Regards, >>>> Rafał Kuć >>>> >>>> >>>> >>> >>> >>> >>> >> >> >
