I'm looking into it. I guess I goofed up somewhere
2009/8/19 Rafał Kuć <ra...@alud.com.pl>: > Hello! > > Same here, assertion fails on our hudson. > > > >> On Wed, Aug 19, 2009 at 10:07 PM, Koji >> Sekiguchi<k...@r.email.ne.jp> wrote: >>> Noble, >>> >>> testExpungeDeletes fails: >> oh . It passed twice in my box.is any one else noticing this failure? >>> >>> <failure message="null" type="junit.framework.AssertionFailedError"> >>> junit.framework.AssertionFailedError: null >>> at >>> org.apache.solr.update.DirectUpdateHandlerTest.testExpungeDeletes(DirectUpdateHandlerTest.java:303) >>> </failure> >>> >>> Koji >>> >>> no...@apache.org wrote: >>>> >>>> Author: noble >>>> Date: Wed Aug 19 12:21:22 2009 >>>> New Revision: 805774 >>>> >>>> URL: http://svn.apache.org/viewvc?rev=805774&view=rev >>>> Log: >>>> SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 >>>> >>>> Modified: >>>> lucene/solr/trunk/CHANGES.txt >>>> >>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >>>> >>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >>>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >>>> >>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >>>> >>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >>>> >>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >>>> >>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >>>> >>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >>>> >>>> Modified: lucene/solr/trunk/CHANGES.txt >>>> URL: >>>> http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=805774&r1=805773&r2=805774&view=diff >>>> >>>> ============================================================================== >>>> --- lucene/solr/trunk/CHANGES.txt (original) >>>> +++ lucene/solr/trunk/CHANGES.txt Wed Aug 19 12:21:22 2009 >>>> @@ -267,7 +267,9 @@ >>>> HTMLStripStandardTokenizerFactory deprecated. To strip HTML tags, >>>> HTMLStripCharFilter can be used >>>> with an arbitrary Tokenizer. (koji) >>>> -68. SOLR-1367: Added callback mechanism for converting DocList to >>>> SolrDocumentList in SolrPluginUtils (gsingers) +68. SOLR-1367: Added >>>> callback mechanism for converting DocList to SolrDocumentList in >>>> SolrPluginUtils (gsingers) >>>> + >>>> +68. SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 (noble) >>>> Optimizations >>>> >>>> Modified: >>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >>>> URL: >>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java?rev=805774&r1=805773&r2=805774&view=diff >>>> >>>> ============================================================================== >>>> --- >>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >>>> (original) >>>> +++ >>>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >>>> Wed Aug 19 12:21:22 2009 >>>> @@ -49,4 +49,6 @@ >>>> * If optimizing, set the maximum number of segments left in the index >>>> after optimization. 1 is the default (and is equivalent to calling >>>> IndexWriter.optimize() in Lucene). >>>> */ >>>> public static final String MAX_OPTIMIZE_SEGMENTS = "maxSegments"; >>>> + >>>> + public static final String EXPUNGE_DELETES = "expungeDeletes"; >>>> } >>>> >>>> Modified: >>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >>>> URL: >>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java?rev=805774&r1=805773&r2=805774&view=diff >>>> >>>> ============================================================================== >>>> --- >>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >>>> (original) >>>> +++ >>>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >>>> Wed Aug 19 12:21:22 2009 >>>> @@ -68,6 +68,7 @@ >>>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize ); >>>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH, >>>> cmd.waitFlush ); >>>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, >>>> cmd.waitSearcher ); >>>> + cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES, >>>> cmd.expungeDeletes); >>>> cmd.maxOptimizeSegments = >>>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments); >>>> req.getCore().getUpdateHandler().commit( cmd ); >>>> @@ -101,6 +102,7 @@ >>>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize ); >>>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH, >>>> cmd.waitFlush ); >>>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, >>>> cmd.waitSearcher ); >>>> + cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES, >>>> cmd.expungeDeletes); cmd.maxOptimizeSegments = >>>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments); >>>> processor.processCommit( cmd ); >>>> return true; >>>> >>>> Modified: >>>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >>>> URL: >>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java?rev=805774&r1=805773&r2=805774&view=diff >>>> >>>> ============================================================================== >>>> --- lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >>>> (original) >>>> +++ lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java Wed >>>> Aug 19 12:21:22 2009 >>>> @@ -155,6 +155,8 @@ >>>> sawWaitSearcher = true; >>>> } else if >>>> (UpdateParams.MAX_OPTIMIZE_SEGMENTS.equals(attrName)) { >>>> cmd.maxOptimizeSegments = Integer.parseInt(attrVal); >>>> + } else if (UpdateParams.EXPUNGE_DELETES.equals(attrName)) { >>>> + cmd.expungeDeletes = StrUtils.parseBoolean(attrVal); >>>> } else { >>>> XmlUpdateRequestHandler.log.warn("unexpected attribute >>>> commit/@" + attrName); >>>> } >>>> >>>> Modified: >>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >>>> URL: >>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java?rev=805774&r1=805773&r2=805774&view=diff >>>> >>>> ============================================================================== >>>> --- >>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >>>> (original) >>>> +++ >>>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >>>> Wed Aug 19 12:21:22 2009 >>>> @@ -23,6 +23,7 @@ >>>> public boolean optimize; >>>> public boolean waitFlush; >>>> public boolean waitSearcher=true; >>>> + public boolean expungeDeletes = false; >>>> /** >>>> * During optimize, optimize down to <= this many segments. Must be >= >>>> 1 >>>> @@ -39,6 +40,7 @@ >>>> return "commit(optimize="+optimize >>>> +",waitFlush="+waitFlush >>>> +",waitSearcher="+waitSearcher >>>> + +",expungeDeletes="+expungeDeletes >>>> +')'; >>>> } >>>> } >>>> >>>> Modified: >>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >>>> URL: >>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java?rev=805774&r1=805773&r2=805774&view=diff >>>> >>>> ============================================================================== >>>> --- >>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >>>> (original) >>>> +++ >>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >>>> Wed Aug 19 12:21:22 2009 >>>> @@ -241,9 +241,10 @@ >>>> synchronized (this) { >>>> pset.clear(); >>>> closeSearcher(); // flush any deletes >>>> - if (cmd.optimize) { >>>> + if (cmd.optimize || cmd.expungeDeletes) { >>>> openWriter(); // writer needs to be open to optimize >>>> - writer.optimize(cmd.maxOptimizeSegments); >>>> + if(cmd.optimize) writer.optimize(cmd.maxOptimizeSegments); >>>> + if(cmd.expungeDeletes) writer.expungeDeletes(cmd.expungeDeletes); >>>> } >>>> closeWriter(); >>>> >>>> Modified: >>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >>>> URL: >>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=805774&r1=805773&r2=805774&view=diff >>>> >>>> ============================================================================== >>>> --- >>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >>>> (original) >>>> +++ >>>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >>>> Wed Aug 19 12:21:22 2009 >>>> @@ -130,6 +130,7 @@ >>>> AtomicLong deleteByIdCommandsCumulative= new AtomicLong(); >>>> AtomicLong deleteByQueryCommands= new AtomicLong(); >>>> AtomicLong deleteByQueryCommandsCumulative= new AtomicLong(); >>>> + AtomicLong expungeDeleteCommands = new AtomicLong(); >>>> AtomicLong mergeIndexesCommands = new AtomicLong(); >>>> AtomicLong commitCommands= new AtomicLong(); >>>> AtomicLong optimizeCommands= new AtomicLong(); >>>> @@ -382,6 +383,8 @@ >>>> if (cmd.optimize) { >>>> optimizeCommands.incrementAndGet(); >>>> + } else if (cmd.expungeDeletes) { >>>> + expungeDeleteCommands.incrementAndGet(); >>>> } else { >>>> commitCommands.incrementAndGet(); >>>> } >>>> @@ -402,6 +405,10 @@ >>>> } >>>> closeWriter(); >>>> + if (!cmd.optimize && cmd.expungeDeletes) { >>>> + openWriter(); >>>> + writer.expungeDeletes(); >>>> + } >>>> callPostCommitCallbacks(); >>>> if (cmd.optimize) { >>>> >>>> Modified: >>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >>>> URL: >>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java?rev=805774&r1=805773&r2=805774&view=diff >>>> >>>> ============================================================================== >>>> --- >>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >>>> (original) >>>> +++ >>>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >>>> Wed Aug 19 12:21:22 2009 >>>> @@ -153,6 +153,12 @@ >>>> return this; >>>> } >>>> + public UpdateRequest setAction(ACTION action, boolean waitFlush, >>>> boolean waitSearcher, int maxSegments , boolean expungeDeletes) { >>>> + setAction(action, waitFlush, waitSearcher,maxSegments) ; >>>> + params.set(UpdateParams.EXPUNGE_DELETES,""+expungeDeletes); >>>> + return this; >>>> + } >>>> + >>>> /** >>>> * @since Solr 1.4 >>>> */ >>>> >>>> Modified: >>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >>>> URL: >>>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java?rev=805774&r1=805773&r2=805774&view=diff >>>> >>>> ============================================================================== >>>> --- >>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >>>> (original) >>>> +++ >>>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >>>> Wed Aug 19 12:21:22 2009 >>>> @@ -17,20 +17,32 @@ >>>> package org.apache.solr.update; >>>> +import java.io.IOException; >>>> +import java.util.ArrayList; >>>> import java.util.HashMap; >>>> +import java.util.HashSet; >>>> +import java.util.List; >>>> import java.util.Map; >>>> +import java.util.Set; >>>> import org.apache.lucene.document.Document; >>>> import org.apache.lucene.document.Field; >>>> import org.apache.lucene.document.Field.Index; >>>> import org.apache.lucene.document.Field.Store; >>>> +import org.apache.lucene.index.IndexReader; >>>> +import org.apache.lucene.index.SegmentReader; >>>> +import org.apache.lucene.index.Term; >>>> +import org.apache.lucene.index.TermEnum; >>>> import org.apache.solr.common.SolrException; >>>> import org.apache.solr.common.params.CommonParams; >>>> import org.apache.solr.common.params.MapSolrParams; >>>> import org.apache.solr.core.SolrCore; >>>> import org.apache.solr.request.LocalSolrQueryRequest; >>>> import org.apache.solr.request.SolrQueryRequest; >>>> +import org.apache.solr.search.SolrIndexReader; >>>> +import org.apache.solr.search.SolrIndexSearcher; >>>> import org.apache.solr.util.AbstractSolrTestCase; >>>> +import org.apache.solr.util.RefCounted; >>>> /** >>>> * @@ -247,6 +259,90 @@ >>>> ); >>>> } >>>> + public void testExpungeDeletes() throws Exception { >>>> + for (int x = 0; x < 3000; x++) { >>>> + addSimpleDoc(x + ""); >>>> + } >>>> + SolrCore core = h.getCore(); >>>> + UpdateHandler updater = core.getUpdateHandler(); >>>> + CommitUpdateCommand cmtCmd = new CommitUpdateCommand(false); >>>> + cmtCmd.waitSearcher = true; >>>> + updater.commit(cmtCmd); >>>> + >>>> + List<String> todelete = new ArrayList<String>(); >>>> + >>>> + Set<String> segsdel = new HashSet<String>(); >>>> + >>>> + SegmentReader[] sirs = getSegmentReaders(core); >>>> + assertTrue(sirs.length > 6); >>>> + todelete.add(getNthIDTerm(2, sirs[0])); >>>> + segsdel.add(sirs[0].getSegmentName()); >>>> + + todelete.add(getNthIDTerm(7, sirs[2])); >>>> + segsdel.add(sirs[2].getSegmentName()); >>>> + + todelete.add(getNthIDTerm(4, sirs[5])); >>>> + segsdel.add(sirs[5].getSegmentName()); >>>> + + for (String id : todelete) { >>>> + deleteSimpleDoc(id); >>>> + } >>>> + // commit the deletes >>>> + cmtCmd = new CommitUpdateCommand(false); >>>> + cmtCmd.waitSearcher = true; >>>> + updater.commit(cmtCmd); >>>> + + // expunge deletes >>>> + cmtCmd = new CommitUpdateCommand(false); >>>> + cmtCmd.waitSearcher = true; >>>> + cmtCmd.expungeDeletes = true; >>>> + updater.commit(cmtCmd); >>>> + + // we'll have fewer segments >>>> + SegmentReader[] sirs2 = getSegmentReaders(core); >>>> + assertTrue(sirs.length > sirs2.length); >>>> + // check the actual segment names >>>> + for (SegmentReader sr : sirs2) { >>>> + assertTrue(!segsdel.contains(sr.getSegmentName())); >>>> + } >>>> + } >>>> + >>>> + SegmentReader[] getSegmentReaders(SolrCore core) throws IOException { >>>> + RefCounted<SolrIndexSearcher> ref = core.getSearcher(true, true, >>>> null); >>>> + SolrIndexSearcher is = ref.get(); >>>> + SegmentReader[] segmentReaders = null; >>>> + try { >>>> + SolrIndexReader reader = is.getReader(); >>>> + IndexReader[] subreaders = reader.getSequentialSubReaders(); >>>> + segmentReaders = new SegmentReader[subreaders.length]; >>>> + for (int x = 0; x < subreaders.length; x++) { >>>> + assert subreaders[x] instanceof SolrIndexReader; >>>> + SolrIndexReader sir = (SolrIndexReader) subreaders[x]; >>>> + SegmentReader sr = (SegmentReader) sir.getWrappedReader(); >>>> + segmentReaders[x] = sr; >>>> + } >>>> + } finally { >>>> + ref.decref(); >>>> + } >>>> + return segmentReaders; >>>> + } >>>> + >>>> + private String getNthIDTerm(int n, IndexReader r) throws IOException { >>>> + TermEnum te = r.terms(new Term("id", "")); >>>> + try { >>>> + int x = 0; >>>> + do { >>>> + if (x >= n) { >>>> + return te.term().text(); >>>> + } >>>> + x++; >>>> + } while (te.next()); >>>> + } finally { >>>> + te.close(); >>>> + } >>>> + return null; >>>> + } >>>> + private void addSimpleDoc(String id) throws Exception { >>>> SolrCore core = h.getCore(); >>>> >>>> >>>> >>>> >>> >>> > > > > > > > -- > Regards, > Rafał Kuć > > -- ----------------------------------------------------- Noble Paul | Principal Engineer| AOL | http://aol.com
