Hello! Same here, assertion fails on our hudson.
> On Wed, Aug 19, 2009 at 10:07 PM, Koji > Sekiguchi<k...@r.email.ne.jp> wrote: >> Noble, >> >> testExpungeDeletes fails: > oh . It passed twice in my box.is any one else noticing this failure? >> >> <failure message="null" type="junit.framework.AssertionFailedError"> >> junit.framework.AssertionFailedError: null >> at >> org.apache.solr.update.DirectUpdateHandlerTest.testExpungeDeletes(DirectUpdateHandlerTest.java:303) >> </failure> >> >> Koji >> >> no...@apache.org wrote: >>> >>> Author: noble >>> Date: Wed Aug 19 12:21:22 2009 >>> New Revision: 805774 >>> >>> URL: http://svn.apache.org/viewvc?rev=805774&view=rev >>> Log: >>> SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 >>> >>> Modified: >>> lucene/solr/trunk/CHANGES.txt >>> >>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >>> >>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >>> >>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >>> >>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >>> >>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >>> >>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >>> >>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >>> >>> Modified: lucene/solr/trunk/CHANGES.txt >>> URL: >>> http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=805774&r1=805773&r2=805774&view=diff >>> >>> ============================================================================== >>> --- lucene/solr/trunk/CHANGES.txt (original) >>> +++ lucene/solr/trunk/CHANGES.txt Wed Aug 19 12:21:22 2009 >>> @@ -267,7 +267,9 @@ >>> HTMLStripStandardTokenizerFactory deprecated. To strip HTML tags, >>> HTMLStripCharFilter can be used >>> with an arbitrary Tokenizer. (koji) >>> -68. SOLR-1367: Added callback mechanism for converting DocList to >>> SolrDocumentList in SolrPluginUtils (gsingers) +68. SOLR-1367: Added >>> callback mechanism for converting DocList to SolrDocumentList in >>> SolrPluginUtils (gsingers) >>> + >>> +68. SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 (noble) >>> Optimizations >>> >>> Modified: >>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >>> URL: >>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java?rev=805774&r1=805773&r2=805774&view=diff >>> >>> ============================================================================== >>> --- >>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >>> (original) >>> +++ >>> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >>> Wed Aug 19 12:21:22 2009 >>> @@ -49,4 +49,6 @@ >>> * If optimizing, set the maximum number of segments left in the index >>> after optimization. 1 is the default (and is equivalent to calling >>> IndexWriter.optimize() in Lucene). >>> */ >>> public static final String MAX_OPTIMIZE_SEGMENTS = "maxSegments"; >>> + >>> + public static final String EXPUNGE_DELETES = "expungeDeletes"; >>> } >>> >>> Modified: >>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >>> URL: >>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java?rev=805774&r1=805773&r2=805774&view=diff >>> >>> ============================================================================== >>> --- >>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >>> (original) >>> +++ >>> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >>> Wed Aug 19 12:21:22 2009 >>> @@ -68,6 +68,7 @@ >>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize ); >>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH, >>> cmd.waitFlush ); >>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, >>> cmd.waitSearcher ); >>> + cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES, >>> cmd.expungeDeletes); >>> cmd.maxOptimizeSegments = >>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments); >>> req.getCore().getUpdateHandler().commit( cmd ); >>> @@ -101,6 +102,7 @@ >>> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize ); >>> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH, >>> cmd.waitFlush ); >>> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, >>> cmd.waitSearcher ); >>> + cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES, >>> cmd.expungeDeletes); cmd.maxOptimizeSegments = >>> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments); >>> processor.processCommit( cmd ); >>> return true; >>> >>> Modified: >>> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >>> URL: >>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java?rev=805774&r1=805773&r2=805774&view=diff >>> >>> ============================================================================== >>> --- lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >>> (original) >>> +++ lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java Wed >>> Aug 19 12:21:22 2009 >>> @@ -155,6 +155,8 @@ >>> sawWaitSearcher = true; >>> } else if >>> (UpdateParams.MAX_OPTIMIZE_SEGMENTS.equals(attrName)) { >>> cmd.maxOptimizeSegments = Integer.parseInt(attrVal); >>> + } else if (UpdateParams.EXPUNGE_DELETES.equals(attrName)) { >>> + cmd.expungeDeletes = StrUtils.parseBoolean(attrVal); >>> } else { >>> XmlUpdateRequestHandler.log.warn("unexpected attribute >>> commit/@" + attrName); >>> } >>> >>> Modified: >>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >>> URL: >>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java?rev=805774&r1=805773&r2=805774&view=diff >>> >>> ============================================================================== >>> --- >>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >>> (original) >>> +++ >>> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >>> Wed Aug 19 12:21:22 2009 >>> @@ -23,6 +23,7 @@ >>> public boolean optimize; >>> public boolean waitFlush; >>> public boolean waitSearcher=true; >>> + public boolean expungeDeletes = false; >>> /** >>> * During optimize, optimize down to <= this many segments. Must be >= >>> 1 >>> @@ -39,6 +40,7 @@ >>> return "commit(optimize="+optimize >>> +",waitFlush="+waitFlush >>> +",waitSearcher="+waitSearcher >>> + +",expungeDeletes="+expungeDeletes >>> +')'; >>> } >>> } >>> >>> Modified: >>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >>> URL: >>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java?rev=805774&r1=805773&r2=805774&view=diff >>> >>> ============================================================================== >>> --- >>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >>> (original) >>> +++ >>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >>> Wed Aug 19 12:21:22 2009 >>> @@ -241,9 +241,10 @@ >>> synchronized (this) { >>> pset.clear(); >>> closeSearcher(); // flush any deletes >>> - if (cmd.optimize) { >>> + if (cmd.optimize || cmd.expungeDeletes) { >>> openWriter(); // writer needs to be open to optimize >>> - writer.optimize(cmd.maxOptimizeSegments); >>> + if(cmd.optimize) writer.optimize(cmd.maxOptimizeSegments); >>> + if(cmd.expungeDeletes) writer.expungeDeletes(cmd.expungeDeletes); >>> } >>> closeWriter(); >>> >>> Modified: >>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >>> URL: >>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=805774&r1=805773&r2=805774&view=diff >>> >>> ============================================================================== >>> --- >>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >>> (original) >>> +++ >>> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >>> Wed Aug 19 12:21:22 2009 >>> @@ -130,6 +130,7 @@ >>> AtomicLong deleteByIdCommandsCumulative= new AtomicLong(); >>> AtomicLong deleteByQueryCommands= new AtomicLong(); >>> AtomicLong deleteByQueryCommandsCumulative= new AtomicLong(); >>> + AtomicLong expungeDeleteCommands = new AtomicLong(); >>> AtomicLong mergeIndexesCommands = new AtomicLong(); >>> AtomicLong commitCommands= new AtomicLong(); >>> AtomicLong optimizeCommands= new AtomicLong(); >>> @@ -382,6 +383,8 @@ >>> if (cmd.optimize) { >>> optimizeCommands.incrementAndGet(); >>> + } else if (cmd.expungeDeletes) { >>> + expungeDeleteCommands.incrementAndGet(); >>> } else { >>> commitCommands.incrementAndGet(); >>> } >>> @@ -402,6 +405,10 @@ >>> } >>> closeWriter(); >>> + if (!cmd.optimize && cmd.expungeDeletes) { >>> + openWriter(); >>> + writer.expungeDeletes(); >>> + } >>> callPostCommitCallbacks(); >>> if (cmd.optimize) { >>> >>> Modified: >>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >>> URL: >>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java?rev=805774&r1=805773&r2=805774&view=diff >>> >>> ============================================================================== >>> --- >>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >>> (original) >>> +++ >>> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >>> Wed Aug 19 12:21:22 2009 >>> @@ -153,6 +153,12 @@ >>> return this; >>> } >>> + public UpdateRequest setAction(ACTION action, boolean waitFlush, >>> boolean waitSearcher, int maxSegments , boolean expungeDeletes) { >>> + setAction(action, waitFlush, waitSearcher,maxSegments) ; >>> + params.set(UpdateParams.EXPUNGE_DELETES,""+expungeDeletes); >>> + return this; >>> + } >>> + >>> /** >>> * @since Solr 1.4 >>> */ >>> >>> Modified: >>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >>> URL: >>> http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java?rev=805774&r1=805773&r2=805774&view=diff >>> >>> ============================================================================== >>> --- >>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >>> (original) >>> +++ >>> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >>> Wed Aug 19 12:21:22 2009 >>> @@ -17,20 +17,32 @@ >>> package org.apache.solr.update; >>> +import java.io.IOException; >>> +import java.util.ArrayList; >>> import java.util.HashMap; >>> +import java.util.HashSet; >>> +import java.util.List; >>> import java.util.Map; >>> +import java.util.Set; >>> import org.apache.lucene.document.Document; >>> import org.apache.lucene.document.Field; >>> import org.apache.lucene.document.Field.Index; >>> import org.apache.lucene.document.Field.Store; >>> +import org.apache.lucene.index.IndexReader; >>> +import org.apache.lucene.index.SegmentReader; >>> +import org.apache.lucene.index.Term; >>> +import org.apache.lucene.index.TermEnum; >>> import org.apache.solr.common.SolrException; >>> import org.apache.solr.common.params.CommonParams; >>> import org.apache.solr.common.params.MapSolrParams; >>> import org.apache.solr.core.SolrCore; >>> import org.apache.solr.request.LocalSolrQueryRequest; >>> import org.apache.solr.request.SolrQueryRequest; >>> +import org.apache.solr.search.SolrIndexReader; >>> +import org.apache.solr.search.SolrIndexSearcher; >>> import org.apache.solr.util.AbstractSolrTestCase; >>> +import org.apache.solr.util.RefCounted; >>> /** >>> * @@ -247,6 +259,90 @@ >>> ); >>> } >>> + public void testExpungeDeletes() throws Exception { >>> + for (int x = 0; x < 3000; x++) { >>> + addSimpleDoc(x + ""); >>> + } >>> + SolrCore core = h.getCore(); >>> + UpdateHandler updater = core.getUpdateHandler(); >>> + CommitUpdateCommand cmtCmd = new CommitUpdateCommand(false); >>> + cmtCmd.waitSearcher = true; >>> + updater.commit(cmtCmd); >>> + >>> + List<String> todelete = new ArrayList<String>(); >>> + >>> + Set<String> segsdel = new HashSet<String>(); >>> + >>> + SegmentReader[] sirs = getSegmentReaders(core); >>> + assertTrue(sirs.length > 6); >>> + todelete.add(getNthIDTerm(2, sirs[0])); >>> + segsdel.add(sirs[0].getSegmentName()); >>> + + todelete.add(getNthIDTerm(7, sirs[2])); >>> + segsdel.add(sirs[2].getSegmentName()); >>> + + todelete.add(getNthIDTerm(4, sirs[5])); >>> + segsdel.add(sirs[5].getSegmentName()); >>> + + for (String id : todelete) { >>> + deleteSimpleDoc(id); >>> + } >>> + // commit the deletes >>> + cmtCmd = new CommitUpdateCommand(false); >>> + cmtCmd.waitSearcher = true; >>> + updater.commit(cmtCmd); >>> + + // expunge deletes >>> + cmtCmd = new CommitUpdateCommand(false); >>> + cmtCmd.waitSearcher = true; >>> + cmtCmd.expungeDeletes = true; >>> + updater.commit(cmtCmd); >>> + + // we'll have fewer segments >>> + SegmentReader[] sirs2 = getSegmentReaders(core); >>> + assertTrue(sirs.length > sirs2.length); >>> + // check the actual segment names >>> + for (SegmentReader sr : sirs2) { >>> + assertTrue(!segsdel.contains(sr.getSegmentName())); >>> + } >>> + } >>> + >>> + SegmentReader[] getSegmentReaders(SolrCore core) throws IOException { >>> + RefCounted<SolrIndexSearcher> ref = core.getSearcher(true, true, >>> null); >>> + SolrIndexSearcher is = ref.get(); >>> + SegmentReader[] segmentReaders = null; >>> + try { >>> + SolrIndexReader reader = is.getReader(); >>> + IndexReader[] subreaders = reader.getSequentialSubReaders(); >>> + segmentReaders = new SegmentReader[subreaders.length]; >>> + for (int x = 0; x < subreaders.length; x++) { >>> + assert subreaders[x] instanceof SolrIndexReader; >>> + SolrIndexReader sir = (SolrIndexReader) subreaders[x]; >>> + SegmentReader sr = (SegmentReader) sir.getWrappedReader(); >>> + segmentReaders[x] = sr; >>> + } >>> + } finally { >>> + ref.decref(); >>> + } >>> + return segmentReaders; >>> + } >>> + >>> + private String getNthIDTerm(int n, IndexReader r) throws IOException { >>> + TermEnum te = r.terms(new Term("id", "")); >>> + try { >>> + int x = 0; >>> + do { >>> + if (x >= n) { >>> + return te.term().text(); >>> + } >>> + x++; >>> + } while (te.next()); >>> + } finally { >>> + te.close(); >>> + } >>> + return null; >>> + } >>> + private void addSimpleDoc(String id) throws Exception { >>> SolrCore core = h.getCore(); >>> >>> >>> >>> >> >> -- Regards, Rafał Kuć
