On Wed, Aug 19, 2009 at 10:07 PM, Koji Sekiguchi<k...@r.email.ne.jp> wrote: > Noble, > > testExpungeDeletes fails: oh . It passed twice in my box.is any one else noticing this failure? > > <failure message="null" type="junit.framework.AssertionFailedError"> > junit.framework.AssertionFailedError: null > at > org.apache.solr.update.DirectUpdateHandlerTest.testExpungeDeletes(DirectUpdateHandlerTest.java:303) > </failure> > > Koji > > no...@apache.org wrote: >> >> Author: noble >> Date: Wed Aug 19 12:21:22 2009 >> New Revision: 805774 >> >> URL: http://svn.apache.org/viewvc?rev=805774&view=rev >> Log: >> SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 >> >> Modified: >> lucene/solr/trunk/CHANGES.txt >> >> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >> >> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >> >> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >> >> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >> >> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >> >> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >> >> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >> >> Modified: lucene/solr/trunk/CHANGES.txt >> URL: >> http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=805774&r1=805773&r2=805774&view=diff >> >> ============================================================================== >> --- lucene/solr/trunk/CHANGES.txt (original) >> +++ lucene/solr/trunk/CHANGES.txt Wed Aug 19 12:21:22 2009 >> @@ -267,7 +267,9 @@ >> HTMLStripStandardTokenizerFactory deprecated. To strip HTML tags, >> HTMLStripCharFilter can be used >> with an arbitrary Tokenizer. (koji) >> -68. SOLR-1367: Added callback mechanism for converting DocList to >> SolrDocumentList in SolrPluginUtils (gsingers) +68. SOLR-1367: Added >> callback mechanism for converting DocList to SolrDocumentList in >> SolrPluginUtils (gsingers) >> + >> +68. SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 (noble) >> Optimizations >> >> Modified: >> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >> URL: >> http://svn.apache.org/viewvc/lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java?rev=805774&r1=805773&r2=805774&view=diff >> >> ============================================================================== >> --- >> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >> (original) >> +++ >> lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java >> Wed Aug 19 12:21:22 2009 >> @@ -49,4 +49,6 @@ >> * If optimizing, set the maximum number of segments left in the index >> after optimization. 1 is the default (and is equivalent to calling >> IndexWriter.optimize() in Lucene). >> */ >> public static final String MAX_OPTIMIZE_SEGMENTS = "maxSegments"; >> + >> + public static final String EXPUNGE_DELETES = "expungeDeletes"; >> } >> >> Modified: >> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >> URL: >> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java?rev=805774&r1=805773&r2=805774&view=diff >> >> ============================================================================== >> --- >> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >> (original) >> +++ >> lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java >> Wed Aug 19 12:21:22 2009 >> @@ -68,6 +68,7 @@ >> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize ); >> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH, >> cmd.waitFlush ); >> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, >> cmd.waitSearcher ); >> + cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES, >> cmd.expungeDeletes); >> cmd.maxOptimizeSegments = >> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments); >> req.getCore().getUpdateHandler().commit( cmd ); >> @@ -101,6 +102,7 @@ >> CommitUpdateCommand cmd = new CommitUpdateCommand( optimize ); >> cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH, >> cmd.waitFlush ); >> cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER, >> cmd.waitSearcher ); >> + cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES, >> cmd.expungeDeletes); cmd.maxOptimizeSegments = >> params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments); >> processor.processCommit( cmd ); >> return true; >> >> Modified: >> lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >> URL: >> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java?rev=805774&r1=805773&r2=805774&view=diff >> >> ============================================================================== >> --- lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java >> (original) >> +++ lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java Wed >> Aug 19 12:21:22 2009 >> @@ -155,6 +155,8 @@ >> sawWaitSearcher = true; >> } else if >> (UpdateParams.MAX_OPTIMIZE_SEGMENTS.equals(attrName)) { >> cmd.maxOptimizeSegments = Integer.parseInt(attrVal); >> + } else if (UpdateParams.EXPUNGE_DELETES.equals(attrName)) { >> + cmd.expungeDeletes = StrUtils.parseBoolean(attrVal); >> } else { >> XmlUpdateRequestHandler.log.warn("unexpected attribute >> commit/@" + attrName); >> } >> >> Modified: >> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >> URL: >> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java?rev=805774&r1=805773&r2=805774&view=diff >> >> ============================================================================== >> --- >> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >> (original) >> +++ >> lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java >> Wed Aug 19 12:21:22 2009 >> @@ -23,6 +23,7 @@ >> public boolean optimize; >> public boolean waitFlush; >> public boolean waitSearcher=true; >> + public boolean expungeDeletes = false; >> /** >> * During optimize, optimize down to <= this many segments. Must be >= >> 1 >> @@ -39,6 +40,7 @@ >> return "commit(optimize="+optimize >> +",waitFlush="+waitFlush >> +",waitSearcher="+waitSearcher >> + +",expungeDeletes="+expungeDeletes >> +')'; >> } >> } >> >> Modified: >> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >> URL: >> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java?rev=805774&r1=805773&r2=805774&view=diff >> >> ============================================================================== >> --- >> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >> (original) >> +++ >> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java >> Wed Aug 19 12:21:22 2009 >> @@ -241,9 +241,10 @@ >> synchronized (this) { >> pset.clear(); >> closeSearcher(); // flush any deletes >> - if (cmd.optimize) { >> + if (cmd.optimize || cmd.expungeDeletes) { >> openWriter(); // writer needs to be open to optimize >> - writer.optimize(cmd.maxOptimizeSegments); >> + if(cmd.optimize) writer.optimize(cmd.maxOptimizeSegments); >> + if(cmd.expungeDeletes) writer.expungeDeletes(cmd.expungeDeletes); >> } >> closeWriter(); >> >> Modified: >> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >> URL: >> http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=805774&r1=805773&r2=805774&view=diff >> >> ============================================================================== >> --- >> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >> (original) >> +++ >> lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java >> Wed Aug 19 12:21:22 2009 >> @@ -130,6 +130,7 @@ >> AtomicLong deleteByIdCommandsCumulative= new AtomicLong(); >> AtomicLong deleteByQueryCommands= new AtomicLong(); >> AtomicLong deleteByQueryCommandsCumulative= new AtomicLong(); >> + AtomicLong expungeDeleteCommands = new AtomicLong(); >> AtomicLong mergeIndexesCommands = new AtomicLong(); >> AtomicLong commitCommands= new AtomicLong(); >> AtomicLong optimizeCommands= new AtomicLong(); >> @@ -382,6 +383,8 @@ >> if (cmd.optimize) { >> optimizeCommands.incrementAndGet(); >> + } else if (cmd.expungeDeletes) { >> + expungeDeleteCommands.incrementAndGet(); >> } else { >> commitCommands.incrementAndGet(); >> } >> @@ -402,6 +405,10 @@ >> } >> closeWriter(); >> + if (!cmd.optimize && cmd.expungeDeletes) { >> + openWriter(); >> + writer.expungeDeletes(); >> + } >> callPostCommitCallbacks(); >> if (cmd.optimize) { >> >> Modified: >> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >> URL: >> http://svn.apache.org/viewvc/lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java?rev=805774&r1=805773&r2=805774&view=diff >> >> ============================================================================== >> --- >> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >> (original) >> +++ >> lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java >> Wed Aug 19 12:21:22 2009 >> @@ -153,6 +153,12 @@ >> return this; >> } >> + public UpdateRequest setAction(ACTION action, boolean waitFlush, >> boolean waitSearcher, int maxSegments , boolean expungeDeletes) { >> + setAction(action, waitFlush, waitSearcher,maxSegments) ; >> + params.set(UpdateParams.EXPUNGE_DELETES,""+expungeDeletes); >> + return this; >> + } >> + >> /** >> * @since Solr 1.4 >> */ >> >> Modified: >> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >> URL: >> http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java?rev=805774&r1=805773&r2=805774&view=diff >> >> ============================================================================== >> --- >> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >> (original) >> +++ >> lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java >> Wed Aug 19 12:21:22 2009 >> @@ -17,20 +17,32 @@ >> package org.apache.solr.update; >> +import java.io.IOException; >> +import java.util.ArrayList; >> import java.util.HashMap; >> +import java.util.HashSet; >> +import java.util.List; >> import java.util.Map; >> +import java.util.Set; >> import org.apache.lucene.document.Document; >> import org.apache.lucene.document.Field; >> import org.apache.lucene.document.Field.Index; >> import org.apache.lucene.document.Field.Store; >> +import org.apache.lucene.index.IndexReader; >> +import org.apache.lucene.index.SegmentReader; >> +import org.apache.lucene.index.Term; >> +import org.apache.lucene.index.TermEnum; >> import org.apache.solr.common.SolrException; >> import org.apache.solr.common.params.CommonParams; >> import org.apache.solr.common.params.MapSolrParams; >> import org.apache.solr.core.SolrCore; >> import org.apache.solr.request.LocalSolrQueryRequest; >> import org.apache.solr.request.SolrQueryRequest; >> +import org.apache.solr.search.SolrIndexReader; >> +import org.apache.solr.search.SolrIndexSearcher; >> import org.apache.solr.util.AbstractSolrTestCase; >> +import org.apache.solr.util.RefCounted; >> /** >> * @@ -247,6 +259,90 @@ >> ); >> } >> + public void testExpungeDeletes() throws Exception { >> + for (int x = 0; x < 3000; x++) { >> + addSimpleDoc(x + ""); >> + } >> + SolrCore core = h.getCore(); >> + UpdateHandler updater = core.getUpdateHandler(); >> + CommitUpdateCommand cmtCmd = new CommitUpdateCommand(false); >> + cmtCmd.waitSearcher = true; >> + updater.commit(cmtCmd); >> + >> + List<String> todelete = new ArrayList<String>(); >> + >> + Set<String> segsdel = new HashSet<String>(); >> + >> + SegmentReader[] sirs = getSegmentReaders(core); >> + assertTrue(sirs.length > 6); >> + todelete.add(getNthIDTerm(2, sirs[0])); >> + segsdel.add(sirs[0].getSegmentName()); >> + + todelete.add(getNthIDTerm(7, sirs[2])); >> + segsdel.add(sirs[2].getSegmentName()); >> + + todelete.add(getNthIDTerm(4, sirs[5])); >> + segsdel.add(sirs[5].getSegmentName()); >> + + for (String id : todelete) { >> + deleteSimpleDoc(id); >> + } >> + // commit the deletes >> + cmtCmd = new CommitUpdateCommand(false); >> + cmtCmd.waitSearcher = true; >> + updater.commit(cmtCmd); >> + + // expunge deletes >> + cmtCmd = new CommitUpdateCommand(false); >> + cmtCmd.waitSearcher = true; >> + cmtCmd.expungeDeletes = true; >> + updater.commit(cmtCmd); >> + + // we'll have fewer segments >> + SegmentReader[] sirs2 = getSegmentReaders(core); >> + assertTrue(sirs.length > sirs2.length); >> + // check the actual segment names >> + for (SegmentReader sr : sirs2) { >> + assertTrue(!segsdel.contains(sr.getSegmentName())); >> + } >> + } >> + >> + SegmentReader[] getSegmentReaders(SolrCore core) throws IOException { >> + RefCounted<SolrIndexSearcher> ref = core.getSearcher(true, true, >> null); >> + SolrIndexSearcher is = ref.get(); >> + SegmentReader[] segmentReaders = null; >> + try { >> + SolrIndexReader reader = is.getReader(); >> + IndexReader[] subreaders = reader.getSequentialSubReaders(); >> + segmentReaders = new SegmentReader[subreaders.length]; >> + for (int x = 0; x < subreaders.length; x++) { >> + assert subreaders[x] instanceof SolrIndexReader; >> + SolrIndexReader sir = (SolrIndexReader) subreaders[x]; >> + SegmentReader sr = (SegmentReader) sir.getWrappedReader(); >> + segmentReaders[x] = sr; >> + } >> + } finally { >> + ref.decref(); >> + } >> + return segmentReaders; >> + } >> + >> + private String getNthIDTerm(int n, IndexReader r) throws IOException { >> + TermEnum te = r.terms(new Term("id", "")); >> + try { >> + int x = 0; >> + do { >> + if (x >= n) { >> + return te.term().text(); >> + } >> + x++; >> + } while (te.next()); >> + } finally { >> + te.close(); >> + } >> + return null; >> + } >> + private void addSimpleDoc(String id) throws Exception { >> SolrCore core = h.getCore(); >> >> >> >> > >
-- ----------------------------------------------------- Noble Paul | Principal Engineer| AOL | http://aol.com
