Author: noble
Date: Wed Aug 19 12:21:22 2009
New Revision: 805774
URL: http://svn.apache.org/viewvc?rev=805774&view=rev
Log:
SOLR-1275: Add expungeDeletes to DirectUpdateHandler2
Modified:
lucene/solr/trunk/CHANGES.txt
lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
Modified: lucene/solr/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/CHANGES.txt?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
--- lucene/solr/trunk/CHANGES.txt (original)
+++ lucene/solr/trunk/CHANGES.txt Wed Aug 19 12:21:22 2009
@@ -267,7 +267,9 @@
HTMLStripStandardTokenizerFactory deprecated. To strip HTML tags,
HTMLStripCharFilter can be used
with an arbitrary Tokenizer. (koji)
-68. SOLR-1367: Added callback mechanism for converting DocList to
SolrDocumentList in SolrPluginUtils (gsingers) +68. SOLR-1367: Added
callback mechanism for converting DocList to SolrDocumentList in
SolrPluginUtils (gsingers)
+
+68. SOLR-1275: Add expungeDeletes to DirectUpdateHandler2 (noble)
Optimizations
Modified:
lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
---
lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
(original)
+++
lucene/solr/trunk/src/common/org/apache/solr/common/params/UpdateParams.java
Wed Aug 19 12:21:22 2009
@@ -49,4 +49,6 @@
* If optimizing, set the maximum number of segments left in the index
after optimization. 1 is the default (and is equivalent to calling
IndexWriter.optimize() in Lucene).
*/
public static final String MAX_OPTIMIZE_SEGMENTS = "maxSegments";
+
+ public static final String EXPUNGE_DELETES = "expungeDeletes";
}
Modified:
lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
---
lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
(original)
+++
lucene/solr/trunk/src/java/org/apache/solr/handler/RequestHandlerUtils.java
Wed Aug 19 12:21:22 2009
@@ -68,6 +68,7 @@
CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH,
cmd.waitFlush );
cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER,
cmd.waitSearcher );
+ cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES,
cmd.expungeDeletes);
cmd.maxOptimizeSegments =
params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
req.getCore().getUpdateHandler().commit( cmd );
@@ -101,6 +102,7 @@
CommitUpdateCommand cmd = new CommitUpdateCommand( optimize );
cmd.waitFlush = params.getBool( UpdateParams.WAIT_FLUSH,
cmd.waitFlush );
cmd.waitSearcher = params.getBool( UpdateParams.WAIT_SEARCHER,
cmd.waitSearcher );
+ cmd.expungeDeletes = params.getBool( UpdateParams.EXPUNGE_DELETES,
cmd.expungeDeletes); cmd.maxOptimizeSegments =
params.getInt(UpdateParams.MAX_OPTIMIZE_SEGMENTS, cmd.maxOptimizeSegments);
processor.processCommit( cmd );
return true;
Modified:
lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
--- lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java
(original)
+++ lucene/solr/trunk/src/java/org/apache/solr/handler/XMLLoader.java Wed
Aug 19 12:21:22 2009
@@ -155,6 +155,8 @@
sawWaitSearcher = true;
} else if
(UpdateParams.MAX_OPTIMIZE_SEGMENTS.equals(attrName)) {
cmd.maxOptimizeSegments = Integer.parseInt(attrVal);
+ } else if (UpdateParams.EXPUNGE_DELETES.equals(attrName)) {
+ cmd.expungeDeletes = StrUtils.parseBoolean(attrVal);
} else {
XmlUpdateRequestHandler.log.warn("unexpected attribute
commit/@" + attrName);
}
Modified:
lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
---
lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
(original)
+++
lucene/solr/trunk/src/java/org/apache/solr/update/CommitUpdateCommand.java
Wed Aug 19 12:21:22 2009
@@ -23,6 +23,7 @@
public boolean optimize;
public boolean waitFlush;
public boolean waitSearcher=true;
+ public boolean expungeDeletes = false;
/**
* During optimize, optimize down to <= this many segments. Must be >=
1
@@ -39,6 +40,7 @@
return "commit(optimize="+optimize
+",waitFlush="+waitFlush
+",waitSearcher="+waitSearcher
+ +",expungeDeletes="+expungeDeletes
+')';
}
}
Modified:
lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
---
lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
(original)
+++
lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler.java
Wed Aug 19 12:21:22 2009
@@ -241,9 +241,10 @@
synchronized (this) {
pset.clear();
closeSearcher(); // flush any deletes
- if (cmd.optimize) {
+ if (cmd.optimize || cmd.expungeDeletes) {
openWriter(); // writer needs to be open to optimize
- writer.optimize(cmd.maxOptimizeSegments);
+ if(cmd.optimize) writer.optimize(cmd.maxOptimizeSegments);
+ if(cmd.expungeDeletes) writer.expungeDeletes(cmd.expungeDeletes);
}
closeWriter();
Modified:
lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
---
lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
(original)
+++
lucene/solr/trunk/src/java/org/apache/solr/update/DirectUpdateHandler2.java
Wed Aug 19 12:21:22 2009
@@ -130,6 +130,7 @@
AtomicLong deleteByIdCommandsCumulative= new AtomicLong();
AtomicLong deleteByQueryCommands= new AtomicLong();
AtomicLong deleteByQueryCommandsCumulative= new AtomicLong();
+ AtomicLong expungeDeleteCommands = new AtomicLong();
AtomicLong mergeIndexesCommands = new AtomicLong();
AtomicLong commitCommands= new AtomicLong();
AtomicLong optimizeCommands= new AtomicLong();
@@ -382,6 +383,8 @@
if (cmd.optimize) {
optimizeCommands.incrementAndGet();
+ } else if (cmd.expungeDeletes) {
+ expungeDeleteCommands.incrementAndGet();
} else {
commitCommands.incrementAndGet();
}
@@ -402,6 +405,10 @@
}
closeWriter();
+ if (!cmd.optimize && cmd.expungeDeletes) {
+ openWriter();
+ writer.expungeDeletes();
+ }
callPostCommitCallbacks();
if (cmd.optimize) {
Modified:
lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
---
lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
(original)
+++
lucene/solr/trunk/src/solrj/org/apache/solr/client/solrj/request/UpdateRequest.java
Wed Aug 19 12:21:22 2009
@@ -153,6 +153,12 @@
return this;
}
+ public UpdateRequest setAction(ACTION action, boolean waitFlush,
boolean waitSearcher, int maxSegments , boolean expungeDeletes) {
+ setAction(action, waitFlush, waitSearcher,maxSegments) ;
+ params.set(UpdateParams.EXPUNGE_DELETES,""+expungeDeletes);
+ return this;
+ }
+
/**
* @since Solr 1.4
*/
Modified:
lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
URL:
http://svn.apache.org/viewvc/lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java?rev=805774&r1=805773&r2=805774&view=diff
==============================================================================
---
lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
(original)
+++
lucene/solr/trunk/src/test/org/apache/solr/update/DirectUpdateHandlerTest.java
Wed Aug 19 12:21:22 2009
@@ -17,20 +17,32 @@
package org.apache.solr.update;
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.SegmentReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermEnum;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.search.SolrIndexReader;
+import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.AbstractSolrTestCase;
+import org.apache.solr.util.RefCounted;
/**
* @@ -247,6 +259,90 @@
);
}
+ public void testExpungeDeletes() throws Exception {
+ for (int x = 0; x < 3000; x++) {
+ addSimpleDoc(x + "");
+ }
+ SolrCore core = h.getCore();
+ UpdateHandler updater = core.getUpdateHandler();
+ CommitUpdateCommand cmtCmd = new CommitUpdateCommand(false);
+ cmtCmd.waitSearcher = true;
+ updater.commit(cmtCmd);
+
+ List<String> todelete = new ArrayList<String>();
+
+ Set<String> segsdel = new HashSet<String>();
+
+ SegmentReader[] sirs = getSegmentReaders(core);
+ assertTrue(sirs.length > 6);
+ todelete.add(getNthIDTerm(2, sirs[0]));
+ segsdel.add(sirs[0].getSegmentName());
+ + todelete.add(getNthIDTerm(7, sirs[2]));
+ segsdel.add(sirs[2].getSegmentName());
+ + todelete.add(getNthIDTerm(4, sirs[5]));
+ segsdel.add(sirs[5].getSegmentName());
+ + for (String id : todelete) {
+ deleteSimpleDoc(id);
+ }
+ // commit the deletes
+ cmtCmd = new CommitUpdateCommand(false);
+ cmtCmd.waitSearcher = true;
+ updater.commit(cmtCmd);
+ + // expunge deletes
+ cmtCmd = new CommitUpdateCommand(false);
+ cmtCmd.waitSearcher = true;
+ cmtCmd.expungeDeletes = true;
+ updater.commit(cmtCmd);
+ + // we'll have fewer segments
+ SegmentReader[] sirs2 = getSegmentReaders(core);
+ assertTrue(sirs.length > sirs2.length);
+ // check the actual segment names
+ for (SegmentReader sr : sirs2) {
+ assertTrue(!segsdel.contains(sr.getSegmentName()));
+ }
+ }
+
+ SegmentReader[] getSegmentReaders(SolrCore core) throws IOException {
+ RefCounted<SolrIndexSearcher> ref = core.getSearcher(true, true,
null);
+ SolrIndexSearcher is = ref.get();
+ SegmentReader[] segmentReaders = null;
+ try {
+ SolrIndexReader reader = is.getReader();
+ IndexReader[] subreaders = reader.getSequentialSubReaders();
+ segmentReaders = new SegmentReader[subreaders.length];
+ for (int x = 0; x < subreaders.length; x++) {
+ assert subreaders[x] instanceof SolrIndexReader;
+ SolrIndexReader sir = (SolrIndexReader) subreaders[x];
+ SegmentReader sr = (SegmentReader) sir.getWrappedReader();
+ segmentReaders[x] = sr;
+ }
+ } finally {
+ ref.decref();
+ }
+ return segmentReaders;
+ }
+
+ private String getNthIDTerm(int n, IndexReader r) throws IOException {
+ TermEnum te = r.terms(new Term("id", ""));
+ try {
+ int x = 0;
+ do {
+ if (x >= n) {
+ return te.term().text();
+ }
+ x++;
+ } while (te.next());
+ } finally {
+ te.close();
+ }
+ return null;
+ }
+ private void addSimpleDoc(String id) throws Exception {
SolrCore core = h.getCore();
