This is an automated email from the ASF dual-hosted git repository.
epugh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr-mcp.git
The following commit(s) were added to refs/heads/main by this push:
new e8d678cd perf(indexing): soft-commit instead of hard-commit, keeping
documents searchable (#196)
e8d678cd is described below
commit e8d678cd04aa7440c786846ab50f1c4d5e5c7f12
Author: Aditya Parikh <[email protected]>
AuthorDate: Wed Sep 16 10:40:00 2026 -0400
perf(indexing): soft-commit instead of hard-commit, keeping documents
searchable (#196)
Every indexing tool -- JSON, CSV, XML and markdown -- funnels through
indexDocuments, and indexDocuments ended with solrClient.commit(collection).
That is a hard commit: it fsyncs the segments, so each tool call waited on
the
storage device.
That is not what Solr's own defaults do. The _default configset ships
autoCommit
at maxTime 15000 with openSearcher=false, and autoSoftCommit at 3000: a
background hard commit purely to truncate the transaction log, and soft
commits
for visibility. Forcing a synchronous fsync per tool call fought that
design.
The commit is now waitFlush=false, waitSearcher=true, softCommit=true.
waitSearcher keeps the guarantee that matters to a tool caller: the
documents
are searchable the moment the call returns. Verified 30/30 with zero delay.
Durability is unchanged. The transaction log is written on the add, before
any
commit, so documents survive a crash regardless of commit mode; a hard
commit
governs how much tlog must be replayed on recovery, not whether data is
lost.
That housekeeping stays with autoCommit.
Measured against Solr, 20 interleaved reps, same endpoint and document, only
the commit parameter varying:
no commit 4.05 ms median
soft commit 8.61 ms median, p90 10.65
hard commit 18.94 ms median, p90 41.32
2.2x faster and far tighter -- the hard commit's p90 is four times its
median,
which is fsync variance. Over the MCP tools, 61 single-document calls go
from
1853 ms to 585 ms.
Operators running a custom configset with autoCommit disabled should enable
it,
or the transaction log grows until something else commits.
Signed-off-by: Aditya Parikh <[email protected]>
Co-authored-by: Claude Opus 5 (1M context) <[email protected]>
---
.../solr/mcp/server/indexing/IndexingService.java | 13 ++--
.../mcp/server/indexing/IndexingServiceTest.java | 71 +++++++++++++---------
2 files changed, 51 insertions(+), 33 deletions(-)
diff --git
a/src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java
b/src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java
index 13504967..c186bca6 100644
--- a/src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java
+++ b/src/main/java/org/apache/solr/mcp/server/indexing/IndexingService.java
@@ -494,8 +494,8 @@ public class IndexingService {
* failure
* <li><strong>Success Tracking</strong>: Accurate count of
successfully indexed
* documents
- * <li><strong>Commit Strategy</strong>: Single commit after all
batches for
- * consistency
+ * <li><strong>Commit Strategy</strong>: Single soft commit after all
batches
+ * for consistency
* </ul>
*
* <p>
@@ -521,7 +521,7 @@ public class IndexingService {
* <strong>Transaction Behavior:</strong>
*
* <p>
- * The method commits changes after all batches are processed, making
indexed
+ * The method soft-commits after all batches are processed, making
indexed
* documents immediately searchable. This ensures atomicity at the
operation
* level while maintaining performance through batching.
*
@@ -536,7 +536,7 @@ public class IndexingService {
* if there are critical errors in commit operations
* @see SolrInputDocument
* @see SolrClient#add(String, java.util.Collection)
- * @see SolrClient#commit(String)
+ * @see SolrClient#commit(String, boolean, boolean, boolean)
*/
/**
* Maximum number of distinct field names listed in an indexing
response before
@@ -599,7 +599,10 @@ public class IndexingService {
}
try {
- solrClient.commit(collection);
+ // waitFlush=false, waitSearcher=true, softCommit=true:
the documents are
+ // searchable when this method returns, while the hard
commit (segment fsync)
+ // is left to Solr's autoCommit, so many small calls do
not each force one.
+ solrClient.commit(collection, false, true, true);
} catch (SolrServerException | IOException e) {
logger.error("Failed to commit after indexing to
collection: {}", collection, e);
throw e;
diff --git
a/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceTest.java
b/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceTest.java
index b1491d16..9b94a0be 100644
--- a/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceTest.java
+++ b/src/test/java/org/apache/solr/mcp/server/indexing/IndexingServiceTest.java
@@ -67,13 +67,13 @@ class IndexingServiceTest {
List<SolrInputDocument> mockDocs = createMockDocuments(1);
when(indexingDocumentCreator.createSchemalessDocumentsFromJson(json)).thenReturn(mockDocs);
when(solrClient.add(eq("test_collection"),
any(Collection.class))).thenReturn(null);
- when(solrClient.commit("test_collection")).thenReturn(null);
+ when(solrClient.commit("test_collection", false, true,
true)).thenReturn(null);
indexingService.indexJsonDocuments("test_collection", json);
verify(indexingDocumentCreator).createSchemalessDocumentsFromJson(json);
verify(solrClient).add(eq("test_collection"),
any(Collection.class));
- verify(solrClient).commit("test_collection");
+ verify(solrClient).commit("test_collection", false, true, true);
}
@Test
@@ -86,7 +86,7 @@ class IndexingServiceTest {
indexingService.indexJsonDocuments("test_collection",
invalidJson);
});
verify(solrClient, never()).add(anyString(),
any(Collection.class));
- verify(solrClient, never()).commit(anyString());
+ verify(solrClient, never()).commit(anyString(), anyBoolean(),
anyBoolean(), anyBoolean());
}
@Test
@@ -95,13 +95,13 @@ class IndexingServiceTest {
List<SolrInputDocument> mockDocs = createMockDocuments(2);
when(indexingDocumentCreator.createSchemalessDocumentsFromCsv(csv)).thenReturn(mockDocs);
when(solrClient.add(eq("test_collection"),
any(Collection.class))).thenReturn(null);
- when(solrClient.commit("test_collection")).thenReturn(null);
+ when(solrClient.commit("test_collection", false, true,
true)).thenReturn(null);
indexingService.indexCsvDocuments("test_collection", csv);
verify(indexingDocumentCreator).createSchemalessDocumentsFromCsv(csv);
verify(solrClient).add(eq("test_collection"),
any(Collection.class));
- verify(solrClient).commit("test_collection");
+ verify(solrClient).commit("test_collection", false, true, true);
}
@Test
@@ -114,7 +114,7 @@ class IndexingServiceTest {
indexingService.indexCsvDocuments("test_collection",
invalidCsv);
});
verify(solrClient, never()).add(anyString(),
any(Collection.class));
- verify(solrClient, never()).commit(anyString());
+ verify(solrClient, never()).commit(anyString(), anyBoolean(),
anyBoolean(), anyBoolean());
}
@Test
@@ -123,13 +123,13 @@ class IndexingServiceTest {
List<SolrInputDocument> mockDocs = createMockDocuments(1);
when(indexingDocumentCreator.createSchemalessDocumentsFromXml(xml)).thenReturn(mockDocs);
when(solrClient.add(eq("test_collection"),
any(Collection.class))).thenReturn(null);
- when(solrClient.commit("test_collection")).thenReturn(null);
+ when(solrClient.commit("test_collection", false, true,
true)).thenReturn(null);
indexingService.indexXmlDocuments("test_collection", xml);
verify(indexingDocumentCreator).createSchemalessDocumentsFromXml(xml);
verify(solrClient).add(eq("test_collection"),
any(Collection.class));
- verify(solrClient).commit("test_collection");
+ verify(solrClient).commit("test_collection", false, true, true);
}
@Test
@@ -142,7 +142,7 @@ class IndexingServiceTest {
indexingService.indexXmlDocuments("test_collection",
xml);
});
verify(solrClient, never()).add(anyString(),
any(Collection.class));
- verify(solrClient, never()).commit(anyString());
+ verify(solrClient, never()).commit(anyString(), anyBoolean(),
anyBoolean(), anyBoolean());
}
@Test
@@ -156,33 +156,33 @@ class IndexingServiceTest {
indexingService.indexXmlDocuments("test_collection",
xml);
});
verify(solrClient, never()).add(anyString(),
any(Collection.class));
- verify(solrClient, never()).commit(anyString());
+ verify(solrClient, never()).commit(anyString(), anyBoolean(),
anyBoolean(), anyBoolean());
}
@Test
void indexDocuments_WithSmallBatch_ShouldIndexSuccessfully() throws
Exception {
List<SolrInputDocument> docs = createMockDocuments(5);
when(solrClient.add(eq("test_collection"),
any(Collection.class))).thenReturn(null);
- when(solrClient.commit("test_collection")).thenReturn(null);
+ when(solrClient.commit("test_collection", false, true,
true)).thenReturn(null);
int result = indexingService.indexDocuments("test_collection",
docs);
assertEquals(5, result);
verify(solrClient).add(eq("test_collection"),
any(Collection.class));
- verify(solrClient).commit("test_collection");
+ verify(solrClient).commit("test_collection", false, true, true);
}
@Test
void indexDocuments_WithLargeBatch_ShouldProcessInBatches() throws
Exception {
List<SolrInputDocument> docs = createMockDocuments(2500);
when(solrClient.add(eq("test_collection"),
any(Collection.class))).thenReturn(null);
- when(solrClient.commit(eq("test_collection"))).thenReturn(null);
+ when(solrClient.commit(eq("test_collection"), eq(false),
eq(true), eq(true))).thenReturn(null);
int result = indexingService.indexDocuments("test_collection",
docs);
assertEquals(2500, result);
verify(solrClient, times(3)).add(eq("test_collection"),
any(Collection.class));
- verify(solrClient).commit("test_collection");
+ verify(solrClient).commit("test_collection", false, true, true);
}
@Test
@@ -192,14 +192,14 @@ class IndexingServiceTest {
when(solrClient.add(eq("test_collection"),
any(List.class))).thenThrow(new SolrServerException("Batch error"));
when(solrClient.add(eq("test_collection"),
any(SolrInputDocument.class))).thenReturn(null);
- when(solrClient.commit("test_collection")).thenReturn(null);
+ when(solrClient.commit("test_collection", false, true,
true)).thenReturn(null);
int result = indexingService.indexDocuments("test_collection",
docs);
assertEquals(3, result);
verify(solrClient).add(eq("test_collection"),
any(Collection.class));
verify(solrClient, times(3)).add(eq("test_collection"),
any(SolrInputDocument.class));
- verify(solrClient).commit("test_collection");
+ verify(solrClient).commit("test_collection", false, true, true);
}
@Test
@@ -211,46 +211,61 @@ class IndexingServiceTest {
when(solrClient.add(eq("test_collection"),
any(SolrInputDocument.class))).thenReturn(null)
.thenThrow(new SolrServerException("Document
error")).thenReturn(null);
- when(solrClient.commit("test_collection")).thenReturn(null);
+ when(solrClient.commit("test_collection", false, true,
true)).thenReturn(null);
int result = indexingService.indexDocuments("test_collection",
docs);
assertEquals(2, result);
verify(solrClient).add(eq("test_collection"),
any(Collection.class));
verify(solrClient, times(3)).add(eq("test_collection"),
any(SolrInputDocument.class));
- verify(solrClient).commit("test_collection");
+ verify(solrClient).commit("test_collection", false, true, true);
}
@Test
void indexDocuments_WithEmptyList_ShouldStillCommit() throws Exception {
List<SolrInputDocument> emptyDocs = new ArrayList<>();
- when(solrClient.commit("test_collection")).thenReturn(null);
+ when(solrClient.commit("test_collection", false, true,
true)).thenReturn(null);
int result = indexingService.indexDocuments("test_collection",
emptyDocs);
assertEquals(0, result);
verify(solrClient, never()).add(anyString(), any(List.class));
- verify(solrClient).commit("test_collection");
+ verify(solrClient).commit("test_collection", false, true, true);
}
@Test
void indexDocuments_WhenCommitFails_ShouldPropagateException() throws
Exception {
List<SolrInputDocument> docs = createMockDocuments(2);
when(solrClient.add(eq("test_collection"),
any(Collection.class))).thenReturn(null);
- when(solrClient.commit("test_collection")).thenThrow(new
IOException("Commit failed"));
+ when(solrClient.commit("test_collection", false, true,
true)).thenThrow(new IOException("Commit failed"));
assertThrows(IOException.class, () -> {
indexingService.indexDocuments("test_collection", docs);
});
verify(solrClient).add(eq("test_collection"),
any(Collection.class));
- verify(solrClient).commit("test_collection");
+ verify(solrClient).commit("test_collection", false, true, true);
+ }
+
+ @Test
+ void
indexDocuments_SoftCommitsSoDocumentsAreSearchableWithoutForcingAnFsync()
throws Exception {
+ List<SolrInputDocument> docs = createMockDocuments(2);
+ when(solrClient.add(eq("test_collection"),
any(Collection.class))).thenReturn(null);
+
+ indexingService.indexDocuments("test_collection", docs);
+
+ // waitFlush=false, waitSearcher=true, softCommit=true.
waitSearcher is what
+ // keeps the documents searchable the moment the tool returns;
softCommit is
+ // what avoids an fsync per call. Measured against Solr: 8.6 ms
versus 18.9 ms
+ // for a hard commit, and a p90 of 10.7 ms versus 41.3 ms.
+ verify(solrClient).commit("test_collection", false, true, true);
+ verify(solrClient, never()).commit(anyString());
}
@Test
void indexDocuments_ShouldBatchCorrectly() throws Exception {
List<SolrInputDocument> docs = createMockDocuments(1000);
when(solrClient.add(eq("test_collection"),
any(Collection.class))).thenReturn(null);
- when(solrClient.commit("test_collection")).thenReturn(null);
+ when(solrClient.commit("test_collection", false, true,
true)).thenReturn(null);
int result = indexingService.indexDocuments("test_collection",
docs);
@@ -259,7 +274,7 @@ class IndexingServiceTest {
ArgumentCaptor<Collection<SolrInputDocument>> captor =
ArgumentCaptor.forClass(Collection.class);
verify(solrClient).add(eq("test_collection"), captor.capture());
assertEquals(1000, captor.getValue().size());
- verify(solrClient).commit("test_collection");
+ verify(solrClient).commit("test_collection", false, true, true);
}
@Test
@@ -271,7 +286,7 @@ class IndexingServiceTest {
.thenThrow(new SolrServerException("Solr
connection error"));
when(solrClient.add(eq("test_collection"),
any(SolrInputDocument.class)))
.thenThrow(new SolrServerException("Solr
connection error"));
- when(solrClient.commit("test_collection")).thenReturn(null);
+ when(solrClient.commit("test_collection", false, true,
true)).thenReturn(null);
indexingService.indexJsonDocuments("test_collection", json);
@@ -287,7 +302,7 @@ class IndexingServiceTest {
when(solrClient.add(eq("test_collection"),
any(List.class))).thenThrow(new IOException("Network error"));
when(solrClient.add(eq("test_collection"),
any(SolrInputDocument.class)))
.thenThrow(new IOException("Network error"));
- when(solrClient.commit("test_collection")).thenReturn(null);
+ when(solrClient.commit("test_collection", false, true,
true)).thenReturn(null);
indexingService.indexCsvDocuments("test_collection", csv);
@@ -303,14 +318,14 @@ class IndexingServiceTest {
.thenThrow(new RuntimeException("Unexpected
error"));
when(solrClient.add(eq("test_collection"),
any(SolrInputDocument.class))).thenReturn(null);
- when(solrClient.commit("test_collection")).thenReturn(null);
+ when(solrClient.commit("test_collection", false, true,
true)).thenReturn(null);
int result = indexingService.indexDocuments("test_collection",
docs);
assertEquals(2, result);
verify(solrClient).add(eq("test_collection"),
any(Collection.class));
verify(solrClient, times(2)).add(eq("test_collection"),
any(SolrInputDocument.class));
- verify(solrClient).commit("test_collection");
+ verify(solrClient).commit("test_collection", false, true, true);
}
private List<SolrInputDocument> createMockDocuments(int count) {
