This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 33949044b2c SOLR-18415: CrossDC Common: improve serialization for
large docs. (#4887)
33949044b2c is described below
commit 33949044b2c92bb137d02546ac0cbd030e8733a2
Author: Andrzej Białecki <[email protected]>
AuthorDate: Wed Sep 9 17:29:08 2026 +0200
SOLR-18415: CrossDC Common: improve serialization for large docs. (#4887)
(cherry picked from commit ba8f866d05371bc54898ad93ec06cf2f998d05ac)
---
changelog/unreleased/solr-18415.yml | 8 ++
.../common/MirroredSolrRequestSerializer.java | 27 ++++-
.../common/MirroredSolrRequestSerializerTest.java | 115 +++++++++++++++++++++
3 files changed, 147 insertions(+), 3 deletions(-)
diff --git a/changelog/unreleased/solr-18415.yml
b/changelog/unreleased/solr-18415.yml
new file mode 100644
index 00000000000..3aba47c964a
--- /dev/null
+++ b/changelog/unreleased/solr-18415.yml
@@ -0,0 +1,8 @@
+title: CrossDC Common optimize serialization for large docs
+
+type: changed
+authors:
+ - name: Andrzej Bialecki
+links:
+ - name: SOLR-18415
+ url: https://issues.apache.org/jira/browse/SOLR-18415
diff --git
a/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/MirroredSolrRequestSerializer.java
b/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/MirroredSolrRequestSerializer.java
index 4bac11fea92..137d46f106e 100644
---
a/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/MirroredSolrRequestSerializer.java
+++
b/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/MirroredSolrRequestSerializer.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.crossdc.common;
+import com.google.common.annotations.VisibleForTesting;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -202,7 +203,7 @@ public class MirroredSolrRequestSerializer
try (JavaBinCodec codec = new JavaBinCodec(null)) {
- ExposedByteArrayOutputStream baos = new ExposedByteArrayOutputStream();
+ ExposedByteArrayOutputStream baos = newExposedByteArrayOutputStream();
Map<String, Object> map = CollectionUtil.newHashMap(8);
map.put("attempt", request.getAttempt());
map.put("submitTimeNanos", request.getSubmitTimeNanos());
@@ -219,7 +220,9 @@ public class MirroredSolrRequestSerializer
map.put("deletes", deletes.keySet());
map.put("deletesParams", deletes.values());
}
- map.put("deleteQuery", update.getDeleteQuery());
+ if (update.getDeleteQuery() != null &&
!update.getDeleteQuery().isEmpty()) {
+ map.put("deleteQuery", update.getDeleteQuery());
+ }
} else if (solrRequest instanceof
MirroredSolrRequest.MirroredConfigSetRequest config) {
map.put("method", config.getMethod().toString());
if (config.getContentStreams() != null) {
@@ -248,6 +251,11 @@ public class MirroredSolrRequestSerializer
}
}
+ @VisibleForTesting
+ ExposedByteArrayOutputStream newExposedByteArrayOutputStream() {
+ return new ExposedByteArrayOutputStream();
+ }
+
/**
* Close this serializer.
*
@@ -258,12 +266,25 @@ public class MirroredSolrRequestSerializer
Serializer.super.close();
}
- private static final class ExposedByteArrayOutputStream extends
ByteArrayOutputStream {
+ @VisibleForTesting
+ static final class ExposedByteArrayOutputStream extends
ByteArrayOutputStream {
ExposedByteArrayOutputStream() {
super();
}
byte[] byteArray() {
+ // padding exceeds 1/10th of the buffer size
+ // it's worth it to do an extra array copy to avoid sending
+ // the padding bytes over the wire
+ if (buf.length - count > buf.length / 10) {
+ return toByteArray();
+ } else {
+ return buf;
+ }
+ }
+
+ @VisibleForTesting
+ byte[] getBuffer() {
return buf;
}
}
diff --git
a/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/common/MirroredSolrRequestSerializerTest.java
b/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/common/MirroredSolrRequestSerializerTest.java
index 0b211b8bc32..598ef314846 100644
---
a/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/common/MirroredSolrRequestSerializerTest.java
+++
b/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/common/MirroredSolrRequestSerializerTest.java
@@ -16,8 +16,11 @@
*/
package org.apache.solr.crossdc.common;
+import static org.apache.solr.SolrTestCaseJ4.assumeWorkingMockito;
+
import java.util.Arrays;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.solr.SolrTestCase;
@@ -26,12 +29,19 @@ import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
+import org.junit.BeforeClass;
import org.junit.Test;
+import org.mockito.Mockito;
public class MirroredSolrRequestSerializerTest extends SolrTestCase {
private static final byte[] EMPTY_ARR = new byte[3];
+ @BeforeClass
+ public static void ensureWorkingMockito() {
+ assumeWorkingMockito();
+ }
+
@Test
public void testSerializationBufferOptimization() {
MirroredSolrRequestSerializer serializer = new
MirroredSolrRequestSerializer();
@@ -166,4 +176,109 @@ public class MirroredSolrRequestSerializerTest extends
SolrTestCase {
assertEquals(List.of("field:value"), deserializedReq.getDeleteQuery());
}
+
+ /**
+ * Confirms the waste pattern from {@link
MirroredSolrRequestSerializer#serialize(String,
+ * MirroredSolrRequest)} testing both small and large documents around the
8192 bytes boundary.
+ */
+ @Test
+ public void testSerializationWasteForRealDocumentSizes() {
+ MirroredSolrRequest<?> emptyRequest = new MirroredSolrRequest<>(new
UpdateRequest());
+ Sizes emptySizes = serializeAndMeasure(emptyRequest);
+ assertTrue(
+ "expected empty-request framing to exceed the 32-byte floor, was "
+ + emptySizes.actualSize(),
+ emptySizes.actualSize() > 32);
+ assertTrue(
+ "expected empty-request framing to land in the 32-64 zone, was " +
emptySizes.actualSize(),
+ emptySizes.actualSize() <= 64);
+ logWaste(emptySizes);
+
+ // 64B-8K zone: single doc, single flush -> exact-fit growth, zero padding
waste
+ for (int fieldLen = 50; fieldLen < 8000; fieldLen +=
random().nextInt(1000) + 100) {
+ Sizes sizes = serializeAndMeasure(docWithField(fieldLen));
+ assertTrue("expected to stay under 8k for fieldLen=" + fieldLen,
sizes.actualSize() < 8192);
+ assertEquals(
+ "expected zero padding waste in the 64B-8K zone, actualSize=" +
sizes.actualSize(),
+ sizes.actualSize(),
+ sizes.bufferSize());
+ logWaste(sizes);
+ }
+
+ // 8k+ zone: a single large doc forces FastOutputStream to flush in
8192-byte chunks,
+ // re-triggering the buffer growth and real, measurable waste. This waste
is clamped down at 10%
+ for (int fieldLen = 8192; fieldLen < 100000; fieldLen +=
random().nextInt(8000) + 1000) {
+ Sizes sizes = serializeAndMeasure(docWithField(fieldLen));
+ assertTrue("expected to exceed 8k for fieldLen=" + fieldLen,
sizes.actualSize() > 8192);
+ double wasteRatio = sizes.waste() / (double) sizes.bufferSize();
+ logWaste(sizes);
+ if (sizes.waste() == 0) {
+ // expect no waste but only due to clamped down padding
+ assertTrue(
+ "allocated buffer size "
+ + sizes.bufferSize()
+ + " should be larger than the returned buffer size "
+ + sizes.returnedSize(),
+ sizes.bufferSize() > sizes.returnedSize());
+ }
+ assertTrue(
+ "waste should never exceed ~10% of the transmitted bufferSize,
ratio=" + wasteRatio,
+ wasteRatio < 0.11);
+ }
+ }
+
+ private static void logWaste(Sizes sizes) {
+ System.err.println(
+ "actualSize="
+ + sizes.actualSize()
+ + "\treturnedSize="
+ + sizes.returnedSize()
+ + "\t(allocatedBufferSize="
+ + sizes.bufferSize()
+ + ")"
+ + "\twaste="
+ + sizes.waste()
+ + "\t("
+ + String.format(Locale.ROOT, "%.1f%%", 100.0 * sizes.waste() /
sizes.bufferSize())
+ + " of transmitted bytes)");
+ }
+
+ private static MirroredSolrRequest<?> docWithField(int fieldLen) {
+ UpdateRequest req = new UpdateRequest();
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.setField("id", "1");
+ doc.setField("body", "a".repeat(fieldLen));
+ req.add(doc);
+ return new MirroredSolrRequest<>(req);
+ }
+
+ private record Sizes(int actualSize, int returnedSize, int bufferSize) {
+ int waste() {
+ return returnedSize - actualSize;
+ }
+ }
+
+ /**
+ * Serializes {@code request} and measures both the true, unpadded size and
the padded transmitted
+ * bufferSize.
+ */
+ private static Sizes serializeAndMeasure(MirroredSolrRequest<?> request) {
+ MirroredSolrRequestSerializer serializer = Mockito.spy(new
MirroredSolrRequestSerializer());
+ // capture the actual instance of the ExposedByteArrayOutputStream, so we
can measure
+ // both the raw buffer size and the size() of just the serialized data in
it
+ MirroredSolrRequestSerializer.ExposedByteArrayOutputStream[] captured =
+ new MirroredSolrRequestSerializer.ExposedByteArrayOutputStream[1];
+ Mockito.doAnswer(
+ invocation -> {
+ captured[0] =
+ (MirroredSolrRequestSerializer.ExposedByteArrayOutputStream)
+ invocation.callRealMethod();
+ return captured[0];
+ })
+ .when(serializer)
+ .newExposedByteArrayOutputStream();
+
+ byte[] data = serializer.serialize("test", request);
+ return new Sizes(captured[0].size(), data.length,
captured[0].getBuffer().length);
+ }
}
