This is an automated email from the ASF dual-hosted git repository. dsmiley pushed a commit to branch branch_10x in repository https://gitbox.apache.org/repos/asf/solr.git
commit ca957478839b2e03cf0a7a60732f3413c016af0b Author: Andrzej Białecki <[email protected]> AuthorDate: Wed Sep 9 11:31:27 2026 +0200 SOLR-18412: CrossDC - ZooKeeper /crossdc.properties no longer overrides solr.crossdc.kafka.* sysprops or env vars (#4884) (cherry picked from commit 62aace94a13f1f2a6831e6b189393ceecb9195d4) --- changelog/unreleased/SOLR-18412.yml | 8 +++ .../solr/crossdc/common/KafkaCrossDcConf.java | 13 +++- .../apache/solr/crossdc/common/ConfUtilTest.java | 81 ++++++++++++++++++++++ 3 files changed, 100 insertions(+), 2 deletions(-) diff --git a/changelog/unreleased/SOLR-18412.yml b/changelog/unreleased/SOLR-18412.yml new file mode 100644 index 00000000000..6fdc63a806f --- /dev/null +++ b/changelog/unreleased/SOLR-18412.yml @@ -0,0 +1,8 @@ +title: CrossDC - ZooKeeper /crossdc.properties no longer overrides solr.crossdc.kafka.* sysprops or env vars +type: fixed +authors: + - name: Andrzej Bialecki + nick: ab +links: + - name: SOLR-18412 + url: https://issues.apache.org/jira/browse/SOLR-18412 diff --git a/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/KafkaCrossDcConf.java b/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/KafkaCrossDcConf.java index 88db372683e..64dfc2cae7a 100644 --- a/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/KafkaCrossDcConf.java +++ b/solr/modules/cross-dc/src/java/org/apache/solr/crossdc/common/KafkaCrossDcConf.java @@ -295,8 +295,17 @@ public class KafkaCrossDcConf extends CrossDcConf { } zkPropsUnprocessed.forEach( (key, val) -> { - if (properties.get(key) == null) { - properties.put((String) key, val); + String strKey = (String) key; + String targetKey = strKey; + if (strKey.startsWith(ConfUtil.KAFKA_ENV_PREFIX)) { + targetKey = ConfUtil.normalizeKafkaEnvKey(strKey); + } else if (strKey.startsWith(ConfUtil.KAFKA_PROP_PREFIX)) { + targetKey = ConfUtil.normalizeKafkaSysPropKey(strKey); + } + // A JVM system property or env var pass-through override always takes precedence + // over the same property coming from ZooKeeper's /crossdc.properties. + if (properties.get(targetKey) == null) { + properties.put(targetKey, val); } }); } diff --git a/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/common/ConfUtilTest.java b/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/common/ConfUtilTest.java index 6afcc35ae2a..05591e8f8d5 100644 --- a/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/common/ConfUtilTest.java +++ b/solr/modules/cross-dc/src/test/org/apache/solr/crossdc/common/ConfUtilTest.java @@ -426,6 +426,87 @@ public class ConfUtilTest extends SolrTestCaseJ4 { assertEquals("lz4", conf.getAdditionalProperties().get("compression.type")); } + @Test + public void testFillProperties_ZkOnlyKafkaPrefixPropIsNormalized() throws Exception { + Map<String, Object> properties = new HashMap<>(); + + System.setProperty(KafkaCrossDcConf.BOOTSTRAP_SERVERS, "sys-kafka:9092"); + System.setProperty(KafkaCrossDcConf.TOPIC_NAME, "sys-topic"); + + Properties zkProps = new Properties(); + zkProps.setProperty("solr.crossdc.kafka.compression.type", "zk-value"); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + OutputStreamWriter writer = new OutputStreamWriter(baos, StandardCharsets.UTF_8); + zkProps.store(writer, null); + writer.close(); + byte[] zkData = baos.toByteArray(); + + when(mockZkClient.exists(anyString())).thenReturn(true); + when(mockZkClient.getData(anyString(), isNull(), isNull())).thenReturn(zkData); + + ConfUtil.fillProperties(mockZkClient, properties); + + // ZK value lands under the normalized key, not the raw solr.crossdc.kafka.* key. + assertEquals("zk-value", properties.get("compression.type")); + assertNull(properties.get("solr.crossdc.kafka.compression.type")); + } + + @Test + public void testFillProperties_KafkaPrefixSysPropPrecedenceOverZk() throws Exception { + Map<String, Object> properties = new HashMap<>(); + + System.setProperty(KafkaCrossDcConf.BOOTSTRAP_SERVERS, "sys-kafka:9092"); + System.setProperty(KafkaCrossDcConf.TOPIC_NAME, "sys-topic"); + System.setProperty("solr.crossdc.kafka.compression.type", "sys-value"); + + Properties zkProps = new Properties(); + zkProps.setProperty("solr.crossdc.kafka.compression.type", "zk-value"); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + OutputStreamWriter writer = new OutputStreamWriter(baos, StandardCharsets.UTF_8); + zkProps.store(writer, null); + writer.close(); + byte[] zkData = baos.toByteArray(); + + when(mockZkClient.exists(anyString())).thenReturn(true); + when(mockZkClient.getData(anyString(), isNull(), isNull())).thenReturn(zkData); + + ConfUtil.fillProperties(mockZkClient, properties); + + assertEquals("sys-value", properties.get("compression.type")); + } + + @Test + public void testFillProperties_SysPropPrecedenceOverDoubleZk() throws Exception { + Map<String, Object> properties = new HashMap<>(); + + System.setProperty(KafkaCrossDcConf.BOOTSTRAP_SERVERS, "sys-kafka:9092"); + System.setProperty(KafkaCrossDcConf.TOPIC_NAME, "sys-topic"); + System.setProperty("solr.crossdc.kafka.compression.type", "sys-value"); + + // First call, as Consumer.start() does, with no ZK client yet. + ConfUtil.fillProperties(null, properties); + assertEquals("sys-value", properties.get("compression.type")); + + Properties zkProps = new Properties(); + zkProps.setProperty("solr.crossdc.kafka.compression.type", "zk-value"); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + OutputStreamWriter writer = new OutputStreamWriter(baos, StandardCharsets.UTF_8); + zkProps.store(writer, null); + writer.close(); + byte[] zkData = baos.toByteArray(); + + when(mockZkClient.exists(anyString())).thenReturn(true); + when(mockZkClient.getData(anyString(), isNull(), isNull())).thenReturn(zkData); + + // Second call, now with the real ZK client, as Consumer.start() does. + ConfUtil.fillProperties(mockZkClient, properties); + + assertEquals("sys-value", properties.get("compression.type")); + } + // we can't easily modify envvars, test just the key conversion in properties @Test public void testUnderscoreToDotsConversion() {
