This is an automated email from the ASF dual-hosted git repository.
raghavyadav01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 6ac289762d9 Add ignoredKeys config to OPEN_STRUCT columns (#19314)
6ac289762d9 is described below
commit 6ac289762d9290b8800afc6811f57ce75f72fb1e
Author: tarun11Mavani <[email protected]>
AuthorDate: Wed Aug 26 22:09:31 2026 +0530
Add ignoredKeys config to OPEN_STRUCT columns (#19314)
* test(open_struct): add end-to-end ignoredKeys coverage
Add a "debug" key present in every ingested row but listed in
ignoredKeys, configured so it would otherwise land in the sparse
column (maxDenseKeys is already saturated by explicit denseKeys).
Covers both OFFLINE and REALTIME via the shared test base:
- testIgnoredKeyNotQueryable: the key is never queryable end-to-end
- testCommittedSegmentIndexMap: the key has no materialized column
in the committed segment
* fix(open_struct): only meter ignored-key drops for non-null values
A null value is already skipped by the null-value check regardless of
ignoredKeys, so counting it toward OPEN_STRUCT_IGNORED_KEY_DROPS
misattributed drops that had nothing to do with the ignoredKeys
config. Check the null value before the ignored-key check in both
MutableOpenStructIndex and OpenStructColumnSplitter so the metric
only reflects real data suppressed by ignoredKeys.
---
.../apache/pinot/common/metrics/ServerMeter.java | 3 +
.../custom/OpenStructIngestionCommitTestBase.java | 23 +++++-
.../impl/openstruct/OpenStructColumnSplitter.java | 14 ++++
.../index/openstruct/MutableOpenStructIndex.java | 15 ++++
.../index/openstruct/OpenStructIndexType.java | 25 ++++++
.../openstruct/OpenStructColumnSplitterTest.java | 88 ++++++++++++++++++++++
.../openstruct/MutableOpenStructIndexTest.java | 64 ++++++++++++++++
.../index/openstruct/OpenStructIndexTypeTest.java | 71 +++++++++++++++++
.../spi/config/table/OpenStructIndexConfig.java | 30 +++++++-
.../config/table/OpenStructIndexConfigTest.java | 28 +++++++
10 files changed, 358 insertions(+), 3 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java
index 055d2cb2cbe..c87e7dee4af 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ServerMeter.java
@@ -246,6 +246,9 @@ public enum ServerMeter implements AbstractMetrics.Meter {
"Number of OPEN_STRUCT values stored as their serialized string form
because the value's Java type maps to "
+ "no Pinot DataType. A value that is instead dropped for failing
coercion against a key's "
+ "already-established non-STRING type is counted by
openStructTypeCoercionFailures, not here"),
+ OPEN_STRUCT_IGNORED_KEY_DROPS("values", false,
+ "Number of OPEN_STRUCT map entries dropped at ingestion because the key
is listed in the "
+ + "column's ignoredKeys config"),
// Workload related metrics
WORKLOAD_BUDGET_EXCEEDED("workloadBudgetExceeded", true, "Number of times
workload budget exceeded"),
WORKLOAD_QUERIES("queries", false),
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/OpenStructIngestionCommitTestBase.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/OpenStructIngestionCommitTestBase.java
index 7ee3e17d191..aae6cec3954 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/OpenStructIngestionCommitTestBase.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/OpenStructIngestionCommitTestBase.java
@@ -116,9 +116,12 @@ public abstract class OpenStructIngestionCommitTestBase
extends CustomDataQueryC
FieldConfig hostCfg = new FieldConfig.Builder("host")
.withEncodingType(FieldConfig.EncodingType.RAW)
.build();
- // First arg is `disabled` — false => enabled.
+ // First arg is `disabled` — false => enabled. "debug" is present in every
row (100% fill
+ // rate) but maxDenseKeys=3 is already saturated by the explicit
denseKeys, so absent
+ // ignoredKeys handling it would land in the sparse column instead of
being dropped —
+ // that's what makes testIgnoredKeyNotQueryable below a meaningful
end-to-end check.
OpenStructIndexConfig osConfig = new OpenStructIndexConfig(false, null, 3,
- Set.of("views", "cpu", "host"), 0.5, List.of(viewsCfg, cpuCfg,
hostCfg), null);
+ Set.of("views", "cpu", "host"), 0.5, List.of(viewsCfg, cpuCfg,
hostCfg), null, null, Set.of("debug"));
ObjectNode indexes = JsonUtils.newObjectNode();
indexes.set(OPEN_STRUCT_INDEX_NAME, JsonUtils.objectToJsonNode(osConfig));
FieldConfig metricsCfg = new
FieldConfig.Builder(METRICS).withIndexes(indexes).build();
@@ -156,6 +159,7 @@ public abstract class OpenStructIngestionCommitTestBase
extends CustomDataQueryC
metrics.put("host", "host-" + (i % 5)); // STRING, small
set (raw forward)
metrics.put("region", "region-" + (i % 4)); // sparse
metrics.put("latencyMs", String.valueOf(i % 100)); // sparse
+ metrics.put("debug", "noise-" + i); // ignoredKeys:
dropped at ingestion
GenericData.Record record = new GenericData.Record(avroSchema);
record.put(METRICS, metrics);
record.put(TIMESTAMP_FIELD_NAME, tsBase + i);
@@ -214,6 +218,17 @@ public abstract class OpenStructIngestionCommitTestBase
extends CustomDataQueryC
assertEquals(response.get("resultTable").get("rows").get(0).get(0).asLong(),
49500);
}
+ @Test
+ public void testIgnoredKeyNotQueryable()
+ throws Exception {
+ // "debug" is present in every ingested row (see createAvroFiles) but is
listed in
+ // ignoredKeys, so it must be dropped at ingestion and never queryable
end-to-end.
+ JsonNode response = postQuery(
+ "SELECT COUNT(*) FROM " + getTableName() + " WHERE " + METRICS +
"['debug'] = 'noise-0'");
+ assertEquals(response.get("exceptions").size(), 0);
+
assertEquals(response.get("resultTable").get("rows").get(0).get(0).asLong(), 0);
+ }
+
@Test
public void testManifestShortCircuitNonexistentKey()
throws Exception {
@@ -255,6 +270,10 @@ public abstract class OpenStructIngestionCommitTestBase
extends CustomDataQueryC
assertTrue(cols.containsKey(sparse), "sparse JSON column
metrics$__sparse__ missing");
assertEquals(cols.get(sparse).getDataType(), FieldSpec.DataType.STRING);
+ // ignoredKeys: "debug" is dropped entirely, so it's neither dense nor
sparse.
+ String debug = OpenStructNaming.materializedColumnName(METRICS, "debug");
+ assertFalse(cols.containsKey(debug), "metrics$debug must NOT be
materialized (ignoredKeys)");
+
// index_map per key.
try (SegmentDirectory dir = new SegmentLocalFSDirectory(segmentDir,
ReadMode.mmap);
SegmentDirectory.Reader reader = dir.createReader()) {
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitter.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitter.java
index 2bdbf8a0e71..01565e06605 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitter.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitter.java
@@ -100,6 +100,7 @@ public class OpenStructColumnSplitter implements
ColumnarOpenStructIndexCreator
private final Map<String, Long> _coercionFailuresPerKey = new HashMap<>();
private final Map<String, Long> _inferenceFailuresPerKey = new HashMap<>();
private int _numDocs;
+ private int _ignoredKeyDropCount;
// Resolved at seal time
@Nullable
@@ -200,6 +201,10 @@ public class OpenStructColumnSplitter implements
ColumnarOpenStructIndexCreator
if (rawValue == null) {
continue;
}
+ if (_config.isIgnoredKey(key)) {
+ _ignoredKeyDropCount++;
+ continue;
+ }
FieldSpec keySpec = _childFieldSpecs.get(key);
DataType valueType;
if (keySpec != null) {
@@ -287,6 +292,15 @@ public class OpenStructColumnSplitter implements
ColumnarOpenStructIndexCreator
}
emitMetrics(sparseKeys.size(), totalCoercionFailures,
totalInferenceFailures);
+ if (_ignoredKeyDropCount > 0) {
+ LOGGER.info("OPEN_STRUCT '{}': dropped {} entries for ignored keys",
_columnName, _ignoredKeyDropCount);
+ ServerMetrics serverMetrics = ServerMetrics.get();
+ if (serverMetrics != null) {
+ serverMetrics.addMeteredTableValue(_tableNameWithType, _columnName,
+ ServerMeter.OPEN_STRUCT_IGNORED_KEY_DROPS, _ignoredKeyDropCount);
+ }
+ }
+
emitParentColumnMetadata(sparseKeys);
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructIndex.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructIndex.java
index 648f0dd5ee9..23bb33795a1 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructIndex.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructIndex.java
@@ -65,6 +65,10 @@ public class MutableOpenStructIndex implements
OpenStructIndexReader<ForwardInde
// Volatile for lock-free reader access; writer always holds the
consuming-thread lock.
private volatile Map<String, MutableKeyColumn> _keyColumns = new HashMap<>();
+ // Single-writer (see #index), but close() may run on a different thread, so
volatile for
+ // visibility; flushed to ServerMetrics on close() to avoid a metered-value
call on every
+ // ignored key of every consumed row.
+ private volatile long _ignoredKeyDropCount;
public MutableOpenStructIndex(String openStructColumn, String
tableNameWithType, ComplexFieldSpec fieldSpec,
OpenStructIndexConfig config, PinotDataBufferMemoryManager
memoryManager, int capacity) {
@@ -102,6 +106,10 @@ public class MutableOpenStructIndex implements
OpenStructIndexReader<ForwardInde
if (rawValue == null) {
continue;
}
+ if (_config.isIgnoredKey(key)) {
+ _ignoredKeyDropCount++;
+ continue;
+ }
MutableKeyColumn keyCol = _keyColumns.get(key);
if (keyCol == null) {
@@ -282,6 +290,13 @@ public class MutableOpenStructIndex implements
OpenStructIndexReader<ForwardInde
@Override
public void close()
throws IOException {
+ if (_ignoredKeyDropCount > 0) {
+ ServerMetrics serverMetrics = ServerMetrics.get();
+ if (serverMetrics != null) {
+ serverMetrics.addMeteredTableValue(_tableNameWithType,
_openStructColumn,
+ ServerMeter.OPEN_STRUCT_IGNORED_KEY_DROPS, _ignoredKeyDropCount);
+ }
+ }
for (MutableKeyColumn keyCol : _keyColumns.values()) {
keyCol.close();
}
diff --git
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructIndexType.java
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructIndexType.java
index 383975c1ba4..1b0e5c296aa 100644
---
a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructIndexType.java
+++
b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructIndexType.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import javax.annotation.Nullable;
import
org.apache.pinot.segment.local.segment.creator.impl.openstruct.OpenStructColumnSplitter;
import org.apache.pinot.segment.spi.ColumnMetadata;
@@ -43,6 +44,7 @@ import org.apache.pinot.segment.spi.store.SegmentDirectory;
import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.config.table.OpenStructIndexConfig;
import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.data.ComplexFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.Schema;
@@ -85,6 +87,7 @@ public class OpenStructIndexType
"OPEN_STRUCT index can only be created on single-value columns, but
column '%s' is multi-value",
fieldSpec.getName());
validatePerKeyIndexes(config);
+ validateIgnoredKeys(config, fieldSpec);
}
}
@@ -111,6 +114,28 @@ public class OpenStructIndexType
}
}
+ private void validateIgnoredKeys(OpenStructIndexConfig config, FieldSpec
fieldSpec) {
+ Set<String> ignoredKeys = config.getIgnoredKeys();
+ if (ignoredKeys.isEmpty()) {
+ return;
+ }
+ for (String key : ignoredKeys) {
+ Preconditions.checkState(!config.getDenseKeys().contains(key),
+ "OPEN_STRUCT column '%s': key '%s' is in both ignoredKeys and
denseKeys", fieldSpec.getName(), key);
+ Preconditions.checkState(config.getValueFieldConfig(key) == null,
+ "OPEN_STRUCT column '%s': key '%s' is in ignoredKeys but also has a
valueFieldConfigs entry",
+ fieldSpec.getName(), key);
+ }
+ if (fieldSpec instanceof ComplexFieldSpec) {
+ Map<String, FieldSpec> childFieldSpecs = ((ComplexFieldSpec)
fieldSpec).getChildFieldSpecs();
+ for (String key : ignoredKeys) {
+ Preconditions.checkState(childFieldSpecs == null ||
!childFieldSpecs.containsKey(key),
+ "OPEN_STRUCT column '%s': key '%s' is in ignoredKeys but also
declared in childFieldSpecs",
+ fieldSpec.getName(), key);
+ }
+ }
+ }
+
@Override
public String getPrettyName() {
return INDEX_DISPLAY_NAME;
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitterTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitterTest.java
index abb140a8b60..b8795f02c53 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitterTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/creator/impl/openstruct/OpenStructColumnSplitterTest.java
@@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import java.io.File;
import java.math.BigDecimal;
import java.nio.file.Files;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -56,6 +57,7 @@ import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
@@ -588,6 +590,74 @@ public class OpenStructColumnSplitterTest {
}
}
+ @Test
+ public void testIgnoredKeyNeverMaterializedDenseOrSparse()
+ throws Exception {
+ // "debug" has a fill rate far below the default denseKeyMinFillRate
(0.5), so if it were not
+ // dropped by ignoredKeys it would land in the sparse manifest (see
testRareKeyDroppedFromDense).
+ // This is what makes the sparse-manifest assertion below meaningful
rather than vacuous.
+ OpenStructIndexConfig cfg = new OpenStructIndexConfig(false, null, -1,
null, 0.5, null, null, null,
+ Set.of("debug"));
+ OpenStructColumnSplitter s = new OpenStructColumnSplitter(_tempDir,
"metrics", "testTable_OFFLINE", spec(), cfg);
+ s.add(Map.of("debug", "noise", "clicks", 0L), 0);
+ for (int d = 1; d < 100; d++) {
+ s.add(Map.of("clicks", (long) d), d);
+ }
+ s.seal();
+
+ Set<String> dense = s.getResolvedDenseKeys();
+ assertFalse(dense.contains("debug"));
+ assertTrue(dense.contains("clicks"));
+
+ PropertiesConfiguration parentProps =
s.getMaterializedColumnMetadata().get("metrics");
+ assertNotNull(parentProps);
+ assertFalse(parentProps.containsKey(
+ V1Constants.MetadataKeys.Column.getKeyFor("metrics",
V1Constants.MetadataKeys.Column.SPARSE_KEYS)));
+ }
+
+ @Test
+ public void testIgnoredKeyMeteredOnSeal()
+ throws Exception {
+ OpenStructIndexConfig cfg = JsonUtils.stringToObject(
+ "{\"ignoredKeys\": [\"debug\"]}", OpenStructIndexConfig.class);
+ ServerMetrics metrics = mock(ServerMetrics.class);
+ assertTrue(ServerMetrics.register(metrics), "another ServerMetrics is
already registered");
+ try {
+ OpenStructColumnSplitter s = new OpenStructColumnSplitter(_tempDir,
"metrics", "testTable_OFFLINE", spec(),
+ cfg);
+ s.add(Map.of("debug", "a"), 0);
+ s.add(Map.of("debug", "b", "clicks", 1L), 1);
+ s.seal();
+
+ verify(metrics, times(1)).addMeteredTableValue("testTable_OFFLINE",
"metrics",
+ ServerMeter.OPEN_STRUCT_IGNORED_KEY_DROPS, 2L);
+ } finally {
+ ServerMetrics.deregister();
+ }
+ }
+
+ @Test
+ public void testIgnoredKeyWithNullValueNotMetered()
+ throws Exception {
+ OpenStructIndexConfig cfg = JsonUtils.stringToObject(
+ "{\"ignoredKeys\": [\"debug\"]}", OpenStructIndexConfig.class);
+ ServerMetrics metrics = mock(ServerMetrics.class);
+ assertTrue(ServerMetrics.register(metrics), "another ServerMetrics is
already registered");
+ try {
+ OpenStructColumnSplitter s = new OpenStructColumnSplitter(_tempDir,
"metrics", "testTable_OFFLINE", spec(),
+ cfg);
+ Map<String, Object> row = new HashMap<>();
+ row.put("debug", null);
+ s.add(row, 0);
+ s.seal();
+
+ verify(metrics, never()).addMeteredTableValue(anyString(), anyString(),
+ eq(ServerMeter.OPEN_STRUCT_IGNORED_KEY_DROPS), anyLong());
+ } finally {
+ ServerMetrics.deregister();
+ }
+ }
+
/// A value that cannot be mapped to a DataType but lands on a key whose
type is already
/// established as something other than STRING is dropped by coercion, not
stored as STRING. It
/// must be counted once, against the coercion meter only — counting it as
an inference failure
@@ -792,4 +862,22 @@ public class OpenStructColumnSplitterTest {
ServerMetrics.deregister();
}
}
+
+ @Test
+ public void testNoIgnoredKeyDropsNotMetered()
+ throws Exception {
+ ServerMetrics metrics = mock(ServerMetrics.class);
+ assertTrue(ServerMetrics.register(metrics), "another ServerMetrics is
already registered");
+ try {
+ OpenStructColumnSplitter s = new OpenStructColumnSplitter(_tempDir,
"metrics", "testTable_OFFLINE", spec(),
+ config(0.5, -1, null));
+ s.add(Map.of("clicks", 1L), 0);
+ s.seal();
+
+ verify(metrics, never()).addMeteredTableValue(anyString(), anyString(),
+ eq(ServerMeter.OPEN_STRUCT_IGNORED_KEY_DROPS), anyLong());
+ } finally {
+ ServerMetrics.deregister();
+ }
+ }
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructIndexTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructIndexTest.java
index 492cbeac33e..b08eb2ae1d0 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructIndexTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/MutableOpenStructIndexTest.java
@@ -19,6 +19,7 @@
package org.apache.pinot.segment.local.segment.index.openstruct;
import java.io.IOException;
+import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.apache.pinot.common.metrics.ServerMeter;
@@ -28,6 +29,7 @@ import
org.apache.pinot.segment.spi.memory.PinotDataBufferMemoryManager;
import org.apache.pinot.spi.config.table.OpenStructIndexConfig;
import org.apache.pinot.spi.data.ComplexFieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.utils.JsonUtils;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -213,4 +215,66 @@ public class MutableOpenStructIndexTest {
ServerMetrics.deregister();
}
}
+
+ @Test
+ public void testIgnoredKeyNeverAllocatesColumn()
+ throws Exception {
+ OpenStructIndexConfig config = JsonUtils.stringToObject(
+ "{\"ignoredKeys\": [\"debug\"]}", OpenStructIndexConfig.class);
+ try (MutableOpenStructIndex idx = new MutableOpenStructIndex(
+ "metrics", "testTable_REALTIME", openStructSpec(), config, _memMgr,
1000)) {
+ idx.index(0, Map.of("debug", "noise", "clicks", 5L));
+
+ assertNull(idx.getKeyColumn("debug"), "Ignored key must never allocate a
column");
+ assertTrue(idx.getKeys().contains("clicks"), "Non-ignored key must still
be indexed");
+ assertEquals(idx.getKeys().size(), 1);
+ assertEquals(idx.getMapValue(0), Map.of("clicks", 5L));
+ }
+ }
+
+ @Test
+ public void testIgnoredKeyMeteredOnceOnClose()
+ throws IOException {
+ OpenStructIndexConfig config = JsonUtils.stringToObject(
+ "{\"ignoredKeys\": [\"debug\"]}", OpenStructIndexConfig.class);
+ ServerMetrics metrics = mock(ServerMetrics.class);
+ assertTrue(ServerMetrics.register(metrics), "another ServerMetrics is
already registered");
+ try {
+ try (MutableOpenStructIndex idx = new MutableOpenStructIndex(
+ "metrics", "testTable_REALTIME", openStructSpec(), config, _memMgr,
1000)) {
+ idx.index(0, Map.of("debug", "a"));
+ idx.index(1, Map.of("debug", "b", "clicks", 1L));
+
+ verify(metrics, never()).addMeteredTableValue(anyString(), anyString(),
+ eq(ServerMeter.OPEN_STRUCT_IGNORED_KEY_DROPS), anyLong());
+ }
+
+ verify(metrics, times(1)).addMeteredTableValue("testTable_REALTIME",
"metrics",
+ ServerMeter.OPEN_STRUCT_IGNORED_KEY_DROPS, 2L);
+ } finally {
+ ServerMetrics.deregister();
+ }
+ }
+
+ @Test
+ public void testIgnoredKeyWithNullValueNotMetered()
+ throws IOException {
+ OpenStructIndexConfig config = JsonUtils.stringToObject(
+ "{\"ignoredKeys\": [\"debug\"]}", OpenStructIndexConfig.class);
+ ServerMetrics metrics = mock(ServerMetrics.class);
+ assertTrue(ServerMetrics.register(metrics), "another ServerMetrics is
already registered");
+ try {
+ Map<String, Object> row = new HashMap<>();
+ row.put("debug", null);
+ try (MutableOpenStructIndex idx = new MutableOpenStructIndex(
+ "metrics", "testTable_REALTIME", openStructSpec(), config, _memMgr,
1000)) {
+ idx.index(0, row);
+ }
+
+ verify(metrics, never()).addMeteredTableValue(anyString(), anyString(),
+ eq(ServerMeter.OPEN_STRUCT_IGNORED_KEY_DROPS), anyLong());
+ } finally {
+ ServerMetrics.deregister();
+ }
+ }
}
diff --git
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructIndexTypeTest.java
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructIndexTypeTest.java
index 85f1927334f..2e300de3f65 100644
---
a/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructIndexTypeTest.java
+++
b/pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/openstruct/OpenStructIndexTypeTest.java
@@ -21,11 +21,13 @@ package
org.apache.pinot.segment.local.segment.index.openstruct;
import com.fasterxml.jackson.databind.JsonNode;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.pinot.segment.spi.index.FieldIndexConfigs;
import org.apache.pinot.segment.spi.index.StandardIndexes;
import org.apache.pinot.spi.config.table.FieldConfig;
import org.apache.pinot.spi.config.table.OpenStructIndexConfig;
import org.apache.pinot.spi.data.ComplexFieldSpec;
+import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.utils.JsonUtils;
import org.testng.annotations.Test;
@@ -83,4 +85,73 @@ public class OpenStructIndexTypeTest {
// Must not throw.
StandardIndexes.openStruct().validate(fieldIndexConfigs, openStructSpec,
null);
}
+
+ @Test
+ public void testValidateRejectsIgnoredKeyAlsoDense()
+ throws Exception {
+ OpenStructIndexConfig config = JsonUtils.stringToObject(
+ "{\"denseKeys\": [\"clicks\"], \"ignoredKeys\": [\"clicks\"]}",
OpenStructIndexConfig.class);
+ FieldIndexConfigs fieldIndexConfigs =
+ new FieldIndexConfigs.Builder().add(StandardIndexes.openStruct(),
config).build();
+ FieldSpec openStructSpec = new ComplexFieldSpec("payload",
FieldSpec.DataType.OPEN_STRUCT, true, Map.of());
+
+ assertThrows(IllegalStateException.class,
+ () -> StandardIndexes.openStruct().validate(fieldIndexConfigs,
openStructSpec, null));
+ }
+
+ @Test
+ public void testValidateRejectsIgnoredKeyAlsoHasValueFieldConfig()
+ throws Exception {
+ FieldConfig keyConfig = new FieldConfig.Builder("clicks").build();
+ OpenStructIndexConfig config = new OpenStructIndexConfig(false, null, -1,
null, 0.5, List.of(keyConfig), null,
+ null, Set.of("clicks"));
+ FieldIndexConfigs fieldIndexConfigs =
+ new FieldIndexConfigs.Builder().add(StandardIndexes.openStruct(),
config).build();
+ FieldSpec openStructSpec = new ComplexFieldSpec("payload",
FieldSpec.DataType.OPEN_STRUCT, true, Map.of());
+
+ assertThrows(IllegalStateException.class,
+ () -> StandardIndexes.openStruct().validate(fieldIndexConfigs,
openStructSpec, null));
+ }
+
+ @Test
+ public void testValidateRejectsIgnoredKeyAlsoDeclaredInSchema()
+ throws Exception {
+ OpenStructIndexConfig config = JsonUtils.stringToObject(
+ "{\"ignoredKeys\": [\"clicks\"]}", OpenStructIndexConfig.class);
+ FieldIndexConfigs fieldIndexConfigs =
+ new FieldIndexConfigs.Builder().add(StandardIndexes.openStruct(),
config).build();
+ FieldSpec openStructSpec = new ComplexFieldSpec("payload",
FieldSpec.DataType.OPEN_STRUCT, true,
+ Map.of("clicks", new DimensionFieldSpec("clicks",
FieldSpec.DataType.LONG, true)));
+
+ assertThrows(IllegalStateException.class,
+ () -> StandardIndexes.openStruct().validate(fieldIndexConfigs,
openStructSpec, null));
+ }
+
+ @Test
+ public void testValidateAllowsNonConflictingIgnoredKeys()
+ throws Exception {
+ OpenStructIndexConfig config = JsonUtils.stringToObject(
+ "{\"denseKeys\": [\"clicks\"], \"ignoredKeys\": [\"debug\"]}",
OpenStructIndexConfig.class);
+ FieldIndexConfigs fieldIndexConfigs =
+ new FieldIndexConfigs.Builder().add(StandardIndexes.openStruct(),
config).build();
+ FieldSpec openStructSpec = new ComplexFieldSpec("payload",
FieldSpec.DataType.OPEN_STRUCT, true,
+ Map.of("clicks", new DimensionFieldSpec("clicks",
FieldSpec.DataType.LONG, true)));
+
+ // Must not throw.
+ StandardIndexes.openStruct().validate(fieldIndexConfigs, openStructSpec,
null);
+ }
+
+ @Test
+ public void testValidateSkipsIgnoredKeyChecksWhenIndexDisabled()
+ throws Exception {
+ OpenStructIndexConfig config = JsonUtils.stringToObject(
+ "{\"disabled\": true, \"denseKeys\": [\"clicks\"], \"ignoredKeys\":
[\"clicks\"]}",
+ OpenStructIndexConfig.class);
+ FieldIndexConfigs fieldIndexConfigs =
+ new FieldIndexConfigs.Builder().add(StandardIndexes.openStruct(),
config).build();
+ FieldSpec openStructSpec = new ComplexFieldSpec("payload",
FieldSpec.DataType.OPEN_STRUCT, true, Map.of());
+
+ // Must not throw - validation is skipped entirely when the index is
disabled.
+ StandardIndexes.openStruct().validate(fieldIndexConfigs, openStructSpec,
null);
+ }
}
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/OpenStructIndexConfig.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/OpenStructIndexConfig.java
index 0c09fde132c..1389909e03c 100644
---
a/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/OpenStructIndexConfig.java
+++
b/pinot-spi/src/main/java/org/apache/pinot/spi/config/table/OpenStructIndexConfig.java
@@ -63,6 +63,7 @@ public class OpenStructIndexConfig extends IndexConfig {
private final List<FieldConfig> _valueFieldConfigs;
private final boolean _sparseJsonIndex;
private final boolean _perKeyMetricsEnabled;
+ private final Set<String> _ignoredKeys;
// Eager lookup from key name → FieldConfig for O(1) per-key access. Built
in constructor
// so the config is fully immutable and safe to share across threads.
private final Map<String, FieldConfig> _valueFieldConfigIndex;
@@ -90,6 +91,18 @@ public class OpenStructIndexConfig extends IndexConfig {
sparseJsonIndex, null);
}
+ /// @deprecated Use the 9-arg constructor accepting `ignoredKeys`. Kept for
binary compatibility
+ /// with existing callers built against the pre-`ignoredKeys` signature,
which already shipped
+ /// on master.
+ @Deprecated
+ public OpenStructIndexConfig(Boolean disabled, @Nullable FieldConfig
defaultValueFieldConfig,
+ @Nullable Integer maxDenseKeys, @Nullable Set<String> denseKeys,
@Nullable Double denseKeyMinFillRate,
+ @Nullable List<FieldConfig> valueFieldConfigs, @Nullable Boolean
sparseJsonIndex,
+ @Nullable Boolean perKeyMetricsEnabled) {
+ this(disabled, defaultValueFieldConfig, maxDenseKeys, denseKeys,
denseKeyMinFillRate, valueFieldConfigs,
+ sparseJsonIndex, perKeyMetricsEnabled, null);
+ }
+
@JsonCreator
public OpenStructIndexConfig(
@JsonProperty("disabled") Boolean disabled,
@@ -99,7 +112,8 @@ public class OpenStructIndexConfig extends IndexConfig {
@JsonProperty("denseKeyMinFillRate") @Nullable Double
denseKeyMinFillRate,
@JsonProperty("valueFieldConfigs") @Nullable List<FieldConfig>
valueFieldConfigs,
@JsonProperty("sparseJsonIndex") @Nullable Boolean sparseJsonIndex,
- @JsonProperty("perKeyMetricsEnabled") @Nullable Boolean
perKeyMetricsEnabled) {
+ @JsonProperty("perKeyMetricsEnabled") @Nullable Boolean
perKeyMetricsEnabled,
+ @JsonProperty("ignoredKeys") @Nullable Set<String> ignoredKeys) {
super(disabled);
_defaultValueFieldConfig = defaultValueFieldConfig;
_maxDenseKeys = maxDenseKeys != null ? maxDenseKeys :
DEFAULT_MAX_DENSE_KEYS;
@@ -108,6 +122,7 @@ public class OpenStructIndexConfig extends IndexConfig {
_valueFieldConfigs = valueFieldConfigs;
_sparseJsonIndex = sparseJsonIndex != null && sparseJsonIndex;
_perKeyMetricsEnabled = perKeyMetricsEnabled != null &&
perKeyMetricsEnabled;
+ _ignoredKeys = ignoredKeys;
if (valueFieldConfigs == null || valueFieldConfigs.isEmpty()) {
_valueFieldConfigIndex = Map.of();
} else {
@@ -203,6 +218,19 @@ public class OpenStructIndexConfig extends IndexConfig {
return _perKeyMetricsEnabled;
}
+ /// Keys listed here are dropped entirely at ingestion for this OPEN_STRUCT
column: never
+ /// materialized dense, never written to the sparse `$__sparse__` column,
not queryable. Use for
+ /// keys that shouldn't be persisted at all (e.g. debug/internal fields).
Not retroactive —
+ /// changing this only affects data ingested after the change;
already-sealed segments are
+ /// unaffected.
+ public Set<String> getIgnoredKeys() {
+ return _ignoredKeys != null ? _ignoredKeys : Set.of();
+ }
+
+ public boolean isIgnoredKey(String key) {
+ return _ignoredKeys != null && _ignoredKeys.contains(key);
+ }
+
private static boolean invertedFromIndexes(FieldConfig fieldConfig, String
key) {
JsonNode indexes = fieldConfig.getIndexes();
if (indexes == null || !indexes.isObject()) {
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/config/table/OpenStructIndexConfigTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/config/table/OpenStructIndexConfigTest.java
index d4f26c23702..a0d63754ab0 100644
---
a/pinot-spi/src/test/java/org/apache/pinot/spi/config/table/OpenStructIndexConfigTest.java
+++
b/pinot-spi/src/test/java/org/apache/pinot/spi/config/table/OpenStructIndexConfigTest.java
@@ -301,4 +301,32 @@ public class OpenStructIndexConfigTest {
JsonUtils.stringToObject(JsonUtils.objectToString(config),
OpenStructIndexConfig.class);
assertTrue(reparsed.isSparseJsonIndex());
}
+
+ @Test
+ public void testIgnoredKeysDefaultsToEmpty() {
+ assertTrue(OpenStructIndexConfig.DEFAULT.getIgnoredKeys().isEmpty());
+ assertFalse(OpenStructIndexConfig.DEFAULT.isIgnoredKey("anything"));
+ }
+
+ @Test
+ public void testIgnoredKeysRoundTripsFromJson()
+ throws Exception {
+ String json = "{\"ignoredKeys\": [\"debug_payload\",
\"internal_trace_id\"]}";
+ OpenStructIndexConfig config = JsonUtils.stringToObject(json,
OpenStructIndexConfig.class);
+ assertEquals(config.getIgnoredKeys(), Set.of("debug_payload",
"internal_trace_id"));
+ assertTrue(config.isIgnoredKey("debug_payload"));
+ assertFalse(config.isIgnoredKey("clicks"));
+
+ String reJson = JsonUtils.objectToString(config);
+ OpenStructIndexConfig reDeserialized = JsonUtils.stringToObject(reJson,
OpenStructIndexConfig.class);
+ assertEquals(reDeserialized.getIgnoredKeys(), Set.of("debug_payload",
"internal_trace_id"));
+ }
+
+ @Test
+ @SuppressWarnings("deprecation")
+ public void testDeprecatedSevenArgConstructorDefaultsIgnoredKeysToEmpty() {
+ OpenStructIndexConfig config = new OpenStructIndexConfig(false, null, -1,
null, 0.5, null, true);
+ assertTrue(config.getIgnoredKeys().isEmpty());
+ assertTrue(config.isSparseJsonIndex());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
