This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new a9b5207a632 Add optional fieldId and aliases to FieldSpec (#18833)
a9b5207a632 is described below
commit a9b5207a6326548f84e5d129c7597583cc1ddda9
Author: Navina Ramesh <[email protected]>
AuthorDate: Thu Jun 25 19:43:07 2026 -0700
Add optional fieldId and aliases to FieldSpec (#18833)
Extend FieldSpec with optional fieldId and aliases metadata for stable
column identity. Preserve the fields across manual schema JSON serialization,
including legacy TimeFieldSpec, while omitting unset or empty values and
excluding them from schema backward compatibility checks.
Normalize empty aliases to null so schema equality matches the JSON
contract and no-op schema updates do not churn after round-trip serialization.
Tests cover FieldSpec serde, TimeFieldSpec schema round-trip, omitted
fields, equality/hashCode participation, and backward compatibility.
---
.../java/org/apache/pinot/spi/data/FieldSpec.java | 52 +++++++++-
.../org/apache/pinot/spi/data/TimeFieldSpec.java | 1 +
.../org/apache/pinot/spi/data/FieldSpecTest.java | 108 +++++++++++++++++++++
3 files changed, 159 insertions(+), 2 deletions(-)
diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
index f5548080526..1bff0ff2e82 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/FieldSpec.java
@@ -160,6 +160,16 @@ public abstract class FieldSpec implements
Comparable<FieldSpec>, Serializable {
@Nullable
protected List<String> _tags;
+ @JsonProperty("fieldId")
+ @JsonInclude(JsonInclude.Include.NON_NULL)
+ @Nullable
+ protected Integer _fieldId;
+
+ @JsonProperty("aliases")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ @Nullable
+ protected List<String> _aliases;
+
protected String _name;
protected DataType _dataType;
protected boolean _singleValueField = true;
@@ -240,6 +250,24 @@ public abstract class FieldSpec implements
Comparable<FieldSpec>, Serializable {
_tags = tags;
}
+ @Nullable
+ public Integer getFieldId() {
+ return _fieldId;
+ }
+
+ public void setFieldId(@Nullable Integer fieldId) {
+ _fieldId = fieldId;
+ }
+
+ @Nullable
+ public List<String> getAliases() {
+ return _aliases;
+ }
+
+ public void setAliases(@Nullable List<String> aliases) {
+ _aliases = aliases == null || aliases.isEmpty() ? null : aliases;
+ }
+
public DataType getDataType() {
return _dataType;
}
@@ -585,9 +613,27 @@ public abstract class FieldSpec implements
Comparable<FieldSpec>, Serializable {
}
jsonObject.set("tags", tagsArray);
}
+ appendFieldIdAndAliases(jsonObject);
return jsonObject;
}
+ /// Appends `fieldId` and `aliases` (when set) to the given JSON object.
+ ///
+ /// Subclasses that build JSON without calling [FieldSpec#toJsonObject()],
such as [TimeFieldSpec], use this helper
+ /// to preserve these fields during schema round-trip serialization.
+ protected void appendFieldIdAndAliases(ObjectNode jsonObject) {
+ if (_fieldId != null) {
+ jsonObject.put("fieldId", _fieldId);
+ }
+ if (_aliases != null && !_aliases.isEmpty()) {
+ ArrayNode aliasesArray = JsonUtils.newArrayNode();
+ for (String alias : _aliases) {
+ aliasesArray.add(alias);
+ }
+ jsonObject.set("aliases", aliasesArray);
+ }
+ }
+
protected void appendDefaultNullValue(ObjectNode jsonNode) {
assert _defaultNullValue != null;
String key = "defaultNullValue";
@@ -658,14 +704,16 @@ public abstract class FieldSpec implements
Comparable<FieldSpec>, Serializable {
&& Objects.equals(_transformFunction, that._transformFunction)
&& Objects.equals(_virtualColumnProvider, that._virtualColumnProvider)
&& Objects.equals(_description, that._description)
- && Objects.equals(_tags, that._tags);
+ && Objects.equals(_tags, that._tags)
+ && Objects.equals(_fieldId, that._fieldId)
+ && Objects.equals(_aliases, that._aliases);
}
@Override
public int hashCode() {
return Objects.hash(_name, _dataType, _singleValueField, _notNull,
_maxLength, _maxLengthExceedStrategy,
_allowTrailingZeros, _dataType.hashCode(_defaultNullValue),
_transformFunction, _virtualColumnProvider,
- _description, _tags);
+ _description, _tags, _fieldId, _aliases);
}
/**
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/data/TimeFieldSpec.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/data/TimeFieldSpec.java
index a3cc0074803..3ff1d32cf9a 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/data/TimeFieldSpec.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/data/TimeFieldSpec.java
@@ -114,6 +114,7 @@ public final class TimeFieldSpec extends FieldSpec {
}
appendDefaultNullValue(jsonObject);
appendTransformFunction(jsonObject);
+ appendFieldIdAndAliases(jsonObject);
return jsonObject;
}
diff --git
a/pinot-spi/src/test/java/org/apache/pinot/spi/data/FieldSpecTest.java
b/pinot-spi/src/test/java/org/apache/pinot/spi/data/FieldSpecTest.java
index d0f240fc82a..8c97ad61900 100644
--- a/pinot-spi/src/test/java/org/apache/pinot/spi/data/FieldSpecTest.java
+++ b/pinot-spi/src/test/java/org/apache/pinot/spi/data/FieldSpecTest.java
@@ -592,4 +592,112 @@ public class FieldSpecTest {
assertThat(newSpec.isBackwardCompatibleWith(oldSpec)).isTrue();
}
+
+ @Test
+ public void testFieldIdAndAliasesSerdeRoundtrip()
+ throws Exception {
+ DimensionFieldSpec fieldSpec = new DimensionFieldSpec("user_id", STRING,
true);
+ fieldSpec.setFieldId(42);
+ fieldSpec.setAliases(Arrays.asList("uid", "userId"));
+
+ String json = fieldSpec.toJsonObject().toString();
+ assertThat(json).contains("\"fieldId\":42");
+ assertThat(json).contains("\"aliases\":[\"uid\",\"userId\"]");
+
+ DimensionFieldSpec deserialized = JsonUtils.stringToObject(json,
DimensionFieldSpec.class);
+ assertThat(deserialized.getFieldId()).isEqualTo(42);
+ assertThat(deserialized.getAliases()).isEqualTo(Arrays.asList("uid",
"userId"));
+
+ String jacksonJson = JsonUtils.objectToString(fieldSpec);
+ DimensionFieldSpec fromJackson = JsonUtils.stringToObject(jacksonJson,
DimensionFieldSpec.class);
+ assertThat(fromJackson.getFieldId()).isEqualTo(42);
+ assertThat(fromJackson.getAliases()).isEqualTo(Arrays.asList("uid",
"userId"));
+ }
+
+ @Test
+ public void testFieldIdAndAliasesOmittedWhenNotSet()
+ throws Exception {
+ DimensionFieldSpec fieldSpec = new DimensionFieldSpec("col1", INT, true);
+
+ String json = fieldSpec.toJsonObject().toString();
+ assertThat(json).as("fieldId should be absent when
null").doesNotContain("fieldId");
+ assertThat(json).as("aliases should be absent when
null").doesNotContain("aliases");
+
+ String jacksonJson = JsonUtils.objectToString(fieldSpec);
+ assertThat(jacksonJson).as("fieldId should be absent when
null").doesNotContain("fieldId");
+ assertThat(jacksonJson).as("aliases should be absent when
null").doesNotContain("aliases");
+ }
+
+ @Test
+ public void testEmptyAliasesOmittedFromJson()
+ throws Exception {
+ DimensionFieldSpec fieldSpec = new DimensionFieldSpec("col1", INT, true);
+ fieldSpec.setAliases(List.of());
+ assertThat(fieldSpec.getAliases()).isNull();
+
+ String json = fieldSpec.toJsonObject().toString();
+ assertThat(json).as("empty aliases should be
absent").doesNotContain("aliases");
+ assertThat(JsonUtils.stringToObject(json,
DimensionFieldSpec.class)).isEqualTo(fieldSpec);
+
+ String jacksonJson = JsonUtils.objectToString(fieldSpec);
+ assertThat(jacksonJson).as("empty aliases should be absent
(Jackson)").doesNotContain("aliases");
+ assertThat(JsonUtils.stringToObject(jacksonJson,
DimensionFieldSpec.class)).isEqualTo(fieldSpec);
+ }
+
+ @Test
+ public void testTimeFieldSpecRoundTripsFieldIdAndAliasesThroughSchema()
+ throws Exception {
+ // TimeFieldSpec.toJsonObject() builds its JSON without calling
super.toJsonObject(), so ensure the
+ // generic fieldId/aliases still round-trip through schema
(de)serialization for legacy TIME columns.
+ TimeFieldSpec timeFieldSpec = new TimeFieldSpec(new
TimeGranularitySpec(LONG, TimeUnit.DAYS, "ts"));
+ timeFieldSpec.setFieldId(7);
+ timeFieldSpec.setAliases(Arrays.asList("event_ts", "ingestion_ts"));
+
+ Schema schema = new Schema();
+ schema.setSchemaName("ts_schema");
+ schema.addField(timeFieldSpec);
+
+ String json = schema.toSingleLineJsonString();
+ assertThat(json).contains("\"fieldId\":7");
+ assertThat(json).contains("\"aliases\":[\"event_ts\",\"ingestion_ts\"]");
+
+ FieldSpec deserialized = Schema.fromString(json).getFieldSpecFor("ts");
+ assertThat(deserialized.getFieldId()).isEqualTo(7);
+ assertThat(deserialized.getAliases()).isEqualTo(Arrays.asList("event_ts",
"ingestion_ts"));
+ }
+
+ @Test
+ public void testOldJsonWithoutFieldIdAndAliasesDeserializesCleanly()
+ throws Exception {
+ String oldJson = "{\"name\":\"col1\",\"dataType\":\"STRING\"}";
+ DimensionFieldSpec fieldSpec = JsonUtils.stringToObject(oldJson,
DimensionFieldSpec.class);
+ assertThat(fieldSpec.getFieldId()).isNull();
+ assertThat(fieldSpec.getAliases()).isNull();
+ }
+
+ @Test
+ public void testFieldIdAndAliasesInEqualsAndHashCode() {
+ DimensionFieldSpec spec1 = new DimensionFieldSpec("col1", STRING, true);
+ DimensionFieldSpec spec2 = new DimensionFieldSpec("col1", STRING, true);
+ spec2.setFieldId(42);
+ spec2.setAliases(Arrays.asList("uid"));
+
+ assertThat(spec1).isNotEqualTo(spec2);
+ assertThat(spec1.hashCode()).isNotEqualTo(spec2.hashCode());
+
+ spec1.setFieldId(42);
+ spec1.setAliases(Arrays.asList("uid"));
+ assertThat(spec1).isEqualTo(spec2);
+ assertThat(spec1.hashCode()).isEqualTo(spec2.hashCode());
+ }
+
+ @Test
+ public void testFieldIdAndAliasesNotInBackwardCompatibility() {
+ DimensionFieldSpec oldSpec = new DimensionFieldSpec("col1", STRING, true);
+ DimensionFieldSpec newSpec = new DimensionFieldSpec("col1", STRING, true);
+ newSpec.setFieldId(42);
+ newSpec.setAliases(Arrays.asList("uid"));
+
+ assertThat(newSpec.isBackwardCompatibleWith(oldSpec)).isTrue();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
