This is an automated email from the ASF dual-hosted git repository.
ankitsultana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c664074efe Add segmentCreationTimeMillis in validDocIdsMetadata
(#15938)
c664074efe is described below
commit c664074efe917fffa646e06161e0c572a0daa382
Author: tarun11Mavani <[email protected]>
AuthorDate: Fri May 30 23:09:12 2025 +0530
Add segmentCreationTimeMillis in validDocIdsMetadata (#15938)
---
.../restlet/resources/ValidDocIdsMetadataInfo.java | 9 +++-
.../UpsertCompactionTaskGeneratorTest.java | 49 ++++++++++++----------
.../pinot/server/api/resources/TablesResource.java | 1 +
.../pinot/server/api/TablesResourceTest.java | 2 +
4 files changed, 39 insertions(+), 22 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ValidDocIdsMetadataInfo.java
b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ValidDocIdsMetadataInfo.java
index 500cfff946..1038d45f55 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ValidDocIdsMetadataInfo.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/restlet/resources/ValidDocIdsMetadataInfo.java
@@ -31,12 +31,14 @@ public class ValidDocIdsMetadataInfo {
private final String _segmentCrc;
private final ValidDocIdsType _validDocIdsType;
private final long _segmentSizeInBytes;
+ private final long _segmentCreationTimeMillis;
public ValidDocIdsMetadataInfo(@JsonProperty("segmentName") String
segmentName,
@JsonProperty("totalValidDocs") long totalValidDocs,
@JsonProperty("totalInvalidDocs") long totalInvalidDocs,
@JsonProperty("totalDocs") long totalDocs, @JsonProperty("segmentCrc")
String segmentCrc,
@JsonProperty("validDocIdsType") ValidDocIdsType validDocIdsType,
- @JsonProperty("segmentSizeInBytes") long segmentSizeInBytes) {
+ @JsonProperty("segmentSizeInBytes") long segmentSizeInBytes,
+ @JsonProperty("segmentCreationTimeMillis") long
segmentCreationTimeMillis) {
_segmentName = segmentName;
_totalValidDocs = totalValidDocs;
_totalInvalidDocs = totalInvalidDocs;
@@ -44,6 +46,7 @@ public class ValidDocIdsMetadataInfo {
_segmentCrc = segmentCrc;
_validDocIdsType = validDocIdsType;
_segmentSizeInBytes = segmentSizeInBytes;
+ _segmentCreationTimeMillis = segmentCreationTimeMillis;
}
public String getSegmentName() {
@@ -73,4 +76,8 @@ public class ValidDocIdsMetadataInfo {
public long getSegmentSizeInBytes() {
return _segmentSizeInBytes;
}
+
+ public long getSegmentCreationTimeMillis() {
+ return _segmentCreationTimeMillis;
+ }
}
diff --git
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGeneratorTest.java
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGeneratorTest.java
index 2cf90a8ff8..5436487e54 100644
---
a/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGeneratorTest.java
+++
b/pinot-plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/src/test/java/org/apache/pinot/plugin/minion/tasks/upsertcompaction/UpsertCompactionTaskGeneratorTest.java
@@ -221,9 +221,11 @@ public class UpsertCompactionTaskGeneratorTest {
throws IOException {
Map<String, String> compactionConfigs = getCompactionConfigs("1", "10");
String json = "{\"testTable__0\": [{\"totalValidDocs\": 50,
\"totalInvalidDocs\": 50, "
- + "\"segmentName\": \"testTable__0\", \"totalDocs\": 100,
\"segmentCrc\": \"1000\"}], "
+ + "\"segmentName\": \"testTable__0\", \"totalDocs\": 100,
\"segmentCrc\": \"1000\", "
+ + "\"segmentCreationTimeMillis\": 1234567890}], "
+ "\"testTable__1\": [{\"totalValidDocs\": 0, "
- + "\"totalInvalidDocs\": 10, \"segmentName\": \"testTable__1\",
\"totalDocs\": 10, \"segmentCrc\": \"2000\"}]}";
+ + "\"totalInvalidDocs\": 10, \"segmentName\": \"testTable__1\",
\"totalDocs\": 10, \"segmentCrc\": \"2000\", "
+ + "\"segmentCreationTimeMillis\": 9876543210}]}";
Map<String, List<ValidDocIdsMetadataInfo>> validDocIdsMetadataInfo =
JsonUtils.stringToObject(json, new TypeReference<>() {
@@ -279,9 +281,10 @@ public class UpsertCompactionTaskGeneratorTest {
// Test the case where the completedSegment from api has different crc
than segment from zk metadata.
json = "{\"" + _completedSegment.getSegmentName() + "\":
[{\"totalValidDocs\": 50, \"totalInvalidDocs\": 50, "
+ "\"segmentName\": \"" + _completedSegment.getSegmentName() + "\",
\"totalDocs\": 100, \"segmentCrc\": "
- + "\"1234567890\"}], \"" + _completedSegment2.getSegmentName() + "\":
[{\"totalValidDocs\": 0, "
- + "\"totalInvalidDocs\": 10, \"segmentName\": \"" +
_completedSegment2.getSegmentName() + "\", "
- + "\"segmentCrc\": \"" + _completedSegment2.getCrc() + "\",
\"totalDocs\": 10}]}";
+ + "\"1234567890\", \"segmentCreationTimeMillis\": 1111111111}], \"" +
_completedSegment2.getSegmentName()
+ + "\": [{\"totalValidDocs\": 0, " + "\"totalInvalidDocs\": 10,
\"segmentName\": \""
+ + _completedSegment2.getSegmentName() + "\", " + "\"segmentCrc\": \""
+ _completedSegment2.getCrc()
+ + "\", \"totalDocs\": 10, \"segmentCreationTimeMillis\":
2222222222}]}";
validDocIdsMetadataInfo = JsonUtils.stringToObject(json, new
TypeReference<>() {
});
segmentSelectionResult =
@@ -293,16 +296,16 @@ public class UpsertCompactionTaskGeneratorTest {
// completedSegment2 is still supposed to be deleted
Assert.assertEquals(segmentSelectionResult.getSegmentsForDeletion().size(), 1);
- assertEquals(segmentSelectionResult.getSegmentsForDeletion().get(0),
- _completedSegment2.getSegmentName());
+ assertEquals(segmentSelectionResult.getSegmentsForDeletion().get(0),
_completedSegment2.getSegmentName());
// check if both the candidates for compaction are coming in sorted
descending order
json = "{\"" + _completedSegment.getSegmentName() + "\":
[{\"totalValidDocs\": 50, \"totalInvalidDocs\": 50, "
+ "\"segmentName\": \"" + _completedSegment.getSegmentName() + "\",
\"totalDocs\": 100, \"segmentCrc\": \""
- + _completedSegment.getCrc() + "\"}], \"" +
_completedSegment2.getSegmentName() + "\": "
+ + _completedSegment.getCrc() + "\", \"segmentCreationTimeMillis\":
1234567890}], \""
+ + _completedSegment2.getSegmentName() + "\": "
+ "[{\"totalValidDocs\": 10, \"totalInvalidDocs\": 40,
\"segmentName\": \""
+ _completedSegment2.getSegmentName() + "\", \"segmentCrc\": \"" +
_completedSegment2.getCrc() + "\", "
- + "\"totalDocs\": 50}]}";
+ + "\"totalDocs\": 50, \"segmentCreationTimeMillis\": 9876543210}]}";
validDocIdsMetadataInfo = JsonUtils.stringToObject(json, new
TypeReference<>() {
});
compactionConfigs = getCompactionConfigs("30", "0");
@@ -315,6 +318,10 @@ public class UpsertCompactionTaskGeneratorTest {
_completedSegment.getSegmentName());
assertEquals(segmentSelectionResult.getSegmentsForCompaction().get(1).getSegmentName(),
_completedSegment2.getSegmentName());
+
+ // Check segmentCreationTimeMillis is deserialized correctly
+
assertEquals(validDocIdsMetadataInfo.get("testTable__0").get(0).getSegmentCreationTimeMillis(),
1234567890L);
+
assertEquals(validDocIdsMetadataInfo.get("testTable__1").get(0).getSegmentCreationTimeMillis(),
9876543210L);
}
@Test
@@ -324,27 +331,27 @@ public class UpsertCompactionTaskGeneratorTest {
"1");
UpsertConfig upsertConfig = new UpsertConfig(UpsertConfig.Mode.FULL);
upsertConfig.setSnapshot(Enablement.ENABLE);
- TableConfig tableConfig = new
TableConfigBuilder(TableType.REALTIME).setTableName(RAW_TABLE_NAME)
- .setUpsertConfig(upsertConfig)
- .setTaskConfig(new
TableTaskConfig(ImmutableMap.of("UpsertCompactionTask",
upsertCompactionTaskConfig)))
- .build();
+ TableConfig tableConfig =
+ new
TableConfigBuilder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setUpsertConfig(upsertConfig)
+ .setTaskConfig(new
TableTaskConfig(ImmutableMap.of("UpsertCompactionTask",
upsertCompactionTaskConfig)))
+ .build();
_taskGenerator.validateTaskConfigs(tableConfig, new Schema(),
upsertCompactionTaskConfig);
// test with invalidRecordsThresholdPercents as 0
Map<String, String> upsertCompactionTaskConfig1 =
ImmutableMap.of("invalidRecordsThresholdPercent", "0");
- TableConfig zeroPercentTableConfig = new
TableConfigBuilder(TableType.REALTIME).setTableName(RAW_TABLE_NAME)
- .setUpsertConfig(upsertConfig)
- .setTaskConfig(new
TableTaskConfig(ImmutableMap.of("UpsertCompactionTask",
upsertCompactionTaskConfig1)))
- .build();
+ TableConfig zeroPercentTableConfig =
+ new
TableConfigBuilder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setUpsertConfig(upsertConfig)
+ .setTaskConfig(new
TableTaskConfig(ImmutableMap.of("UpsertCompactionTask",
upsertCompactionTaskConfig1)))
+ .build();
_taskGenerator.validateTaskConfigs(zeroPercentTableConfig, new Schema(),
upsertCompactionTaskConfig1);
// test with invalid invalidRecordsThresholdPercents as -1 and 110
Map<String, String> upsertCompactionTaskConfig2 =
ImmutableMap.of("invalidRecordsThresholdPercent", "-1");
- TableConfig negativePercentTableConfig = new
TableConfigBuilder(TableType.REALTIME).setTableName(RAW_TABLE_NAME)
- .setUpsertConfig(upsertConfig)
- .setTaskConfig(new
TableTaskConfig(ImmutableMap.of("UpsertCompactionTask",
upsertCompactionTaskConfig2)))
- .build();
+ TableConfig negativePercentTableConfig =
+ new
TableConfigBuilder(TableType.REALTIME).setTableName(RAW_TABLE_NAME).setUpsertConfig(upsertConfig)
+ .setTaskConfig(new
TableTaskConfig(ImmutableMap.of("UpsertCompactionTask",
upsertCompactionTaskConfig2)))
+ .build();
Assert.assertThrows(IllegalStateException.class,
() -> _taskGenerator.validateTaskConfigs(negativePercentTableConfig,
new Schema(),
upsertCompactionTaskConfig2));
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
index 3df7a4290e..da822cef86 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
@@ -718,6 +718,7 @@ public class TablesResource {
validDocIdsMetadata.put("segmentSizeInBytes",
((ImmutableSegment)
segmentDataManager.getSegment()).getSegmentSizeBytes());
}
+ validDocIdsMetadata.put("segmentCreationTimeMillis",
indexSegment.getSegmentMetadata().getIndexCreationTime());
allValidDocIdsMetadata.add(validDocIdsMetadata);
}
if (nonImmutableSegmentCount > 0) {
diff --git
a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
index 42699a78c0..0d208cc695 100644
---
a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
+++
b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
@@ -348,6 +348,8 @@ public class TablesResourceTest extends BaseResourceTest {
Assert.assertEquals(validDocIdsMetadata.get("segmentCrc").asText(),
"1894900283");
Assert.assertEquals(validDocIdsMetadata.get("validDocIdsType").asText(),
"SNAPSHOT");
Assert.assertEquals(validDocIdsMetadata.get("segmentSizeInBytes").asLong(),
1877636);
+ Assert.assertTrue(validDocIdsMetadata.has("segmentCreationTimeMillis"));
+
Assert.assertTrue(validDocIdsMetadata.get("segmentCreationTimeMillis").asLong()
> 0);
}
// Verify metadata file from segments.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
