Copilot commented on code in PR #18687:
URL: https://github.com/apache/pinot/pull/18687#discussion_r3360192257
##########
pinot-plugins/pinot-stream-ingestion/pinot-kafka-3.0/src/test/java/org/apache/pinot/plugin/stream/kafka30/KafkaPartitionLevelConsumerTest.java:
##########
@@ -282,6 +283,39 @@ public void testSubsetPartitionConsumption()
}
}
+ @Test
+ public void testComputePartitionGroupMetadataUsesKafkaPartitionIds()
+ throws Exception {
+ String streamType = "kafka";
+ String streamKafkaBrokerList = _kafkaBrokerAddress;
+ String clientId = "partition-id-client";
+ String tableNameWithType = "tableName_REALTIME";
+
+ Map<String, String> streamConfigMap = new HashMap<>();
+ streamConfigMap.put("streamType", streamType);
+ streamConfigMap.put("stream.kafka.topic.name",
TEST_TOPIC_SUBSET_PARTITION);
+ streamConfigMap.put("stream.kafka.broker.list", streamKafkaBrokerList);
+ streamConfigMap.put("stream.kafka.consumer.factory.class.name",
getKafkaConsumerFactoryName());
+ streamConfigMap.put("stream.kafka.decoder.class.name", "decoderClass");
+ StreamConfig streamConfig = new StreamConfig(tableNameWithType,
streamConfigMap);
+
+ List<PartitionGroupConsumptionStatus> currentStatuses = List.of(
+ new PartitionGroupConsumptionStatus(0, 0, new LongMsgOffset(0), new
LongMsgOffset(10), "DONE"),
+ new PartitionGroupConsumptionStatus(1, 1, new LongMsgOffset(0), new
LongMsgOffset(10), "DONE"),
+ new PartitionGroupConsumptionStatus(3, 3, new LongMsgOffset(0), new
LongMsgOffset(10), "DONE"),
+ new PartitionGroupConsumptionStatus(4, 4, new LongMsgOffset(0), new
LongMsgOffset(10), "DONE"),
+ new PartitionGroupConsumptionStatus(5, 5, new LongMsgOffset(0), new
LongMsgOffset(10), "DONE"),
+ new PartitionGroupConsumptionStatus(6, 6, new LongMsgOffset(0), new
LongMsgOffset(10), "DONE"),
+ new PartitionGroupConsumptionStatus(7, 7, new LongMsgOffset(0), new
LongMsgOffset(10), "DONE"));
+
+ KafkaStreamMetadataProvider streamMetadataProvider = new
KafkaStreamMetadataProvider(clientId, streamConfig);
+ List<PartitionGroupMetadata> partitionGroupMetadataList =
+ streamMetadataProvider.computePartitionGroupMetadata(clientId,
streamConfig, currentStatuses, 10000);
+
+
assertEquals(partitionGroupMetadataList.stream().map(PartitionGroupMetadata::getPartitionGroupId)
+ .collect(Collectors.toList()), List.of(0, 1, 2, 3, 4, 5, 6, 7));
+ }
Review Comment:
`KafkaStreamMetadataProvider` is `Closeable` (implements
`StreamMetadataProvider`), but this new test doesn't close it. Please wrap it
in try-with-resources to avoid leaking Kafka consumer/admin-client resources
between tests.
##########
pinot-plugins/pinot-stream-ingestion/pinot-kafka-4.0/src/test/java/org/apache/pinot/plugin/stream/kafka40/KafkaPartitionLevelConsumerTest.java:
##########
@@ -286,6 +287,39 @@ public void testSubsetPartitionConsumption() throws
TimeoutException {
}
}
+ @Test
+ public void testComputePartitionGroupMetadataUsesKafkaPartitionIds()
+ throws Exception {
+ String streamType = "kafka";
+ String streamKafkaBrokerList = _kafkaBrokerAddress;
+ String clientId = "partition-id-client";
+ String tableNameWithType = "tableName_REALTIME";
+
+ Map<String, String> streamConfigMap = new HashMap<>();
+ streamConfigMap.put("streamType", streamType);
+ streamConfigMap.put("stream.kafka.topic.name",
TEST_TOPIC_SUBSET_PARTITION);
+ streamConfigMap.put("stream.kafka.broker.list", streamKafkaBrokerList);
+ streamConfigMap.put("stream.kafka.consumer.factory.class.name",
getKafkaConsumerFactoryName());
+ streamConfigMap.put("stream.kafka.decoder.class.name", "decoderClass");
+ StreamConfig streamConfig = new StreamConfig(tableNameWithType,
streamConfigMap);
+
+ List<PartitionGroupConsumptionStatus> currentStatuses = List.of(
+ new PartitionGroupConsumptionStatus(0, 0, new LongMsgOffset(0), new
LongMsgOffset(10), "DONE"),
+ new PartitionGroupConsumptionStatus(1, 1, new LongMsgOffset(0), new
LongMsgOffset(10), "DONE"),
+ new PartitionGroupConsumptionStatus(3, 3, new LongMsgOffset(0), new
LongMsgOffset(10), "DONE"),
+ new PartitionGroupConsumptionStatus(4, 4, new LongMsgOffset(0), new
LongMsgOffset(10), "DONE"),
+ new PartitionGroupConsumptionStatus(5, 5, new LongMsgOffset(0), new
LongMsgOffset(10), "DONE"),
+ new PartitionGroupConsumptionStatus(6, 6, new LongMsgOffset(0), new
LongMsgOffset(10), "DONE"),
+ new PartitionGroupConsumptionStatus(7, 7, new LongMsgOffset(0), new
LongMsgOffset(10), "DONE"));
+
+ KafkaStreamMetadataProvider streamMetadataProvider = new
KafkaStreamMetadataProvider(clientId, streamConfig);
+ List<PartitionGroupMetadata> partitionGroupMetadataList =
+ streamMetadataProvider.computePartitionGroupMetadata(clientId,
streamConfig, currentStatuses, 10000);
+
+
assertEquals(partitionGroupMetadataList.stream().map(PartitionGroupMetadata::getPartitionGroupId)
+ .collect(Collectors.toList()), List.of(0, 1, 2, 3, 4, 5, 6, 7));
+ }
Review Comment:
`KafkaStreamMetadataProvider` implements `Closeable` (via
`StreamMetadataProvider`), but the new test creates one without closing it.
This can leak embedded Kafka resources across tests and cause flakiness; use
try-with-resources here (as the new `KafkaStreamMetadataProviderTest` does).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
