This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new c978ad362fa [UUID 8/8] Add UUID integration coverage (#18876)
c978ad362fa is described below
commit c978ad362faabd16cdf831c9630352937453594e
Author: Xiang Fu <[email protected]>
AuthorDate: Tue Aug 18 13:38:17 2026 -0700
[UUID 8/8] Add UUID integration coverage (#18876)
---
.../CustomDataQueryClusterIntegrationTest.java | 2 +-
.../tests/custom/UuidAggregationTest.java | 9 ++
.../tests/custom/UuidUpsertRealtimeTest.java | 134 +++++++++++++++++++++
3 files changed, 144 insertions(+), 1 deletion(-)
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/CustomDataQueryClusterIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/CustomDataQueryClusterIntegrationTest.java
index 21b188837e3..f6096b7a377 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/CustomDataQueryClusterIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/CustomDataQueryClusterIntegrationTest.java
@@ -140,7 +140,7 @@ public abstract class CustomDataQueryClusterIntegrationTest
extends BaseClusterI
if (isRealtimeTable()) {
// In suite mode multiple realtime tests use different topics, so make
sure
// this class-specific topic exists before the controller validates
stream metadata.
- _sharedClusterTestSuite.createKafkaTopic(getKafkaTopic());
+ _sharedClusterTestSuite.createKafkaTopic(getKafkaTopic(),
getNumKafkaPartitions());
waitForKafkaTopicMetadataReadyForConsumer(getKafkaTopic(),
getNumKafkaPartitions());
// create realtime table
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/UuidAggregationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/UuidAggregationTest.java
index 338f94859de..eff7f109d90 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/UuidAggregationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/UuidAggregationTest.java
@@ -148,6 +148,15 @@ public class UuidAggregationTest extends
CustomDataQueryClusterIntegrationTest {
assertCounts(rows.get(0), 2L);
}
+ @Test(dataProvider = "useV2QueryEngine")
+ public void testMultiStageUuidEqualityJoin(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ JsonNode rows = query(String.format("SELECT COUNT(*) FROM %2$s a JOIN %2$s
b ON a.%1$s = b.%3$s",
+ UUID_DICT_SV_COLUMN, getTableName(), UUID_RAW_SV_COLUMN));
+ assertCounts(rows.get(0), 6L);
+ }
+
@Test
public void testDistinctCountOnUuidColumns()
throws Exception {
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/UuidUpsertRealtimeTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/UuidUpsertRealtimeTest.java
new file mode 100644
index 00000000000..f0c4b402295
--- /dev/null
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/UuidUpsertRealtimeTest.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.integration.tests.custom;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.time.Duration;
+import java.util.List;
+import org.apache.avro.Schema.Field;
+import org.apache.avro.Schema.Type;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.pinot.client.ResultSet;
+import org.apache.pinot.spi.config.table.TableConfig;
+import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.UuidUtils;
+import org.apache.pinot.util.TestUtils;
+import org.testng.annotations.Test;
+
+import static org.apache.avro.Schema.create;
+import static org.testng.Assert.assertEquals;
+
+
+/// Verifies realtime UUID ingestion and full-upsert deduplication with a UUID
primary key.
+@Test(suiteName = "CustomClusterIntegrationTest")
+public class UuidUpsertRealtimeTest extends
CustomDataQueryClusterIntegrationTest {
+ private static final String TABLE_NAME = "UuidUpsertRealtimeTest";
+ private static final String UUID_PK_COLUMN = "uuidPk";
+ private static final long BASE_TIME_MS = 1_700_100_000_000L;
+ private static final List<String> UUID_PK_VALUES = List.of(
+ "550e8400-e29b-41d4-a716-446655440000",
+ "550e8400-e29b-41d4-a716-446655440001",
+ "550e8400-e29b-41d4-a716-446655440000");
+
+ @Override
+ public String getTableName() {
+ return TABLE_NAME;
+ }
+
+ @Override
+ public boolean isRealtimeTable() {
+ return true;
+ }
+
+ @Override
+ protected int getNumKafkaPartitions() {
+ return 1;
+ }
+
+ @Override
+ protected void waitForAllDocsLoaded(long timeoutMs) {
+ TestUtils.waitForCondition(() -> getPinotConnection().execute(
+ "SELECT COUNT(*) FROM " + TABLE_NAME + "
OPTION(skipUpsert=true)").getResultSet(0).getLong(0)
+ == UUID_PK_VALUES.size(),
+ 100L, timeoutMs, "Failed to load all UUID upsert records",
Duration.ofSeconds(5));
+ }
+
+ @Override
+ public int getNumAvroFiles() {
+ return 1;
+ }
+
+ @Override
+ protected int getRealtimeSegmentFlushSize() {
+ return 2;
+ }
+
+ @Override
+ protected TableConfig createRealtimeTableConfig(File sampleAvroFile) {
+ return createUpsertTableConfig(sampleAvroFile, UUID_PK_COLUMN, null,
getNumKafkaPartitions());
+ }
+
+ @Override
+ public Schema createSchema() {
+ return new Schema.SchemaBuilder().setSchemaName(getTableName())
+ .addSingleValueDimension(UUID_PK_COLUMN, DataType.UUID)
+ .addDateTimeField(getTimeColumnName(), DataType.LONG,
"1:MILLISECONDS:EPOCH", "1:MILLISECONDS")
+ .setPrimaryKeyColumns(List.of(UUID_PK_COLUMN))
+ .build();
+ }
+
+ @Override
+ public List<File> createAvroFiles()
+ throws Exception {
+ org.apache.avro.Schema avroSchema =
org.apache.avro.Schema.createRecord("uuidUpsertRecord", null, null, false);
+ avroSchema.setFields(List.of(
+ new Field(UUID_PK_COLUMN, create(Type.BYTES), null, null),
+ new Field(getTimeColumnName(), create(Type.LONG), null, null)));
+
+ try (AvroFilesAndWriters avroFilesAndWriters =
createAvroFilesAndWriters(avroSchema)) {
+ DataFileWriter<GenericData.Record> writer =
avroFilesAndWriters.getWriters().get(0);
+ for (int i = 0; i < UUID_PK_VALUES.size(); i++) {
+ GenericData.Record record = new GenericData.Record(avroSchema);
+ record.put(UUID_PK_COLUMN,
ByteBuffer.wrap(UuidUtils.toBytes(UUID_PK_VALUES.get(i))));
+ record.put(getTimeColumnName(), BASE_TIME_MS + i);
+ writer.append(record);
+ }
+ return avroFilesAndWriters.getAvroFiles();
+ }
+ }
+
+ @Test
+ public void testUuidPrimaryKeyUpsert()
+ throws Exception {
+ ResultSet resultSet = getPinotConnection().execute(
+ "SELECT " + UUID_PK_COLUMN + ", " + getTimeColumnName() + " FROM " +
getTableName()
+ + " ORDER BY " + UUID_PK_COLUMN).getResultSet(0);
+ assertEquals(resultSet.getRowCount(), 2);
+ assertRow(resultSet, 0, UUID_PK_VALUES.get(0), BASE_TIME_MS + 2);
+ assertRow(resultSet, 1, UUID_PK_VALUES.get(1), BASE_TIME_MS + 1);
+ }
+
+ private static void assertRow(ResultSet resultSet, int row, String uuidPk,
long timestamp) {
+ assertEquals(resultSet.getString(row, 0), uuidPk);
+ assertEquals(resultSet.getLong(row, 1), timestamp);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
