This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new dd0a9f37d1 [test] add tests for mixed usage of dv (#5567)
dd0a9f37d1 is described below
commit dd0a9f37d1c1e940c351db50bfc0e3adb670e3ee
Author: LsomeYeah <[email protected]>
AuthorDate: Wed May 7 11:45:42 2025 +0800
[test] add tests for mixed usage of dv (#5567)
---
.../DeletionVectorsIndexFileTest.java | 27 --------
.../DeletionVectorsMaintainerTest.java | 74 ++++++++++++++++++++++
.../apache/paimon/flink/DeletionVectorITCase.java | 42 ++++++++++--
3 files changed, 111 insertions(+), 32 deletions(-)
diff --git
a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
index da34e521b8..2b55cba306 100644
---
a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsIndexFileTest.java
@@ -297,20 +297,6 @@ public class DeletionVectorsIndexFileTest {
@Test
public void testReadOldDeletionVector32Bit() throws IOException {
- // write
- // LocalFileIO fileIO = LocalFileIO.create();
- // try (DeletionFileWriter writer = new DeletionFileWriter(new
- // Path("/tmp/dvindex-32"), fileIO)) {
- // BitmapDeletionVector vector1 = new
BitmapDeletionVector();
- // vector1.delete(2);
- // vector1.delete(3);
- // writer.write("file2.parquet", vector1);
- //
- // BitmapDeletionVector vector2 = new
BitmapDeletionVector();
- // vector2.delete(1);
- // writer.write("file1.parquet", vector2);
- // }
-
try (InputStream inputStream =
DeletionVectorsIndexFile.class
.getClassLoader()
@@ -332,19 +318,6 @@ public class DeletionVectorsIndexFileTest {
@Test
public void testReadOldDeletionVector64Bit() throws IOException {
- // write
- // LocalFileIO fileIO = LocalFileIO.create();
- // try (DeletionFileWriter writer = new DeletionFileWriter(new
- // Path("/tmp/dvindex-64"), fileIO)) {
- // Bitmap64DeletionVector vector1 = new
Bitmap64DeletionVector();
- // vector1.delete(2);
- // vector1.delete(3);
- // writer.write("file2.parquet", vector1);
- //
- // Bitmap64DeletionVector vector2 = new
Bitmap64DeletionVector();
- // vector2.delete(1);
- // writer.write("file1.parquet", vector2);
- // }
try (InputStream inputStream =
DeletionVectorsIndexFile.class
.getClassLoader()
diff --git
a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsMaintainerTest.java
b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsMaintainerTest.java
index cbb44d5eb5..e418350fa1 100644
---
a/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsMaintainerTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/deletionvectors/DeletionVectorsMaintainerTest.java
@@ -178,6 +178,80 @@ public class DeletionVectorsMaintainerTest extends
PrimaryKeyTableTestBase {
assertThat(indexDir.listFiles()).hasSize(1);
}
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testReadAndWriteMixedDv(boolean bitmap64) {
+ // write first kind dv
+ initIndexHandler(bitmap64);
+ DeletionVectorsMaintainer.Factory factory1 =
+ new DeletionVectorsMaintainer.Factory(fileHandler);
+ DeletionVectorsMaintainer dvMaintainer1 = factory1.create();
+ dvMaintainer1.notifyNewDeletion("f1", 1);
+ dvMaintainer1.notifyNewDeletion("f1", 3);
+ dvMaintainer1.notifyNewDeletion("f2", 1);
+ dvMaintainer1.notifyNewDeletion("f2", 3);
+ assertThat(dvMaintainer1.bitmap64()).isEqualTo(bitmap64);
+
+ List<IndexFileMeta> fileMetas1 =
dvMaintainer1.writeDeletionVectorsIndex();
+ assertThat(fileMetas1.size()).isEqualTo(1);
+ CommitMessage commitMessage1 =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 0,
+ 1,
+ DataIncrement.emptyIncrement(),
+ CompactIncrement.emptyIncrement(),
+ new IndexIncrement(fileMetas1));
+ BatchTableCommit commit1 = table.newBatchWriteBuilder().newCommit();
+ commit1.commit(Collections.singletonList(commitMessage1));
+
+ // write second kind dv
+ initIndexHandler(!bitmap64);
+ DeletionVectorsMaintainer.Factory factory2 =
+ new DeletionVectorsMaintainer.Factory(fileHandler);
+ DeletionVectorsMaintainer dvMaintainer2 =
+ factory2.createOrRestore(table.latestSnapshot().get(),
BinaryRow.EMPTY_ROW, 0);
+ dvMaintainer2.notifyNewDeletion("f1", 10);
+ dvMaintainer2.notifyNewDeletion("f3", 1);
+ dvMaintainer2.notifyNewDeletion("f3", 3);
+ assertThat(dvMaintainer2.bitmap64()).isEqualTo(!bitmap64);
+
+ // verify two kinds of dv can exist in the same dv maintainer
+ Map<String, DeletionVector> dvs = dvMaintainer2.deletionVectors();
+ assertThat(dvs.size()).isEqualTo(3);
+ assertThat(dvs.get("f1").getCardinality()).isEqualTo(3);
+ assertThat(dvs.get("f2"))
+ .isInstanceOf(bitmap64 ? Bitmap64DeletionVector.class :
BitmapDeletionVector.class);
+ assertThat(dvs.get("f3"))
+ .isInstanceOf(bitmap64 ? BitmapDeletionVector.class :
Bitmap64DeletionVector.class);
+
+ List<IndexFileMeta> fileMetas2 =
dvMaintainer2.writeDeletionVectorsIndex();
+ assertThat(fileMetas2.size()).isEqualTo(1);
+ CommitMessage commitMessage2 =
+ new CommitMessageImpl(
+ BinaryRow.EMPTY_ROW,
+ 0,
+ 1,
+ DataIncrement.emptyIncrement(),
+ CompactIncrement.emptyIncrement(),
+ new IndexIncrement(fileMetas2));
+ BatchTableCommit commit2 = table.newBatchWriteBuilder().newCommit();
+ commit2.commit(Collections.singletonList(commitMessage2));
+
+ // test read dv index file which contains two kinds of dv
+ Map<String, DeletionVector> readDvs =
+ fileHandler.readAllDeletionVectors(
+ fileHandler.scan(
+ table.latestSnapshot().get(),
+ "DELETION_VECTORS",
+ BinaryRow.EMPTY_ROW,
+ 0));
+ assertThat(readDvs.size()).isEqualTo(3);
+ assertThat(dvs.get("f1").getCardinality()).isEqualTo(3);
+ assertThat(dvs.get("f2").getCardinality()).isEqualTo(2);
+ assertThat(dvs.get("f3").getCardinality()).isEqualTo(2);
+ }
+
private DeletionVector createDeletionVector(boolean bitmap64) {
return bitmap64 ? new Bitmap64DeletionVector() : new
BitmapDeletionVector();
}
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DeletionVectorITCase.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DeletionVectorITCase.java
index e51b617db2..1e31f79f70 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DeletionVectorITCase.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DeletionVectorITCase.java
@@ -22,6 +22,7 @@ import org.apache.paimon.utils.BlockingIterator;
import org.apache.flink.types.Row;
import org.apache.flink.types.RowKind;
+import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
@@ -45,18 +46,19 @@ public class DeletionVectorITCase extends CatalogITCaseBase
{
private static Stream<Arguments> parameters2() {
// parameters: changelogProducer, dvVersion
- return Stream.of(Arguments.of("input", 1), Arguments.of("input", 2));
+ return Stream.of(Arguments.of("input", true), Arguments.of("input",
false));
}
@ParameterizedTest
@MethodSource("parameters2")
- public void testStreamingReadDVTableWhenChangelogProducerIsInput(String
changelogProducer)
- throws Exception {
+ public void testStreamingReadDVTableWhenChangelogProducerIsInput(
+ String changelogProducer, boolean dvBitmap64) throws Exception {
sql(
String.format(
"CREATE TABLE T (id INT PRIMARY KEY NOT ENFORCED, name
STRING) "
- + "WITH ('deletion-vectors.enabled' = 'true',
'changelog-producer' = '%s')",
- changelogProducer));
+ + "WITH ('deletion-vectors.enabled' = 'true',
'changelog-producer' = '%s', "
+ + "'deletion-vectors.bitmap64' = '%s')",
+ changelogProducer, dvBitmap64));
sql("INSERT INTO T VALUES (1, '111111111'), (2, '2'), (3, '3'), (4,
'4')");
@@ -307,4 +309,34 @@ public class DeletionVectorITCase extends
CatalogITCaseBase {
assertThat(batchSql("SELECT * FROM T /*+
OPTIONS('scan.tag-name'='my_tag') */"))
.containsExactlyInAnyOrder(Row.of(1, "1"), Row.of(2, "2"));
}
+
+ @Test
+ public void testChangeToDv64() throws Exception {
+ sql(
+ "CREATE TABLE T (id INT PRIMARY KEY NOT ENFORCED, name STRING)
"
+ + "WITH ('deletion-vectors.enabled' = 'true',
'changelog-producer' = 'lookup', "
+ + "'deletion-vectors.bitmap64' = 'false', 'bucket' =
'1')");
+
+ sql("INSERT INTO T VALUES (1, '1'), (2, '2'), (3, '3'), (4, '4')");
+ sql("INSERT INTO T VALUES (2, '2_1'), (3, '3_1')");
+ sql("INSERT INTO T VALUES (5, '5'), (6, '6'), (7, '8')");
+
+ // change dv to bitmap64
+ sql("ALTER TABLE T SET('deletion-vectors.bitmap64' = 'true')");
+ sql("INSERT INTO T VALUES (2, '2_2'),(6, '6_1'), (7, '7_1')");
+
+ assertThat(batchSql("SELECT * FROM T"))
+ .containsExactlyInAnyOrder(
+ Row.of(1, "1"),
+ Row.of(2, "2_2"),
+ Row.of(3, "3_1"),
+ Row.of(4, "4"),
+ Row.of(5, "5"),
+ Row.of(6, "6_1"),
+ Row.of(7, "7_1"));
+
+ assertThat(batchSql("SELECT * FROM T /*+
OPTIONS('scan.snapshot-id'='4') */"))
+ .containsExactlyInAnyOrder(
+ Row.of(1, "1"), Row.of(2, "2_1"), Row.of(3, "3_1"),
Row.of(4, "4"));
+ }
}
