This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 1baa494444 [core][doc] Document options for 'incremental-to-auto-tag'
(#5465)
1baa494444 is described below
commit 1baa49444461e21d18c12de610e1c7957f9c07ab
Author: Jingsong Lee <[email protected]>
AuthorDate: Mon Apr 14 20:01:45 2025 +0800
[core][doc] Document options for 'incremental-to-auto-tag' (#5465)
---
.../shortcodes/generated/core_configuration.html | 2 +-
.../main/java/org/apache/paimon/CoreOptions.java | 3 +-
.../apache/paimon/table/IncrementalTableTest.java | 49 +++++++++++++++++++++-
3 files changed, 51 insertions(+), 3 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html
b/docs/layouts/shortcodes/generated/core_configuration.html
index 665836462c..b490859717 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -444,7 +444,7 @@ under the License.
<td><h5>incremental-to-auto-tag</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>String</td>
- <td>Used to specify the end tag (inclusive), and Paimon will find
an earlier tag and return changes between them. If the tag doesn't exist or the
earlier tag doesn't exist, return empty. </td>
+ <td>Used to specify the end tag (inclusive), and Paimon will find
an earlier tag and return changes between them. If the tag doesn't exist or the
earlier tag doesn't exist, return empty. This option requires
'tag.creation-period' and 'tag.period-formatter' configured.</td>
</tr>
<tr>
<td><h5>local-merge-buffer-size</h5></td>
diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
index bd671664b8..cf5b417cdf 100644
--- a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
@@ -1147,7 +1147,8 @@ public class CoreOptions implements Serializable {
.noDefaultValue()
.withDescription(
"Used to specify the end tag (inclusive), and
Paimon will find an earlier tag and return changes between them. "
- + "If the tag doesn't exist or the earlier
tag doesn't exist, return empty. ");
+ + "If the tag doesn't exist or the earlier
tag doesn't exist, return empty. "
+ + "This option requires
'tag.creation-period' and 'tag.period-formatter' configured.");
public static final ConfigOption<Boolean> END_INPUT_CHECK_PARTITION_EXPIRE
=
key("end-input.check-partition-expire")
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/IncrementalTableTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/IncrementalTableTest.java
index d3d6aa6ad9..2bc33cb7b3 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/IncrementalTableTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/IncrementalTableTest.java
@@ -27,6 +27,9 @@ import org.apache.paimon.data.Timestamp;
import org.apache.paimon.manifest.ManifestCommittable;
import org.apache.paimon.options.ExpireConfig;
import org.apache.paimon.schema.Schema;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
import org.apache.paimon.table.sink.CommitMessage;
import org.apache.paimon.table.sink.TableCommitImpl;
import org.apache.paimon.table.sink.TableWriteImpl;
@@ -361,7 +364,6 @@ public class IncrementalTableTest extends TableTestBase {
TableWriteImpl<?> write = table.newWrite(commitUser);
TableCommitImpl commit =
table.newCommit(commitUser).ignoreEmptyCommit(false);
- TagManager tagManager = table.tagManager();
write.write(GenericRow.of(1, BinaryString.fromString("a")));
List<CommitMessage> commitMessages = write.prepareCommit(false, 0);
@@ -390,6 +392,51 @@ public class IncrementalTableTest extends TableTestBase {
Collections.emptyMap(),
commitMessages));
+ assertIncrementalToAutoTag(table);
+ }
+
+ @Test
+ public void testIncrementalToAutoTagWithManualTag() throws Exception {
+ Identifier identifier = identifier("T");
+ Schema schema =
+ Schema.newBuilder()
+ .column("a", DataTypes.INT())
+ .column("b", DataTypes.STRING())
+ .primaryKey("a")
+ .option("bucket", "1")
+ .option("tag.creation-period", "daily")
+ .build();
+ catalog.createTable(identifier, schema, false);
+ FileStoreTable table = (FileStoreTable) catalog.getTable(identifier);
+
+ BatchWriteBuilder writeBuilder = table.newBatchWriteBuilder();
+
+ try (BatchTableWrite write = writeBuilder.newWrite();
+ BatchTableCommit commit = writeBuilder.newCommit()) {
+ write.write(GenericRow.of(1, BinaryString.fromString("a")));
+ commit.commit(write.prepareCommit());
+ table.createTag("2024-12-01");
+ }
+
+ try (BatchTableWrite write = writeBuilder.newWrite();
+ BatchTableCommit commit = writeBuilder.newCommit()) {
+ write.write(GenericRow.of(2, BinaryString.fromString("b")));
+ commit.commit(write.prepareCommit());
+ table.createTag("2024-12-02");
+ }
+
+ try (BatchTableWrite write = writeBuilder.newWrite();
+ BatchTableCommit commit = writeBuilder.newCommit()) {
+ write.write(GenericRow.of(3, BinaryString.fromString("c")));
+ commit.commit(write.prepareCommit());
+ table.createTag("2024-12-04");
+ }
+
+ assertIncrementalToAutoTag(table);
+ }
+
+ private void assertIncrementalToAutoTag(FileStoreTable table) throws
Exception {
+ TagManager tagManager = table.tagManager();
assertThat(tagManager.allTagNames()).containsOnly("2024-12-01",
"2024-12-02", "2024-12-04");
assertThat(read(table, Pair.of(INCREMENTAL_TO_AUTO_TAG,
"2024-12-01"))).isEmpty();
