This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 0ac3c60757 [core] fix that auto-created tags without dashes cannot be
expired (#5786)
0ac3c60757 is described below
commit 0ac3c60757902ddb0d32e579f7b3e9464cfa5e0f
Author: zhoulii <[email protected]>
AuthorDate: Fri Jul 11 14:07:35 2025 +0800
[core] fix that auto-created tags without dashes cannot be expired (#5786)
---
.../org/apache/paimon/tag/TagPeriodHandler.java | 59 +++++++-------
.../org/apache/paimon/tag/TagAutoManagerTest.java | 18 +++++
.../apache/paimon/tag/TagPeriodHandlerTest.java | 94 ++++++++++++++++++++++
3 files changed, 141 insertions(+), 30 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/tag/TagPeriodHandler.java
b/paimon-core/src/main/java/org/apache/paimon/tag/TagPeriodHandler.java
index 2d0df61b6a..abd7375cb9 100644
--- a/paimon-core/src/main/java/org/apache/paimon/tag/TagPeriodHandler.java
+++ b/paimon-core/src/main/java/org/apache/paimon/tag/TagPeriodHandler.java
@@ -27,7 +27,6 @@ import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
import java.time.format.ResolverStyle;
-import java.time.format.SignStyle;
import static java.time.temporal.ChronoField.DAY_OF_MONTH;
import static java.time.temporal.ChronoField.HOUR_OF_DAY;
@@ -41,62 +40,62 @@ public interface TagPeriodHandler {
DateTimeFormatter HOUR_FORMATTER =
new DateTimeFormatterBuilder()
- .appendValue(YEAR, 1, 10, SignStyle.NORMAL)
+ .appendValue(YEAR, 4)
.appendLiteral('-')
- .appendValue(MONTH_OF_YEAR, 2, 2, SignStyle.NORMAL)
+ .appendValue(MONTH_OF_YEAR, 2)
.appendLiteral('-')
- .appendValue(DAY_OF_MONTH, 2, 2, SignStyle.NORMAL)
+ .appendValue(DAY_OF_MONTH, 2)
.appendLiteral(" ")
- .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NORMAL)
+ .appendValue(HOUR_OF_DAY, 2)
.toFormatter()
- .withResolverStyle(ResolverStyle.LENIENT);
+ .withResolverStyle(ResolverStyle.STRICT);
DateTimeFormatter HOUR_FORMATTER_WITHOUT_DASHES =
new DateTimeFormatterBuilder()
- .appendValue(YEAR, 1, 10, SignStyle.NORMAL)
- .appendValue(MONTH_OF_YEAR, 2, 2, SignStyle.NORMAL)
- .appendValue(DAY_OF_MONTH, 2, 2, SignStyle.NORMAL)
+ .appendValue(YEAR, 4)
+ .appendValue(MONTH_OF_YEAR, 2)
+ .appendValue(DAY_OF_MONTH, 2)
.appendLiteral(" ")
- .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NORMAL)
+ .appendValue(HOUR_OF_DAY, 2)
.toFormatter()
- .withResolverStyle(ResolverStyle.LENIENT);
+ .withResolverStyle(ResolverStyle.STRICT);
DateTimeFormatter HOUR_FORMATTER_WITHOUT_DASHES_AND_SPACES =
new DateTimeFormatterBuilder()
- .appendValue(YEAR, 1, 10, SignStyle.NORMAL)
- .appendValue(MONTH_OF_YEAR, 2, 2, SignStyle.NORMAL)
- .appendValue(DAY_OF_MONTH, 2, 2, SignStyle.NORMAL)
- .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NORMAL)
+ .appendValue(YEAR, 4)
+ .appendValue(MONTH_OF_YEAR, 2)
+ .appendValue(DAY_OF_MONTH, 2)
+ .appendValue(HOUR_OF_DAY, 2)
.toFormatter()
- .withResolverStyle(ResolverStyle.LENIENT);
+ .withResolverStyle(ResolverStyle.STRICT);
DateTimeFormatter MINUTE_FORMATTER =
new DateTimeFormatterBuilder()
- .appendValue(YEAR, 1, 10, SignStyle.NORMAL)
- .appendValue(MONTH_OF_YEAR, 2, 2, SignStyle.NORMAL)
- .appendValue(DAY_OF_MONTH, 2, 2, SignStyle.NORMAL)
- .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NORMAL)
- .appendValue(MINUTE_OF_HOUR, 2, 2, SignStyle.NORMAL)
+ .appendValue(YEAR, 4)
+ .appendValue(MONTH_OF_YEAR, 2)
+ .appendValue(DAY_OF_MONTH, 2)
+ .appendValue(HOUR_OF_DAY, 2)
+ .appendValue(MINUTE_OF_HOUR, 2)
.toFormatter()
- .withResolverStyle(ResolverStyle.LENIENT);
+ .withResolverStyle(ResolverStyle.STRICT);
DateTimeFormatter DAY_FORMATTER =
new DateTimeFormatterBuilder()
- .appendValue(YEAR, 1, 10, SignStyle.NORMAL)
+ .appendValue(YEAR, 4)
.appendLiteral('-')
- .appendValue(MONTH_OF_YEAR, 2, 2, SignStyle.NORMAL)
+ .appendValue(MONTH_OF_YEAR, 2)
.appendLiteral('-')
- .appendValue(DAY_OF_MONTH, 2, 2, SignStyle.NORMAL)
+ .appendValue(DAY_OF_MONTH, 2)
.toFormatter()
- .withResolverStyle(ResolverStyle.LENIENT);
+ .withResolverStyle(ResolverStyle.STRICT);
DateTimeFormatter DAY_FORMATTER_WITHOUT_DASHES =
new DateTimeFormatterBuilder()
- .appendValue(YEAR, 1, 10, SignStyle.NORMAL)
- .appendValue(MONTH_OF_YEAR, 2, 2, SignStyle.NORMAL)
- .appendValue(DAY_OF_MONTH, 2, 2, SignStyle.NORMAL)
+ .appendValue(YEAR, 4)
+ .appendValue(MONTH_OF_YEAR, 2)
+ .appendValue(DAY_OF_MONTH, 2)
.toFormatter()
- .withResolverStyle(ResolverStyle.LENIENT);
+ .withResolverStyle(ResolverStyle.STRICT);
void validateDelay(Duration delay);
diff --git
a/paimon-core/src/test/java/org/apache/paimon/tag/TagAutoManagerTest.java
b/paimon-core/src/test/java/org/apache/paimon/tag/TagAutoManagerTest.java
index 089a639c50..0d888225dc 100644
--- a/paimon-core/src/test/java/org/apache/paimon/tag/TagAutoManagerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/tag/TagAutoManagerTest.java
@@ -501,6 +501,24 @@ public class TagAutoManagerTest extends
PrimaryKeyTableTestBase {
assertThrows(RuntimeException.class, () ->
table.createTag("2023-07-18", 1));
}
+ @Test
+ public void testAutoExpireTagWithoutDashes() {
+ Options options = new Options();
+ options.set(TAG_AUTOMATIC_CREATION, TagCreationMode.WATERMARK);
+ options.set(TAG_CREATION_PERIOD, TagCreationPeriod.DAILY);
+ options.set(TAG_PERIOD_FORMATTER, TagPeriodFormatter.WITHOUT_DASHES);
+ options.set(TAG_NUM_RETAINED_MAX, 1);
+ FileStoreTable table = this.table.copy(options.toMap());
+ TableCommitImpl commit =
table.newCommit(commitUser).ignoreEmptyCommit(false);
+ TagManager tagManager = table.store().newTagManager();
+
+ commit.commit(new ManifestCommittable(0,
utcMills("2023-07-18T12:12:00")));
+ assertThat(tagManager.allTagNames()).containsOnly("20230717");
+
+ commit.commit(new ManifestCommittable(1,
utcMills("2023-07-19T12:12:00")));
+ assertThat(tagManager.allTagNames()).containsOnly("20230718");
+ }
+
private long localZoneMills(String timestamp) {
return LocalDateTime.parse(timestamp)
.atZone(ZoneId.systemDefault())
diff --git
a/paimon-core/src/test/java/org/apache/paimon/tag/TagPeriodHandlerTest.java
b/paimon-core/src/test/java/org/apache/paimon/tag/TagPeriodHandlerTest.java
new file mode 100644
index 0000000000..0bc8ba5e7a
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/tag/TagPeriodHandlerTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.tag;
+
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.time.LocalDateTime;
+
+import static org.apache.paimon.CoreOptions.TagPeriodFormatter.WITHOUT_DASHES;
+import static
org.apache.paimon.CoreOptions.TagPeriodFormatter.WITHOUT_DASHES_AND_SPACES;
+import static org.apache.paimon.CoreOptions.TagPeriodFormatter.WITH_DASHES;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Test for TagPeriodHandler. */
+public class TagPeriodHandlerTest {
+
+ @Test
+ public void testDailyTagPeriodHandler() {
+ TagPeriodHandler.DailyTagPeriodHandler dailyTagPeriodHandler =
+ new TagPeriodHandler.DailyTagPeriodHandler(WITH_DASHES);
+ assertThat(dailyTagPeriodHandler.timeToTag(LocalDateTime.of(2023, 1,
1, 0, 0)))
+ .isEqualTo("2023-01-01");
+ assertThat(dailyTagPeriodHandler.tagToTime("2023-01-01"))
+ .isEqualTo(LocalDateTime.of(2023, 1, 1, 0, 0));
+ assertThat(dailyTagPeriodHandler.isAutoTag("2023-01-01")).isTrue();
+ assertThat(dailyTagPeriodHandler.isAutoTag("20230101")).isFalse();
+
+ dailyTagPeriodHandler = new
TagPeriodHandler.DailyTagPeriodHandler(WITHOUT_DASHES);
+ assertThat(dailyTagPeriodHandler.timeToTag(LocalDateTime.of(2023, 1,
1, 0, 0)))
+ .isEqualTo("20230101");
+ assertThat(dailyTagPeriodHandler.tagToTime("20230101"))
+ .isEqualTo(LocalDateTime.of(2023, 1, 1, 0, 0));
+ assertThat(dailyTagPeriodHandler.isAutoTag("20230101")).isTrue();
+ assertThat(dailyTagPeriodHandler.isAutoTag("2023-01-01")).isFalse();
+ }
+
+ @Test
+ public void testHourlyTagPeriodHandler() {
+ TagPeriodHandler.HourlyTagPeriodHandler hourlyTagPeriodHandler =
+ new TagPeriodHandler.HourlyTagPeriodHandler(WITH_DASHES);
+ assertThat(hourlyTagPeriodHandler.timeToTag(LocalDateTime.of(2023, 1,
1, 1, 0)))
+ .isEqualTo("2023-01-01 01");
+ assertThat(hourlyTagPeriodHandler.tagToTime("2023-01-01 01"))
+ .isEqualTo(LocalDateTime.of(2023, 1, 1, 1, 0));
+ assertThat(hourlyTagPeriodHandler.isAutoTag("2023-01-01 01")).isTrue();
+ assertThat(hourlyTagPeriodHandler.isAutoTag("20230101 01")).isFalse();
+
+ hourlyTagPeriodHandler = new
TagPeriodHandler.HourlyTagPeriodHandler(WITHOUT_DASHES);
+ assertThat(hourlyTagPeriodHandler.timeToTag(LocalDateTime.of(2023, 1,
1, 1, 0)))
+ .isEqualTo("20230101 01");
+ assertThat(hourlyTagPeriodHandler.tagToTime("20230101 01"))
+ .isEqualTo(LocalDateTime.of(2023, 1, 1, 1, 0));
+ assertThat(hourlyTagPeriodHandler.isAutoTag("20230101 01")).isTrue();
+ assertThat(hourlyTagPeriodHandler.isAutoTag("2023-01-01
01")).isFalse();
+
+ hourlyTagPeriodHandler =
+ new
TagPeriodHandler.HourlyTagPeriodHandler(WITHOUT_DASHES_AND_SPACES);
+ assertThat(hourlyTagPeriodHandler.timeToTag(LocalDateTime.of(2023, 1,
1, 1, 0)))
+ .isEqualTo("2023010101");
+ assertThat(hourlyTagPeriodHandler.tagToTime("2023010101"))
+ .isEqualTo(LocalDateTime.of(2023, 1, 1, 1, 0));
+ assertThat(hourlyTagPeriodHandler.isAutoTag("2023010101")).isTrue();
+ assertThat(hourlyTagPeriodHandler.isAutoTag("20230101 01")).isFalse();
+ }
+
+ @Test
+ public void testPeriodDurationTagPeriodHandler() {
+ TagPeriodHandler.PeriodDurationTagPeriodHandler
periodDurationTagPeriodHandler =
+ new
TagPeriodHandler.PeriodDurationTagPeriodHandler(Duration.ofMinutes(5));
+
assertThat(periodDurationTagPeriodHandler.timeToTag(LocalDateTime.of(2023, 1,
1, 1, 1)))
+ .isEqualTo("202301010101");
+ assertThat(periodDurationTagPeriodHandler.tagToTime("202301010101"))
+ .isEqualTo(LocalDateTime.of(2023, 1, 1, 1, 1));
+
assertThat(periodDurationTagPeriodHandler.isAutoTag("202301010101")).isTrue();
+ assertThat(periodDurationTagPeriodHandler.isAutoTag("20230101
0101")).isFalse();
+ }
+}
