This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 1b64b5d36c [core] Allow change ignore-delete/ignore-update-before from
false to true (#6811)
1b64b5d36c is described below
commit 1b64b5d36c53e548697215a8256084336f901c70
Author: yuzelin <[email protected]>
AuthorDate: Fri Dec 12 17:41:46 2025 +0800
[core] Allow change ignore-delete/ignore-update-before from false to true
(#6811)
---
.../main/java/org/apache/paimon/CoreOptions.java | 2 --
.../mergetree/compact/FirstRowMergeFunction.java | 2 +-
.../org/apache/paimon/schema/SchemaManager.java | 28 +++++++++++++++++
.../apache/paimon/flink/BatchFileStoreITCase.java | 36 +++++++++++++++++++---
4 files changed, 60 insertions(+), 8 deletions(-)
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
index 0151ad96e9..7526ae6099 100644
--- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
@@ -482,7 +482,6 @@ public class CoreOptions implements Serializable {
.defaultValue(MergeEngine.DEDUPLICATE)
.withDescription("Specify the merge engine for table with
primary key.");
- @Immutable
public static final ConfigOption<Boolean> IGNORE_DELETE =
key("ignore-delete")
.booleanType()
@@ -493,7 +492,6 @@ public class CoreOptions implements Serializable {
"partial-update.ignore-delete")
.withDescription("Whether to ignore delete records.");
- @Immutable
public static final ConfigOption<Boolean> IGNORE_UPDATE_BEFORE =
key("ignore-update-before")
.booleanType()
diff --git
a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/FirstRowMergeFunction.java
b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/FirstRowMergeFunction.java
index d795f16bbd..4b9e4a6ff1 100644
---
a/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/FirstRowMergeFunction.java
+++
b/paimon-core/src/main/java/org/apache/paimon/mergetree/compact/FirstRowMergeFunction.java
@@ -54,7 +54,7 @@ public class FirstRowMergeFunction implements
MergeFunction<KeyValue> {
} else {
throw new IllegalArgumentException(
"By default, First row merge engine can not accept
DELETE/UPDATE_BEFORE records.\n"
- + "You can config 'first-row.ignore-delete' to
ignore the DELETE/UPDATE_BEFORE records.");
+ + "You can config 'ignore-delete' to ignore
the DELETE/UPDATE_BEFORE records.");
}
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
index 82b605204b..e726b80326 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
@@ -82,7 +82,9 @@ import static
org.apache.paimon.CoreOptions.DELETION_VECTORS_ENABLED;
import static org.apache.paimon.CoreOptions.DELETION_VECTORS_MODIFIABLE;
import static org.apache.paimon.CoreOptions.DISTINCT;
import static org.apache.paimon.CoreOptions.FIELDS_PREFIX;
+import static org.apache.paimon.CoreOptions.IGNORE_DELETE;
import static org.apache.paimon.CoreOptions.IGNORE_RETRACT;
+import static org.apache.paimon.CoreOptions.IGNORE_UPDATE_BEFORE;
import static org.apache.paimon.CoreOptions.LIST_AGG_DELIMITER;
import static org.apache.paimon.CoreOptions.NESTED_KEY;
import static org.apache.paimon.CoreOptions.SEQUENCE_FIELD;
@@ -1120,6 +1122,32 @@ public class SchemaManager implements Serializable {
}
}
}
+
+ if (IGNORE_DELETE.key().equals(key)) {
+ boolean oldIgnoreDelete =
+ oldValue == null
+ ? IGNORE_DELETE.defaultValue()
+ : Boolean.parseBoolean(oldValue);
+ boolean newIgnoreDelete = Boolean.parseBoolean(newValue);
+ if (oldIgnoreDelete && !newIgnoreDelete) {
+ throw new UnsupportedOperationException(
+ String.format("Cannot change %s from true to false.",
IGNORE_DELETE.key()));
+ }
+ }
+
+ if (IGNORE_UPDATE_BEFORE.key().equals(key)) {
+ boolean oldIgnoreUpdateBefore =
+ oldValue == null
+ ? IGNORE_UPDATE_BEFORE.defaultValue()
+ : Boolean.parseBoolean(oldValue);
+ boolean newIgnoreUpdateBefore = Boolean.parseBoolean(newValue);
+ if (oldIgnoreUpdateBefore && !newIgnoreUpdateBefore) {
+ throw new UnsupportedOperationException(
+ String.format(
+ "Cannot change %s from true to false.",
+ IGNORE_UPDATE_BEFORE.key()));
+ }
+ }
}
public static void checkResetTableOption(String key) {
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BatchFileStoreITCase.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BatchFileStoreITCase.java
index 589eeb14af..af0d7d508d 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BatchFileStoreITCase.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BatchFileStoreITCase.java
@@ -42,6 +42,8 @@ import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
+import javax.annotation.Nullable;
+
import java.math.BigDecimal;
import java.util.Arrays;
import java.util.HashMap;
@@ -63,6 +65,12 @@ public class BatchFileStoreITCase extends CatalogITCaseBase {
return singletonList("CREATE TABLE IF NOT EXISTS T (a INT, b INT, c
INT)");
}
+ @Nullable
+ @Override
+ protected Boolean sqlSyncMode() {
+ return true;
+ }
+
@Test
public void testCsvFileFormat() {
innerTestTextFileFormat("csv");
@@ -605,16 +613,23 @@ public class BatchFileStoreITCase extends
CatalogITCaseBase {
public void testIgnoreDelete() {
sql(
"CREATE TABLE ignore_delete (pk INT PRIMARY KEY NOT ENFORCED,
v STRING) "
- + "WITH ('merge-engine' = 'deduplicate',
'ignore-delete' = 'true', 'bucket' = '1')");
-
- sql("INSERT INTO ignore_delete VALUES (1, 'A')");
+ + "WITH ('merge-engine' = 'deduplicate', 'bucket' =
'1')");
+ sql("INSERT INTO ignore_delete VALUES (1, 'A'), (2, 'B')");
+ sql("DELETE FROM ignore_delete WHERE pk = 2");
assertThat(sql("SELECT * FROM
ignore_delete")).containsExactly(Row.of(1, "A"));
+ // compact to merge the -D record
+ sql("CALL sys.compact(`table` => 'default.ignore_delete')");
+ sql("ALTER TABLE ignore_delete SET ('ignore-delete' = 'true')");
+
sql("DELETE FROM ignore_delete WHERE pk = 1");
assertThat(sql("SELECT * FROM
ignore_delete")).containsExactly(Row.of(1, "A"));
sql("INSERT INTO ignore_delete VALUES (1, 'B')");
assertThat(sql("SELECT * FROM
ignore_delete")).containsExactly(Row.of(1, "B"));
+
+ assertThatThrownBy(() -> sql("ALTER TABLE ignore_delete SET
('ignore-delete' = 'false')"))
+ .hasRootCauseMessage("Cannot change ignore-delete from true to
false.");
}
@Test
@@ -637,16 +652,27 @@ public class BatchFileStoreITCase extends
CatalogITCaseBase {
public void testIgnoreUpdateBeforeWithRowKindField() {
sql(
"CREATE TABLE ignore_delete (pk INT PRIMARY KEY NOT ENFORCED,
v STRING, kind STRING) "
- + "WITH ('ignore-update-before' = 'true', 'bucket' =
'1', 'rowkind.field' = 'kind')");
+ + "WITH ('bucket' = '1', 'rowkind.field' = 'kind')");
- sql("INSERT INTO ignore_delete VALUES (1, 'A', '+I')");
+ sql("INSERT INTO ignore_delete VALUES (1, 'A', '+I'), (2, 'B', '+I')");
+ sql("INSERT INTO ignore_delete VALUES (2, 'B', '-U')");
assertThat(sql("SELECT * FROM
ignore_delete")).containsExactly(Row.of(1, "A", "+I"));
+ // compact to merge the -U record
+ sql("CALL sys.compact(`table` => 'default.ignore_delete')");
+ sql("ALTER TABLE ignore_delete SET ('ignore-update-before' = 'true')");
+
sql("INSERT INTO ignore_delete VALUES (1, 'A', '-U')");
assertThat(sql("SELECT * FROM
ignore_delete")).containsExactly(Row.of(1, "A", "+I"));
sql("INSERT INTO ignore_delete VALUES (1, 'A', '-D')");
assertThat(sql("SELECT * FROM ignore_delete")).isEmpty();
+
+ assertThatThrownBy(
+ () ->
+ sql(
+ "ALTER TABLE ignore_delete SET
('ignore-update-before' = 'false')"))
+ .hasRootCauseMessage("Cannot change ignore-update-before from
true to false.");
}
@Test
