This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 93d70104e4 [core, api] Add option to disable explicit type casting
(#5692)
93d70104e4 is described below
commit 93d70104e4c2596d1d941bd71b7014257a7bf280
Author: Ashish Khatkar <[email protected]>
AuthorDate: Thu Jun 5 11:54:49 2025 +0100
[core, api] Add option to disable explicit type casting (#5692)
---
.../shortcodes/generated/core_configuration.html | 6 +++++
.../main/java/org/apache/paimon/CoreOptions.java | 12 ++++++++++
.../org/apache/paimon/types/DataTypeCasts.java | 25 +++++++------------
.../org/apache/paimon/schema/SchemaManager.java | 14 +++++++++--
.../apache/paimon/schema/SchemaMergingUtils.java | 6 +----
.../apache/paimon/table/SchemaEvolutionTest.java | 15 ++++++++++++
.../apache/paimon/flink/SchemaChangeITCase.java | 28 ++++++++++++++++++++++
7 files changed, 83 insertions(+), 23 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html
b/docs/layouts/shortcodes/generated/core_configuration.html
index 856245b64f..8811915954 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -314,6 +314,12 @@ under the License.
<td>Boolean</td>
<td>Whether to enable deletion vectors mode. In this mode, index
files containing deletion vectors are generated when data is written, which
marks the data for deletion. During read operations, by applying these index
files, merging can be avoided.</td>
</tr>
+ <tr>
+ <td><h5>disable-explicit-type-casting</h5></td>
+ <td style="word-wrap: break-word;">false</td>
+ <td>Boolean</td>
+ <td>If true, it disables explicit type casting. For ex: it
disables converting LONG type to INT type. Users can enable this option to
disable explicit type casting</td>
+ </tr>
<tr>
<td><h5>dynamic-bucket.assigner-parallelism</h5></td>
<td style="word-wrap: break-word;">(none)</td>
diff --git a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
index 88f8ca7582..4d437f19fa 100644
--- a/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-api/src/main/java/org/apache/paimon/CoreOptions.java
@@ -1736,6 +1736,14 @@ public class CoreOptions implements Serializable {
"If true, it disables altering column type from
null to not null. Default is true. "
+ "Users can disable this option to
explicitly convert null column type to not null.");
+ public static final ConfigOption<Boolean> DISABLE_EXPLICIT_TYPE_CASTING =
+ ConfigOptions.key("disable-explicit-type-casting")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription(
+ "If true, it disables explicit type casting. For
ex: it disables converting LONG type to INT type. "
+ + "Users can enable this option to disable
explicit type casting");
+
private final Options options;
public CoreOptions(Map<String, String> options) {
@@ -2251,6 +2259,10 @@ public class CoreOptions implements Serializable {
return options.get(DISABLE_ALTER_COLUMN_NULL_TO_NOT_NULL);
}
+ public boolean disableExplicitTypeCasting() {
+ return options.get(DISABLE_EXPLICIT_TYPE_CASTING);
+ }
+
public LookupStrategy lookupStrategy() {
return LookupStrategy.from(
mergeEngine().equals(MergeEngine.FIRST_ROW),
diff --git
a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeCasts.java
b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeCasts.java
index 0c0d2a8132..454c1c65d7 100644
--- a/paimon-api/src/main/java/org/apache/paimon/types/DataTypeCasts.java
+++ b/paimon-api/src/main/java/org/apache/paimon/types/DataTypeCasts.java
@@ -178,27 +178,20 @@ public final class DataTypeCasts {
}
/**
- * Returns whether the source type can be safely cast to the target type
without loosing
- * information.
+ * allowExplicit false : Returns whether the source type can be safely
cast to the target type
+ * without loosing information. Implicit casts are used for type widening
and type
+ * generalization (finding a common supertype for a set of types).
Implicit casts are similar to
+ * the Java semantics (e.g. this is not possible: {@code int x = (String)
z}).
*
- * <p>Implicit casts are used for type widening and type generalization
(finding a common
- * supertype for a set of types). Implicit casts are similar to the Java
semantics (e.g. this is
- * not possible: {@code int x = (String) z}).
- */
- public static boolean supportsImplicitCast(DataType sourceType, DataType
targetType) {
- return supportsCasting(sourceType, targetType, false);
- }
-
- /**
- * Returns whether the source type can be cast to the target type.
- *
- * <p>Explicit casts correspond to the SQL cast specification and
represent the logic behind a
+ * <p>allowExplicit true : Returns whether the source type can be cast to
the target type.
+ * Explicit casts correspond to the SQL cast specification and represent
the logic behind a
* {@code CAST(sourceType AS targetType)} operation. For example, it
allows for converting most
* types of the {@link DataTypeFamily#PREDEFINED} family to types of the
{@link
* DataTypeFamily#CHARACTER_STRING} family.
*/
- public static boolean supportsExplicitCast(DataType sourceType, DataType
targetType) {
- return supportsCasting(sourceType, targetType, true);
+ public static boolean supportsCast(
+ DataType sourceType, DataType targetType, boolean allowExplicit) {
+ return supportsCasting(sourceType, targetType, allowExplicit);
}
/**
diff --git
a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
index 59c3c42dd5..21484cfc84 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java
@@ -308,6 +308,14 @@ public class SchemaManager implements Serializable {
CoreOptions.DISABLE_ALTER_COLUMN_NULL_TO_NOT_NULL
.defaultValue()
.toString()));
+
+ boolean disableExplicitTypeCasting =
+ Boolean.parseBoolean(
+ oldOptions.getOrDefault(
+
CoreOptions.DISABLE_EXPLICIT_TYPE_CASTING.key(),
+ CoreOptions.DISABLE_EXPLICIT_TYPE_CASTING
+ .defaultValue()
+ .toString()));
List<DataField> newFields = new ArrayList<>(oldTableSchema.fields());
AtomicInteger highestFieldId = new
AtomicInteger(oldTableSchema.highestFieldId());
String newComment = oldTableSchema.comment();
@@ -434,8 +442,10 @@ public class SchemaManager implements Serializable {
disableNullToNotNull);
}
checkState(
- DataTypeCasts.supportsExplicitCast(
- sourceRootType,
targetRootType)
+ DataTypeCasts.supportsCast(
+ sourceRootType,
+ targetRootType,
+
!disableExplicitTypeCasting)
&&
CastExecutors.resolve(sourceRootType, targetRootType)
!= null,
String.format(
diff --git
a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaMergingUtils.java
b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaMergingUtils.java
index d7e21cd656..0fff27dce4 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaMergingUtils.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaMergingUtils.java
@@ -220,11 +220,7 @@ public class SchemaMergingUtils {
private static boolean supportsDataTypesCast(
DataType sourceType, DataType targetType, boolean
allowExplicitCast) {
- if (allowExplicitCast) {
- return DataTypeCasts.supportsExplicitCast(sourceType, targetType);
- } else {
- return DataTypeCasts.supportsImplicitCast(sourceType, targetType);
- }
+ return DataTypeCasts.supportsCast(sourceType, targetType,
allowExplicitCast);
}
private static DataField assignIdForNewField(DataField field,
AtomicInteger highestFieldId) {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTest.java
index 951539299c..76a63eb06e 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTest.java
@@ -314,6 +314,21 @@ public class SchemaEvolutionTest {
assertThat(tableSchema.fields().get(0).type()).isEqualTo(DataTypes.BIGINT());
assertThat(tableSchema.fields().get(0).description()).isEqualTo("f0
field");
+ schemaManager.commitChanges(
+ Collections.singletonList(
+
SchemaChange.setOption("disable-explicit-type-casting", "true")));
+ assertThatThrownBy(
+ () ->
+ schemaManager.commitChanges(
+ Collections.singletonList(
+ SchemaChange.updateColumnType(
+ "f0",
DataTypes.STRING()))))
+ .isInstanceOf(IllegalStateException.class)
+ .hasMessage(
+ "Column type f0[BIGINT] cannot be converted to STRING
without loosing information.");
+ schemaManager.commitChanges(
+ Collections.singletonList(
+
SchemaChange.setOption("disable-explicit-type-casting", "false")));
// bigint to string
tableSchema =
schemaManager.commitChanges(
diff --git
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java
index 03b568202e..da260851a8 100644
---
a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java
+++
b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/SchemaChangeITCase.java
@@ -423,6 +423,7 @@ public class SchemaChangeITCase extends CatalogITCaseBase {
"+I[3.1400, 1, 123, 3.14, 3]", "+I[4.1300, 2, 456,
3.14, 4]");
sql("CREATE TABLE T1 (a STRING, b STRING)");
+ sql("ALTER TABLE T1 SET ('disable-explicit-type-casting' = 'false')");
sql("INSERT INTO T1 VALUES('test', '3.14')");
sql("ALTER TABLE T1 MODIFY (a INT, b TINYINT)");
@@ -1561,4 +1562,31 @@ public class SchemaChangeITCase extends
CatalogITCaseBase {
.hasStackTraceContaining(
"Cannot update column type from nullable to non
nullable for a.c3.element.value");
}
+
+ @ParameterizedTest
+ @ValueSource(strings = {"orc", "avro", "parquet"})
+ public void testDisableExplicitTypeCasting(String formatType) {
+ sql(
+ "CREATE TABLE T "
+ + "( k INT, v INT, PRIMARY KEY (k) NOT ENFORCED ) "
+ + "WITH ( 'bucket' = '1', 'file.format' = '"
+ + formatType
+ + "' )");
+ sql("ALTER TABLE T SET ('disable-explicit-type-casting' = 'true')");
+ sql("INSERT INTO T VALUES (1, 10), (2, 20)");
+ assertThat(sql("SELECT * FROM T")).containsExactlyInAnyOrder(Row.of(1,
10), Row.of(2, 20));
+ assertThatCode(() -> sql("ALTER TABLE T MODIFY v SMALLINT"))
+ .hasStackTraceContaining(
+ "Column type v[INT] cannot be converted to SMALLINT
without loosing information");
+ sql("ALTER TABLE T MODIFY v BIGINT");
+ assertThat(sql("SELECT * FROM T"))
+ .containsExactlyInAnyOrder(Row.of(1, 10L), Row.of(2, 20L));
+ assertThatCode(() -> sql("ALTER TABLE T MODIFY v INT"))
+ .hasStackTraceContaining(
+ "Column type v[BIGINT] cannot be converted to INT
without loosing information");
+ // disable explicit type casting
+ sql("ALTER TABLE T SET ('disable-explicit-type-casting' = 'false')");
+ sql("ALTER TABLE T MODIFY v INT");
+ assertThat(sql("SELECT * FROM T")).containsExactlyInAnyOrder(Row.of(1,
10), Row.of(2, 20));
+ }
}
