This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new f0c364574 [Hive] Fix Hive DDL and paimon schema mismatched bug (#4561)
f0c364574 is described below
commit f0c3645741806a19b4075381555c7f61bc500809
Author: Gang Yang <[email protected]>
AuthorDate: Fri Nov 22 14:47:17 2024 +0800
[Hive] Fix Hive DDL and paimon schema mismatched bug (#4561)
---
.../java/org/apache/paimon/hive/HiveSchema.java | 5 +-
.../apache/paimon/hive/HiveTableSchemaTest.java | 61 ++++++++++++++++++----
2 files changed, 53 insertions(+), 13 deletions(-)
diff --git
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java
index f63765141..108315a96 100644
---
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java
+++
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java
@@ -233,9 +233,10 @@ public class HiveSchema {
}
}
- if (schemaFieldNames.size() != hiveFieldNames.size()) {
+ // It is OK that hive is a subset of paimon
+ if (schemaFieldNames.size() < hiveFieldNames.size()) {
throw new IllegalArgumentException(
- "Hive DDL and paimon schema mismatched! "
+ "Hive DDL is a superset of paimon schema! "
+ "It is recommended not to write any column
definition "
+ "as Paimon external table can read schema from
the specified location.\n"
+ "There are "
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java
index 07cd00c8e..fe7aeac08 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java
@@ -153,6 +153,54 @@ public class HiveTableSchemaTest {
.hasMessageContaining(expected);
}
+ @Test
+ public void testSubsetColumnNameAndType() throws Exception {
+ createSchema();
+ Properties properties = new Properties();
+ List<String> columns = Arrays.asList("a", "b");
+ properties.setProperty("columns", String.join(",", columns));
+ properties.setProperty(
+ "columns.types",
+ String.join(
+ ":",
+ Arrays.asList(
+ TypeInfoFactory.intTypeInfo.getTypeName(),
+ TypeInfoFactory.stringTypeInfo.getTypeName(),
+ TypeInfoFactory.getDecimalTypeInfo(6,
3).getTypeName())));
+ properties.setProperty("columns.comments", "\0\0");
+ properties.setProperty("location", tempDir.toString());
+ List<String> fields = HiveSchema.extract(null,
properties).fieldNames();
+ assertThat(fields).isEqualTo(columns);
+ }
+
+ @Test
+ public void testSupersetColumnNameAndType() throws Exception {
+ createSchema();
+ Properties properties = new Properties();
+ properties.setProperty("columns", "a,b,c,d");
+ properties.setProperty(
+ "columns.types",
+ String.join(
+ ":",
+ Arrays.asList(
+ TypeInfoFactory.intTypeInfo.getTypeName(),
+ TypeInfoFactory.stringTypeInfo.getTypeName(),
+ TypeInfoFactory.decimalTypeInfo.getTypeName(),
+ TypeInfoFactory.stringTypeInfo.getTypeName(),
+ TypeInfoFactory.getDecimalTypeInfo(6,
3).getTypeName())));
+ properties.setProperty("columns.comments", "\0\0");
+ properties.setProperty("location", tempDir.toString());
+ String expected =
+ "Hive DDL is a superset of paimon schema! "
+ + "It is recommended not to write any column
definition "
+ + "as Paimon external table can read schema from the
specified location.\n"
+ + "There are 4 fields in Hive DDL: a, b, c, d\n"
+ + "There are 3 fields in Paimon schema: a, b, c\n";
+ assertThatThrownBy(() -> HiveSchema.extract(null, properties))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining(expected);
+ }
+
@Test
public void testTooFewColumns() throws Exception {
createSchema();
@@ -162,16 +210,7 @@ public class HiveTableSchemaTest {
properties.setProperty("columns.types",
TypeInfoFactory.intTypeInfo.getTypeName());
properties.setProperty("location", tempDir.toString());
properties.setProperty("columns.comments", "");
-
- String expected =
- "Hive DDL and paimon schema mismatched! "
- + "It is recommended not to write any column
definition "
- + "as Paimon external table can read schema from the
specified location.\n"
- + "There are 1 fields in Hive DDL: a\n"
- + "There are 3 fields in Paimon schema: a, b, c";
- assertThatExceptionOfType(IllegalArgumentException.class)
- .isThrownBy(() -> HiveSchema.extract(null, properties))
- .withMessageContaining(expected);
+ assertThat(HiveSchema.extract(null,
properties)).isInstanceOf(HiveSchema.class);
}
@Test
@@ -194,7 +233,7 @@ public class HiveTableSchemaTest {
properties.setProperty("location", tempDir.toString());
String expected =
- "Hive DDL and paimon schema mismatched! "
+ "Hive DDL is a superset of paimon schema! "
+ "It is recommended not to write any column
definition "
+ "as Paimon external table can read schema from the
specified location.\n"
+ "There are 5 fields in Hive DDL: a, b, c, d, e\n"
