This is an automated email from the ASF dual-hosted git repository. lpinter pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 8ec1d4c989 HIVE-26282: Improve iceberg CTAS error message for unsupported types (#3337) (Laszlo Pinter, reviewed by Peter Vary) 8ec1d4c989 is described below commit 8ec1d4c989baa0c4863084297b73e80e5929044d Author: László Pintér <47777102+lcspin...@users.noreply.github.com> AuthorDate: Fri Jun 3 13:43:10 2022 +0200 HIVE-26282: Improve iceberg CTAS error message for unsupported types (#3337) (Laszlo Pinter, reviewed by Peter Vary) --- .../apache/iceberg/hive/HiveSchemaConverter.java | 6 ++- .../iceberg/mr/hive/TestHiveIcebergCTAS.java | 57 ++++++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java index 422546c112..9122577d41 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java @@ -83,7 +83,8 @@ class HiveSchemaConverter { return Types.BooleanType.get(); case BYTE: case SHORT: - Preconditions.checkArgument(autoConvert, "Unsupported Hive type: %s, use integer instead", + Preconditions.checkArgument(autoConvert, "Unsupported Hive type: %s, use integer " + + "instead or enable automatic type conversion, set 'iceberg.mr.schema.auto.conversion' to true", ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()); LOG.debug("Using auto conversion from SHORT/BYTE to INTEGER"); @@ -96,7 +97,8 @@ class HiveSchemaConverter { return Types.BinaryType.get(); case CHAR: case VARCHAR: - Preconditions.checkArgument(autoConvert, "Unsupported Hive type: %s, use string instead", + Preconditions.checkArgument(autoConvert, "Unsupported Hive type: %s, use integer " + + "instead or enable automatic type conversion, set 'iceberg.mr.schema.auto.conversion' to true", ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory()); LOG.debug("Using auto conversion from CHAR/VARCHAR to STRING"); diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCTAS.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCTAS.java index e41a4c6fc6..fda9652e0f 100644 --- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCTAS.java +++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCTAS.java @@ -22,6 +22,7 @@ package org.apache.iceberg.mr.hive; import java.io.IOException; import java.util.Comparator; import java.util.List; +import java.util.Map; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.iceberg.AssertHelpers; import org.apache.iceberg.PartitionSpec; @@ -32,8 +33,10 @@ import org.apache.iceberg.TableProperties; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.data.Record; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.mr.InputFormatConfig; import org.apache.iceberg.mr.TestHelper; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.apache.thrift.TException; import org.junit.Assert; @@ -222,4 +225,58 @@ public class TestHiveIcebergCTAS extends HiveIcebergStorageHandlerWithEngineBase objects = shell.executeStatement("SELECT * FROM target"); Assert.assertTrue(objects.isEmpty()); } + + @Test + public void testCTASUnsupportedTypeWithoutAutoConversion() { + Assume.assumeTrue(HiveIcebergSerDe.CTAS_EXCEPTION_MSG, testTableType == TestTables.TestTableType.HIVE_CATALOG); + Map<String, Type> notSupportedTypes = ImmutableMap.of( + "TINYINT", Types.IntegerType.get(), + "SMALLINT", Types.IntegerType.get(), + "VARCHAR(1)", Types.StringType.get(), + "CHAR(1)", Types.StringType.get()); + + + for (String notSupportedType : notSupportedTypes.keySet()) { + shell.executeStatement(String.format("CREATE TABLE source (s %s) STORED AS ORC", notSupportedType)); + AssertHelpers.assertThrows("should throw exception", IllegalArgumentException.class, + "Unsupported Hive type: ", () -> { + shell.executeStatement(String.format( + "CREATE TABLE target STORED BY ICEBERG %s %s AS SELECT * FROM source", + testTables.locationForCreateTableSQL(TableIdentifier.of("default", "target")), + testTables.propertiesForCreateTableSQL( + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.toString()) + ) + )); + }); + shell.executeStatement("DROP TABLE source"); + } + } + + @Test + public void testCTASUnsupportedTypeWithAutoConversion() { + Assume.assumeTrue(HiveIcebergSerDe.CTAS_EXCEPTION_MSG, testTableType == TestTables.TestTableType.HIVE_CATALOG); + Map<String, Type> notSupportedTypes = ImmutableMap.of( + "TINYINT", Types.IntegerType.get(), + "SMALLINT", Types.IntegerType.get(), + "VARCHAR(1)", Types.StringType.get(), + "CHAR(1)", Types.StringType.get()); + + shell.setHiveSessionValue(InputFormatConfig.SCHEMA_AUTO_CONVERSION, "true"); + + for (String notSupportedType : notSupportedTypes.keySet()) { + shell.executeStatement(String.format("CREATE TABLE source (s %s) STORED AS ORC", notSupportedType)); + shell.executeStatement(String.format( + "CREATE TABLE target STORED BY ICEBERG %s %s AS SELECT * FROM source", + testTables.locationForCreateTableSQL(TableIdentifier.of("default", "target")), + testTables.propertiesForCreateTableSQL( + ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.toString()) + ) + )); + + org.apache.iceberg.Table icebergTable = testTables.loadTable(TableIdentifier.of("default", "target")); + Assert.assertEquals(notSupportedTypes.get(notSupportedType), icebergTable.schema().columns().get(0).type()); + shell.executeStatement("DROP TABLE source"); + shell.executeStatement("DROP TABLE target"); + } + } }