This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new b717e7087eb HIVE-28439: Iceberg: Bucket partition transform with
DECIMAL can throw NPE (Shohei Okumiya, reviewed by Denys Kuzmenko)
b717e7087eb is described below
commit b717e7087eb5521b17a138aa7ac639f789099178
Author: okumin <[email protected]>
AuthorDate: Fri Aug 9 23:26:10 2024 +0900
HIVE-28439: Iceberg: Bucket partition transform with DECIMAL can throw NPE
(Shohei Okumiya, reviewed by Denys Kuzmenko)
Closes #5387
---
.../mr/hive/udf/GenericUDFIcebergBucket.java | 3 +-
.../iceberg_insert_into_partition_transforms.q | 16 ++
.../iceberg_insert_into_partition_transforms.q.out | 221 +++++++++++++++++++++
3 files changed, 239 insertions(+), 1 deletion(-)
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/udf/GenericUDFIcebergBucket.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/udf/GenericUDFIcebergBucket.java
index 0077e6706fd..f23bfdfe0ae 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/udf/GenericUDFIcebergBucket.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/udf/GenericUDFIcebergBucket.java
@@ -33,6 +33,7 @@ import
org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantIntObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableHiveDecimalObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.io.BytesWritable;
@@ -130,7 +131,7 @@ public class GenericUDFIcebergBucket extends GenericUDF {
decimalTypeInfo.getScale());
converter = new
PrimitiveObjectInspectorConverter.HiveDecimalConverter(argumentOI,
-
PrimitiveObjectInspectorFactory.writableHiveDecimalObjectInspector);
+ new WritableHiveDecimalObjectInspector(decimalTypeInfo));
Function<Object, Integer> bigDecimalTransform =
Transforms.bucket(numBuckets).bind(decimalIcebergType);
evaluator = arg -> {
HiveDecimalWritable val = (HiveDecimalWritable)
converter.convert(arg.get());
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_insert_into_partition_transforms.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_insert_into_partition_transforms.q
index 8d34a324398..aab66b6bae5 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_insert_into_partition_transforms.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_insert_into_partition_transforms.q
@@ -119,8 +119,24 @@ insert into ice_parquet_date_transform_bucket partition
(pcol = 'gfhutjkgkd') se
describe formatted ice_parquet_date_transform_bucket;
select * from ice_parquet_date_transform_bucket;
+create external table ice_parquet_decimal_transform_bucket(
+ pcol decimal(38, 0)
+) partitioned by spec (bucket(16, pcol))
+stored by iceberg;
+
+explain insert into ice_parquet_decimal_transform_bucket values
+('0'),
+('50000000000000000000441610525');
+insert into ice_parquet_decimal_transform_bucket values
+('0'),
+('50000000000000000000441610525');
+
+describe formatted ice_parquet_decimal_transform_bucket;
+select * from ice_parquet_decimal_transform_bucket;
+
drop table ice_parquet_date_transform_year;
drop table ice_parquet_date_transform_month;
drop table ice_parquet_date_transform_day;
drop table ice_parquet_date_transform_truncate;
drop table ice_parquet_date_transform_bucket;
+drop table ice_parquet_decimal_transform_bucket;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out
index cb495639532..78cbbe46133 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_insert_into_partition_transforms.q.out
@@ -2808,6 +2808,217 @@ gfhutjkgkd 67489376589302 76859
gfhutjkgkd 67489376589302 76859
gfhutjkgkd 67489376589302 76859
gfhutjkgkd 67489376589302 76859
+PREHOOK: query: create external table ice_parquet_decimal_transform_bucket(
+ pcol decimal(38, 0)
+) partitioned by spec (bucket(16, pcol))
+stored by iceberg
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_parquet_decimal_transform_bucket
+POSTHOOK: query: create external table ice_parquet_decimal_transform_bucket(
+ pcol decimal(38, 0)
+) partitioned by spec (bucket(16, pcol))
+stored by iceberg
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_parquet_decimal_transform_bucket
+PREHOOK: query: explain insert into ice_parquet_decimal_transform_bucket values
+('0'),
+('50000000000000000000441610525')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_parquet_decimal_transform_bucket
+POSTHOOK: query: explain insert into ice_parquet_decimal_transform_bucket
values
+('0'),
+('50000000000000000000441610525')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_parquet_decimal_transform_bucket
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: _dummy_table
+ Row Limit Per Split: 1
+ Statistics: Num rows: 1 Data size: 10 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: array(const struct('0'),const
struct('50000000000000000000441610525')) (type: array<struct<col1:string>>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 56 Basic stats:
COMPLETE Column stats: COMPLETE
+ UDTF Operator
+ Statistics: Num rows: 1 Data size: 56 Basic stats:
COMPLETE Column stats: COMPLETE
+ function name: inline
+ Select Operator
+ expressions: CAST( col1 AS decimal(38,0)) (type:
decimal(38,0))
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 112 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: iceberg_bucket(_col0, 16) (type:
int)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: iceberg_bucket(_col0,
16) (type: int)
+ Statistics: Num rows: 1 Data size: 112 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: decimal(38,0))
+ Select Operator
+ expressions: _col0 (type: decimal(38,0))
+ outputColumnNames: pcol
+ Statistics: Num rows: 1 Data size: 112 Basic stats:
COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: min(pcol), max(pcol), count(1),
count(pcol), compute_bit_vector_hll(pcol)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3,
_col4
+ Statistics: Num rows: 1 Data size: 384 Basic
stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Statistics: Num rows: 1 Data size: 384 Basic
stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: decimal(38,0)),
_col1 (type: decimal(38,0)), _col2 (type: bigint), _col3 (type: bigint), _col4
(type: binary)
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: decimal(38,0)),
KEY.iceberg_bucket(_col0, 16) (type: int)
+ outputColumnNames: _col0, iceberg_bucket(_col0, 16)
+ File Output Operator
+ compressed: false
+ Dp Sort State: PARTITION_SORTED
+ Statistics: Num rows: 1 Data size: 112 Basic stats: COMPLETE
Column stats: COMPLETE
+ table:
+ input format:
org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format:
org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.ice_parquet_decimal_transform_bucket
+ Reducer 3
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: min(VALUE._col0), max(VALUE._col1),
count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 1 Data size: 384 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: 'DECIMAL' (type: string), _col0 (type:
decimal(38,0)), _col1 (type: decimal(38,0)), (_col2 - _col3) (type: bigint),
COALESCE(ndv_compute_bit_vector(_col4),0) (type: bigint), _col4 (type: binary)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 1 Data size: 475 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 475 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: false
+ table:
+ input format: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+ output format: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+ serde: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+ name: default.ice_parquet_decimal_transform_bucket
+
+ Stage: Stage-3
+ Stats Work
+ Basic Stats Work:
+ Column Stats Desc:
+ Columns: pcol
+ Column Types: decimal(38,0)
+ Table: default.ice_parquet_decimal_transform_bucket
+
+PREHOOK: query: insert into ice_parquet_decimal_transform_bucket values
+('0'),
+('50000000000000000000441610525')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_parquet_decimal_transform_bucket
+POSTHOOK: query: insert into ice_parquet_decimal_transform_bucket values
+('0'),
+('50000000000000000000441610525')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_parquet_decimal_transform_bucket
+PREHOOK: query: describe formatted ice_parquet_decimal_transform_bucket
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@ice_parquet_decimal_transform_bucket
+POSTHOOK: query: describe formatted ice_parquet_decimal_transform_bucket
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@ice_parquet_decimal_transform_bucket
+# col_name data_type comment
+pcol decimal(38,0)
+
+# Partition Transform Information
+# col_name transform_type
+pcol BUCKET[16]
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: EXTERNAL_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"pcol\":\"true\"}}
+ EXTERNAL TRUE
+ bucketing_version 2
+ current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"pcol\",\"required\":false,\"type\":\"decimal(38,
0)\"}]}
+ current-snapshot-id #Masked#
+ current-snapshot-summary
{\"added-data-files\":\"2\",\"added-records\":\"2\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"2\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"#Masked#\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\"}
+ current-snapshot-timestamp-ms #Masked#
+ default-partition-spec
{\"spec-id\":0,\"fields\":[{\"name\":\"pcol_bucket\",\"transform\":\"bucket[16]\",\"source-id\":1,\"field-id\":1000}]}
+ format-version 2
+ iceberg.orc.files.only false
+ metadata_location hdfs://### HDFS PATH ###
+ numFiles #Masked#
+ numRows 2
+ parquet.compression zstd
+ previous_metadata_location hdfs://### HDFS PATH ###
+ rawDataSize 0
+ serialization.format 1
+ snapshot-count 1
+ storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
+ table_type ICEBERG
+ totalSize #Masked#
+#### A masked pattern was here ####
+ uuid #Masked#
+ write.delete.mode merge-on-read
+ write.merge.mode merge-on-read
+ write.update.mode merge-on-read
+
+# Storage Information
+SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+Compressed: No
+Sort Columns: []
+PREHOOK: query: select * from ice_parquet_decimal_transform_bucket
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_parquet_decimal_transform_bucket
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from ice_parquet_decimal_transform_bucket
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_parquet_decimal_transform_bucket
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+0
+50000000000000000000441610525
PREHOOK: query: drop table ice_parquet_date_transform_year
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@ice_parquet_date_transform_year
@@ -2858,3 +3069,13 @@ POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@ice_parquet_date_transform_bucket
POSTHOOK: Output: database:default
POSTHOOK: Output: default@ice_parquet_date_transform_bucket
+PREHOOK: query: drop table ice_parquet_decimal_transform_bucket
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@ice_parquet_decimal_transform_bucket
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_parquet_decimal_transform_bucket
+POSTHOOK: query: drop table ice_parquet_decimal_transform_bucket
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@ice_parquet_decimal_transform_bucket
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_parquet_decimal_transform_bucket