This is an automated email from the ASF dual-hosted git repository.

dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 18c434f346d HIVE-28266: Iceberg: Select count(*) from data_files 
metadata tables gives wrong result (Dmitriy Fingerman, reviewed by Butao Zhang, 
Denys Kuzmenko)
18c434f346d is described below

commit 18c434f346dc590201afa4159aeec62b7dd5e2cf
Author: Dmitriy Fingerman <[email protected]>
AuthorDate: Wed May 22 04:10:46 2024 -0400

    HIVE-28266: Iceberg: Select count(*) from data_files metadata tables gives 
wrong result (Dmitriy Fingerman, reviewed by Butao Zhang, Denys Kuzmenko)
    
    Closes #5253
---
 .../iceberg/mr/hive/HiveIcebergStorageHandler.java |   6 +-
 .../apache/iceberg/mr/hive/IcebergAcidUtil.java    |   1 +
 .../apache/iceberg/mr/hive/IcebergTableUtil.java   |   2 +-
 .../iceberg_major_compaction_query_metadata.q      |  43 +++++++
 .../iceberg_major_compaction_query_metadata.q.out  | 142 +++++++++++++++++++++
 .../test/resources/testconfiguration.properties    |   1 +
 6 files changed, 191 insertions(+), 4 deletions(-)

diff --git 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index 5e6d545b3ca..275aa993d46 100644
--- 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++ 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -570,7 +570,7 @@ public class HiveIcebergStorageHandler implements 
HiveStoragePredicateHandler, H
 
   @Override
   public boolean 
canComputeQueryUsingStats(org.apache.hadoop.hive.ql.metadata.Table hmsTable) {
-    if (getStatsSource().equals(HiveMetaHook.ICEBERG)) {
+    if (getStatsSource().equals(HiveMetaHook.ICEBERG) && 
hmsTable.getMetaTable() == null) {
       Table table = getTable(hmsTable);
       if (table.currentSnapshot() != null) {
         Map<String, String> summary = table.currentSnapshot().summary();
@@ -1032,7 +1032,7 @@ public class HiveIcebergStorageHandler implements 
HiveStoragePredicateHandler, H
 
   @Override
   public boolean isValidMetadataTable(String metaTableName) {
-    return IcebergMetadataTables.isValidMetaTable(metaTableName);
+    return metaTableName != null && 
IcebergMetadataTables.isValidMetaTable(metaTableName);
   }
 
   @Override
@@ -1512,7 +1512,7 @@ public class HiveIcebergStorageHandler implements 
HiveStoragePredicateHandler, H
   private void fallbackToNonVectorizedModeBasedOnProperties(Properties 
tableProps) {
     Schema tableSchema = 
SchemaParser.fromJson(tableProps.getProperty(InputFormatConfig.TABLE_SCHEMA));
     if 
(FileFormat.AVRO.name().equalsIgnoreCase(tableProps.getProperty(TableProperties.DEFAULT_FILE_FORMAT))
 ||
-        (tableProps.containsKey("metaTable") && 
isValidMetadataTable(tableProps.getProperty("metaTable"))) ||
+        
isValidMetadataTable(tableProps.getProperty(IcebergAcidUtil.META_TABLE_PROPERTY))
 ||
         hasOrcTimeInSchema(tableProps, tableSchema) ||
         !hasParquetNestedTypeWithinListOrMap(tableProps, tableSchema)) {
       conf.setBoolean(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED.varname, 
false);
diff --git 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java
 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java
index 43195e38846..b4f9566ad15 100644
--- 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java
+++ 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java
@@ -48,6 +48,7 @@ public class IcebergAcidUtil {
   private static final Types.NestedField PARTITION_STRUCT_META_COL = null; // 
placeholder value in the map
   private static final Map<Types.NestedField, Integer> FILE_READ_META_COLS = 
Maps.newLinkedHashMap();
   private static final Map<String, Types.NestedField> 
VIRTUAL_COLS_TO_META_COLS = Maps.newLinkedHashMap();
+  public static final String META_TABLE_PROPERTY = "metaTable";
 
   static {
     FILE_READ_META_COLS.put(MetadataColumns.SPEC_ID, 0);
diff --git 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
index 17576edc9f2..a5df4d7b941 100644
--- 
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
+++ 
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java
@@ -99,7 +99,7 @@ public class IcebergTableUtil {
    * @return an Iceberg table
    */
   static Table getTable(Configuration configuration, Properties properties, 
boolean skipCache) {
-    String metaTable = properties.getProperty("metaTable");
+    String metaTable = 
properties.getProperty(IcebergAcidUtil.META_TABLE_PROPERTY);
     String tableName = properties.getProperty(Catalogs.NAME);
     String location = properties.getProperty(Catalogs.LOCATION);
     if (metaTable != null) {
diff --git 
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_query_metadata.q
 
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_query_metadata.q
new file mode 100644
index 00000000000..f7240dd12ea
--- /dev/null
+++ 
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_query_metadata.q
@@ -0,0 +1,43 @@
+-- Mask random uuid
+--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/
+-- Mask a random snapshot id
+--! qt:replace:/(\s+current-snapshot-id\s+)\S+(\s*)/$1#Masked#/
+-- Mask current-snapshot-timestamp-ms
+--! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/
+-- Mask added file size
+--! qt:replace:/(\S\"added-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
+-- Mask total file size
+--! qt:replace:/(\S\"total-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
+-- Mask show compactions fields that change across runs
+--! 
qt:replace:/(MAJOR\s+succeeded\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/
+
+set hive.llap.io.enabled=true;
+set hive.vectorized.execution.enabled=true;
+set hive.optimize.shared.work.merge.ts.schema=true;
+set iceberg.mr.schema.auto.conversion=true;
+
+CREATE TABLE x (name VARCHAR(50), age TINYINT, num_clicks BIGINT) 
+stored by iceberg stored as orc 
+TBLPROPERTIES ('external.table.purge'='true','format-version'='2');
+
+insert into x values 
+('amy', 35, 123412344),
+('adxfvy', 36, 123412534),
+('amsdfyy', 37, 123417234),
+('asafmy', 38, 123412534);
+
+insert into x values 
+('amerqwy', 39, 123441234),
+('amyxzcv', 40, 123341234),
+('erweramy', 45, 122341234);
+
+select * from default.x.data_files;
+select count(*) from default.x.data_files;
+
+alter table x compact 'major' and wait;
+
+show compactions;
+desc formatted x;
+
+select * from default.x.data_files;
+select count(*) from default.x.data_files;
diff --git 
a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out
 
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out
new file mode 100644
index 00000000000..4e369d322ef
--- /dev/null
+++ 
b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_query_metadata.q.out
@@ -0,0 +1,142 @@
+PREHOOK: query: CREATE TABLE x (name VARCHAR(50), age TINYINT, num_clicks 
BIGINT) 
+stored by iceberg stored as orc 
+TBLPROPERTIES ('external.table.purge'='true','format-version'='2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@x
+POSTHOOK: query: CREATE TABLE x (name VARCHAR(50), age TINYINT, num_clicks 
BIGINT) 
+stored by iceberg stored as orc 
+TBLPROPERTIES ('external.table.purge'='true','format-version'='2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@x
+PREHOOK: query: insert into x values 
+('amy', 35, 123412344),
+('adxfvy', 36, 123412534),
+('amsdfyy', 37, 123417234),
+('asafmy', 38, 123412534)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@x
+POSTHOOK: query: insert into x values 
+('amy', 35, 123412344),
+('adxfvy', 36, 123412534),
+('amsdfyy', 37, 123417234),
+('asafmy', 38, 123412534)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@x
+PREHOOK: query: insert into x values 
+('amerqwy', 39, 123441234),
+('amyxzcv', 40, 123341234),
+('erweramy', 45, 122341234)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@x
+POSTHOOK: query: insert into x values 
+('amerqwy', 39, 123441234),
+('amyxzcv', 40, 123341234),
+('erweramy', 45, 122341234)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@x
+PREHOOK: query: select * from default.x.data_files
+PREHOOK: type: QUERY
+PREHOOK: Input: default@x
+#### A masked pattern was here ####
+POSTHOOK: query: select * from default.x.data_files
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@x
+#### A masked pattern was here ####
+PREHOOK: query: select count(*) from default.x.data_files
+PREHOOK: type: QUERY
+PREHOOK: Input: default@x
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from default.x.data_files
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@x
+#### A masked pattern was here ####
+2
+PREHOOK: query: alter table x compact 'major' and wait
+PREHOOK: type: ALTERTABLE_COMPACT
+PREHOOK: Input: default@x
+PREHOOK: Output: default@x
+POSTHOOK: query: alter table x compact 'major' and wait
+POSTHOOK: type: ALTERTABLE_COMPACT
+POSTHOOK: Input: default@x
+POSTHOOK: Output: default@x
+PREHOOK: query: show compactions
+PREHOOK: type: SHOW COMPACTIONS
+POSTHOOK: query: show compactions
+POSTHOOK: type: SHOW COMPACTIONS
+CompactionId   Database        Table   Partition       Type    State   Worker 
host     Worker  Enqueue Time    Start Time      Duration(ms)    HadoopJobId    
 Error message   Initiator host  Initiator       Pool name       TxnId   Next 
TxnId      Commit Time     Highest WriteId
+1      default x        ---    MAJOR   succeeded       #Masked#        manual  
default 0       0       0        --- 
+PREHOOK: query: desc formatted x
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@x
+POSTHOOK: query: desc formatted x
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@x
+# col_name             data_type               comment             
+name                   string                                      
+age                    int                                         
+num_clicks             bigint                                      
+                
+# Detailed Table Information            
+Database:              default                  
+#### A masked pattern was here ####
+Retention:             0                        
+#### A masked pattern was here ####
+Table Type:            EXTERNAL_TABLE           
+Table Parameters:               
+       COLUMN_STATS_ACCURATE   
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"age\":\"true\",\"name\":\"true\",\"num_clicks\":\"true\"}}
+       EXTERNAL                TRUE                
+       bucketing_version       2                   
+       current-schema          
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"age\",\"required\":false,\"type\":\"int\"},{\"id\":3,\"name\":\"num_clicks\",\"required\":false,\"type\":\"long\"}]}
+       current-snapshot-id     #Masked#
+       current-snapshot-summary        
{\"replace-partitions\":\"true\",\"added-data-files\":\"1\",\"added-records\":\"7\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"7\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\"}
+       current-snapshot-timestamp-ms   #Masked#       
+       external.table.purge    true                
+       format-version          2                   
+       iceberg.orc.files.only  true                
+#### A masked pattern was here ####
+       numFiles                1                   
+       numRows                 7                   
+       parquet.compression     zstd                
+#### A masked pattern was here ####
+       rawDataSize             0                   
+       serialization.format    1                   
+       snapshot-count          4                   
+       storage_handler         
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
+       table_type              ICEBERG             
+       totalSize               #Masked#
+#### A masked pattern was here ####
+       uuid                    #Masked#
+       write.delete.mode       merge-on-read       
+       write.format.default    orc                 
+       write.merge.mode        merge-on-read       
+       write.update.mode       merge-on-read       
+                
+# Storage Information           
+SerDe Library:         org.apache.iceberg.mr.hive.HiveIcebergSerDe      
+InputFormat:           org.apache.iceberg.mr.hive.HiveIcebergInputFormat       
 
+OutputFormat:          org.apache.iceberg.mr.hive.HiveIcebergOutputFormat      
 
+Compressed:            No                       
+Sort Columns:          []                       
+PREHOOK: query: select * from default.x.data_files
+PREHOOK: type: QUERY
+PREHOOK: Input: default@x
+#### A masked pattern was here ####
+POSTHOOK: query: select * from default.x.data_files
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@x
+#### A masked pattern was here ####
+PREHOOK: query: select count(*) from default.x.data_files
+PREHOOK: type: QUERY
+PREHOOK: Input: default@x
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from default.x.data_files
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@x
+#### A masked pattern was here ####
+1
diff --git a/itests/src/test/resources/testconfiguration.properties 
b/itests/src/test/resources/testconfiguration.properties
index 50e16a912f3..7e138e1346c 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -422,6 +422,7 @@ iceberg.llap.query.files=\
 iceberg.llap.query.compactor.files=\
   iceberg_major_compaction_partition_evolution.q,\
   iceberg_major_compaction_partitioned.q,\
+  iceberg_major_compaction_query_metadata.q,\
   iceberg_major_compaction_schema_evolution.q,\
   iceberg_major_compaction_single_partition.q,\
   iceberg_major_compaction_unpartitioned.q,\

Reply via email to