This is an automated email from the ASF dual-hosted git repository.

zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 0e287d4b45b HIVE-28938: Error in LATERAL VIEW with non native tables 
due to incorrect virtual columns (#5798)
0e287d4b45b is described below

commit 0e287d4b45b0ccf14c74353f01abd2e7dfdce62e
Author: Soumyakanti Das <[email protected]>
AuthorDate: Mon May 12 04:30:38 2025 -0700

    HIVE-28938: Error in LATERAL VIEW with non native tables due to incorrect 
virtual columns (#5798)
---
 .../src/test/results/positive/hbase_queries.q.out  | 34 +++++++++---------
 .../queries/positive/iceberg_explain_formatted.q   |  3 ++
 .../positive/iceberg_multiple_lateral_views.q      |  6 ++++
 .../positive/iceberg_explain_formatted.q.out       | 17 +++++++++
 .../positive/iceberg_multiple_lateral_views.q.out  | 40 ++++++++++++++++++++++
 .../results/positive/kudu_complex_queries.q.out    | 26 +++++++-------
 .../org/apache/hadoop/hive/ql/metadata/Table.java  | 19 ++++++++++
 .../hadoop/hive/ql/parse/CalcitePlanner.java       | 26 +++++---------
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java     |  8 +----
 9 files changed, 124 insertions(+), 55 deletions(-)

diff --git a/hbase-handler/src/test/results/positive/hbase_queries.q.out 
b/hbase-handler/src/test/results/positive/hbase_queries.q.out
index 21e28305697..9eb409dc515 100644
--- a/hbase-handler/src/test/results/positive/hbase_queries.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_queries.q.out
@@ -151,14 +151,14 @@ STAGE PLANS:
               predicate: UDFToDouble(key) is not null (type: boolean)
               Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE 
Column stats: NONE
               Select Operator
-                expressions: key (type: int)
+                expressions: UDFToDouble(key) (type: double)
                 outputColumnNames: _col0
                 Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE 
Column stats: NONE
                 Reduce Output Operator
-                  key expressions: UDFToDouble(_col0) (type: double)
+                  key expressions: _col0 (type: double)
                   null sort order: z
                   sort order: +
-                  Map-reduce partition columns: UDFToDouble(_col0) (type: 
double)
+                  Map-reduce partition columns: _col0 (type: double)
                   Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE 
Column stats: NONE
           TableScan
             alias: src
@@ -168,29 +168,29 @@ STAGE PLANS:
               predicate: UDFToDouble(key) is not null (type: boolean)
               Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE 
Column stats: COMPLETE
               Select Operator
-                expressions: key (type: string), value (type: string)
-                outputColumnNames: _col0, _col1
-                Statistics: Num rows: 500 Data size: 89000 Basic stats: 
COMPLETE Column stats: COMPLETE
+                expressions: key (type: string), value (type: string), 
UDFToDouble(key) (type: double)
+                outputColumnNames: _col0, _col1, _col2
+                Statistics: Num rows: 500 Data size: 93000 Basic stats: 
COMPLETE Column stats: COMPLETE
                 Reduce Output Operator
-                  key expressions: UDFToDouble(_col0) (type: double)
+                  key expressions: _col2 (type: double)
                   null sort order: z
                   sort order: +
-                  Map-reduce partition columns: UDFToDouble(_col0) (type: 
double)
-                  Statistics: Num rows: 500 Data size: 89000 Basic stats: 
COMPLETE Column stats: COMPLETE
+                  Map-reduce partition columns: _col2 (type: double)
+                  Statistics: Num rows: 500 Data size: 93000 Basic stats: 
COMPLETE Column stats: COMPLETE
                   value expressions: _col0 (type: string), _col1 (type: string)
       Reduce Operator Tree:
         Join Operator
           condition map:
                Inner Join 0 to 1
           keys:
-            0 UDFToDouble(_col0) (type: double)
-            1 UDFToDouble(_col0) (type: double)
+            0 _col0 (type: double)
+            1 _col2 (type: double)
           outputColumnNames: _col1, _col2
-          Statistics: Num rows: 550 Data size: 97900 Basic stats: COMPLETE 
Column stats: NONE
+          Statistics: Num rows: 550 Data size: 102300 Basic stats: COMPLETE 
Column stats: NONE
           Select Operator
             expressions: _col1 (type: string), _col2 (type: string)
             outputColumnNames: _col0, _col1
-            Statistics: Num rows: 550 Data size: 97900 Basic stats: COMPLETE 
Column stats: NONE
+            Statistics: Num rows: 550 Data size: 102300 Basic stats: COMPLETE 
Column stats: NONE
             File Output Operator
               compressed: false
               table:
@@ -206,20 +206,20 @@ STAGE PLANS:
               key expressions: _col0 (type: string), _col1 (type: string)
               null sort order: zz
               sort order: ++
-              Statistics: Num rows: 550 Data size: 97900 Basic stats: COMPLETE 
Column stats: NONE
+              Statistics: Num rows: 550 Data size: 102300 Basic stats: 
COMPLETE Column stats: NONE
               TopN Hash Memory Usage: 0.1
       Execution mode: vectorized
       Reduce Operator Tree:
         Select Operator
           expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 
(type: string)
           outputColumnNames: _col0, _col1
-          Statistics: Num rows: 550 Data size: 97900 Basic stats: COMPLETE 
Column stats: NONE
+          Statistics: Num rows: 550 Data size: 102300 Basic stats: COMPLETE 
Column stats: NONE
           Limit
             Number of rows: 20
-            Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE 
Column stats: NONE
+            Statistics: Num rows: 20 Data size: 3720 Basic stats: COMPLETE 
Column stats: NONE
             File Output Operator
               compressed: false
-              Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE 
Column stats: NONE
+              Statistics: Num rows: 20 Data size: 3720 Basic stats: COMPLETE 
Column stats: NONE
               table:
                   input format: 
org.apache.hadoop.mapred.SequenceFileInputFormat
                   output format: 
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git 
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_explain_formatted.q 
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_explain_formatted.q
new file mode 100644
index 00000000000..8150ddbba6e
--- /dev/null
+++ 
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_explain_formatted.q
@@ -0,0 +1,3 @@
+-- A simple explain formatted test for an iceberg table to check virtual 
columns in the JSON output.
+create external table test (a int, b int) stored by iceberg;
+explain formatted select * from test;
diff --git 
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_multiple_lateral_views.q
 
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_multiple_lateral_views.q
new file mode 100644
index 00000000000..5d7a959efde
--- /dev/null
+++ 
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_multiple_lateral_views.q
@@ -0,0 +1,6 @@
+create external table test(id int, arr array<string>) stored by iceberg;
+insert into test values (1, array("a", "b")), (2, array("c", "d")), (3, 
array("e", "f"));
+
+select * from test
+lateral view explode(arr) tbl1 as name
+lateral view explode(arr) tbl2 as name1;
diff --git 
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out
 
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out
new file mode 100644
index 00000000000..ee00aa71a4e
--- /dev/null
+++ 
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out
@@ -0,0 +1,17 @@
+PREHOOK: query: create external table test (a int, b int) stored by iceberg
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test
+POSTHOOK: query: create external table test (a int, b int) stored by iceberg
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test
+PREHOOK: query: explain formatted select * from test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain formatted select * from test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+{"CBOPlan":"{\n  \"rels\": [\n    {\n      \"id\": \"0\",\n      \"relOp\": 
\"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n   
   \"table\": [\n        \"default\",\n        \"test\"\n      ],\n      
\"table:alias\": \"test\",\n      \"inputs\": [],\n      \"rowCount\": 1.0,\n   
   \"avgRowSize\": 8.0,\n      \"rowType\": {\n        \"fields\": [\n          
{\n            \"type\": \"INTEGER\",\n            \"nullable\": true,\n        
    \"name\": \"a\"\n   [...]
diff --git 
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_multiple_lateral_views.q.out
 
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_multiple_lateral_views.q.out
new file mode 100644
index 00000000000..0cb7d2c3ab7
--- /dev/null
+++ 
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_multiple_lateral_views.q.out
@@ -0,0 +1,40 @@
+PREHOOK: query: create external table test(id int, arr array<string>) stored 
by iceberg
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test
+POSTHOOK: query: create external table test(id int, arr array<string>) stored 
by iceberg
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test
+PREHOOK: query: insert into test values (1, array("a", "b")), (2, array("c", 
"d")), (3, array("e", "f"))
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test
+POSTHOOK: query: insert into test values (1, array("a", "b")), (2, array("c", 
"d")), (3, array("e", "f"))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test
+PREHOOK: query: select * from test
+lateral view explode(arr) tbl1 as name
+lateral view explode(arr) tbl2 as name1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from test
+lateral view explode(arr) tbl1 as name
+lateral view explode(arr) tbl2 as name1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1      ["a","b"]       a       a
+1      ["a","b"]       a       b
+1      ["a","b"]       b       a
+1      ["a","b"]       b       b
+2      ["c","d"]       c       c
+2      ["c","d"]       c       d
+2      ["c","d"]       d       c
+2      ["c","d"]       d       d
+3      ["e","f"]       e       e
+3      ["e","f"]       e       f
+3      ["e","f"]       f       e
+3      ["e","f"]       f       f
diff --git a/kudu-handler/src/test/results/positive/kudu_complex_queries.q.out 
b/kudu-handler/src/test/results/positive/kudu_complex_queries.q.out
index 29942035010..4cd80f53e75 100644
--- a/kudu-handler/src/test/results/positive/kudu_complex_queries.q.out
+++ b/kudu-handler/src/test/results/positive/kudu_complex_queries.q.out
@@ -91,15 +91,15 @@ STAGE PLANS:
                     predicate: UDFToDouble(key) is not null (type: boolean)
                     Statistics: Num rows: 309 Data size: 1236 Basic stats: 
COMPLETE Column stats: COMPLETE
                     Select Operator
-                      expressions: key (type: int)
+                      expressions: UDFToDouble(key) (type: double)
                       outputColumnNames: _col0
-                      Statistics: Num rows: 309 Data size: 1236 Basic stats: 
COMPLETE Column stats: COMPLETE
+                      Statistics: Num rows: 309 Data size: 2472 Basic stats: 
COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
-                        key expressions: UDFToDouble(_col0) (type: double)
+                        key expressions: _col0 (type: double)
                         null sort order: z
                         sort order: +
-                        Map-reduce partition columns: UDFToDouble(_col0) 
(type: double)
-                        Statistics: Num rows: 309 Data size: 1236 Basic stats: 
COMPLETE Column stats: COMPLETE
+                        Map-reduce partition columns: _col0 (type: double)
+                        Statistics: Num rows: 309 Data size: 2472 Basic stats: 
COMPLETE Column stats: COMPLETE
             Execution mode: vectorized
         Map 4 
             Map Operator Tree:
@@ -111,15 +111,15 @@ STAGE PLANS:
                     predicate: UDFToDouble(key) is not null (type: boolean)
                     Statistics: Num rows: 500 Data size: 89000 Basic stats: 
COMPLETE Column stats: COMPLETE
                     Select Operator
-                      expressions: key (type: string), value (type: string)
-                      outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 500 Data size: 89000 Basic stats: 
COMPLETE Column stats: COMPLETE
+                      expressions: key (type: string), value (type: string), 
UDFToDouble(key) (type: double)
+                      outputColumnNames: _col0, _col1, _col2
+                      Statistics: Num rows: 500 Data size: 93000 Basic stats: 
COMPLETE Column stats: COMPLETE
                       Reduce Output Operator
-                        key expressions: UDFToDouble(_col0) (type: double)
+                        key expressions: _col2 (type: double)
                         null sort order: z
                         sort order: +
-                        Map-reduce partition columns: UDFToDouble(_col0) 
(type: double)
-                        Statistics: Num rows: 500 Data size: 89000 Basic 
stats: COMPLETE Column stats: COMPLETE
+                        Map-reduce partition columns: _col2 (type: double)
+                        Statistics: Num rows: 500 Data size: 93000 Basic 
stats: COMPLETE Column stats: COMPLETE
                         value expressions: _col0 (type: string), _col1 (type: 
string)
             Execution mode: vectorized
         Reducer 2 
@@ -128,8 +128,8 @@ STAGE PLANS:
                 condition map:
                      Inner Join 0 to 1
                 keys:
-                  0 UDFToDouble(_col0) (type: double)
-                  1 UDFToDouble(_col0) (type: double)
+                  0 _col0 (type: double)
+                  1 _col2 (type: double)
                 outputColumnNames: _col1, _col2
                 Statistics: Num rows: 488 Data size: 86864 Basic stats: 
COMPLETE Column stats: COMPLETE
                 Top N Key Operator
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java 
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
index 705082d7caf..a346eb76cc3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
@@ -61,6 +61,7 @@
 import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
 import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
 import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
 import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat;
 import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec;
@@ -85,6 +86,8 @@
 
 import com.google.common.base.Preconditions;
 
+import static org.apache.commons.lang3.StringUtils.isBlank;
+
 /**
  * A Hive Table: is a fundamental unit of data in Hive that shares a common 
schema/DDL.
  *
@@ -1364,4 +1367,20 @@ public SourceTable createSourceTable() {
     sourceTable.setDeletedCount(0L);
     return sourceTable;
   }
+
+  public List<VirtualColumn> getVirtualColumns(HiveConf conf) {
+    List<VirtualColumn> virtualColumns = new ArrayList<>();
+    if (!isNonNative()) {
+      virtualColumns.addAll(VirtualColumn.getRegistry(conf));
+    }
+    if (isNonNative() && AcidUtils.isNonNativeAcidTable(this)) {
+      virtualColumns.addAll(getStorageHandler().acidVirtualColumns());
+    }
+    if (isNonNative() && getStorageHandler().areSnapshotsSupported() &&
+        isBlank(getMetaTable())) {
+      virtualColumns.add(VirtualColumn.SNAPSHOT_ID);
+    }
+    
+    return virtualColumns;
+  }
 }
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index cca4ac3c6b8..bd5c8012ca1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -366,7 +366,6 @@
 
 import javax.sql.DataSource;
 
-import static org.apache.commons.lang3.StringUtils.isBlank;
 import static 
org.apache.hadoop.hive.ql.optimizer.calcite.HiveMaterializedViewASTSubQueryRewriteShuttle.getMaterializedViewByAST;
 import static org.apache.hadoop.hive.ql.metadata.RewriteAlgorithm.ANY;
 
@@ -3094,23 +3093,14 @@ private RelNode genTableLogicalPlan(String tableAlias, 
QB qb) throws SemanticExc
         final TableType tableType = obtainTableType(tabMetaData);
 
         // 3.3 Add column info corresponding to virtual columns
-        List<VirtualColumn> virtualCols = new ArrayList<>();
-        if (tableType == TableType.NATIVE) {
-          virtualCols = VirtualColumn.getRegistry(conf);
-          if (AcidUtils.isNonNativeAcidTable(tabMetaData)) {
-            
virtualCols.addAll(tabMetaData.getStorageHandler().acidVirtualColumns());
-          }
-          if (tabMetaData.isNonNative() && 
tabMetaData.getStorageHandler().areSnapshotsSupported() &&
-              isBlank(tabMetaData.getMetaTable())) {
-            virtualCols.add(VirtualColumn.SNAPSHOT_ID);
-          }
-          for (VirtualColumn vc : virtualCols) {
-            colInfo = new ColumnInfo(vc.getName(), vc.getTypeInfo(), 
tableAlias, true,
-                vc.getIsHidden());
-            rr.put(tableAlias, vc.getName().toLowerCase(), colInfo);
-            cInfoLst.add(colInfo);
-          }
-        }
+        List<VirtualColumn> virtualCols = tabMetaData.getVirtualColumns(conf);
+
+        virtualCols
+            .forEach(vc ->
+                rr.put(tableAlias, vc.getName().toLowerCase(),
+                    new ColumnInfo(vc.getName(), vc.getTypeInfo(), tableAlias, 
true, vc.getIsHidden())
+                )
+            );
 
         // 4. Build operator
         Map<String, String> tabPropsFromQuery = 
qb.getTabPropsForAlias(tableAlias);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java 
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 908f06ab60d..3532c75ecb8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -11988,13 +11988,7 @@ private Operator genTablePlan(String alias, QB qb) 
throws SemanticException {
       }
 
       // put virtual columns into RowResolver.
-      List<VirtualColumn> vcList = new ArrayList<>();
-      if (!tab.isNonNative()) {
-        vcList.addAll(VirtualColumn.getRegistry(conf));
-      }
-      if (tab.isNonNative() && AcidUtils.isNonNativeAcidTable(tab)) {
-        vcList.addAll(tab.getStorageHandler().acidVirtualColumns());
-      }
+      List<VirtualColumn> vcList = tab.getVirtualColumns(conf);
 
       vcList.forEach(vc -> rwsch.put(alias, vc.getName().toLowerCase(), new 
ColumnInfo(vc.getName(),
               vc.getTypeInfo(), alias, true, vc.getIsHidden()

Reply via email to