This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 0e287d4b45b HIVE-28938: Error in LATERAL VIEW with non native tables
due to incorrect virtual columns (#5798)
0e287d4b45b is described below
commit 0e287d4b45b0ccf14c74353f01abd2e7dfdce62e
Author: Soumyakanti Das <[email protected]>
AuthorDate: Mon May 12 04:30:38 2025 -0700
HIVE-28938: Error in LATERAL VIEW with non native tables due to incorrect
virtual columns (#5798)
---
.../src/test/results/positive/hbase_queries.q.out | 34 +++++++++---------
.../queries/positive/iceberg_explain_formatted.q | 3 ++
.../positive/iceberg_multiple_lateral_views.q | 6 ++++
.../positive/iceberg_explain_formatted.q.out | 17 +++++++++
.../positive/iceberg_multiple_lateral_views.q.out | 40 ++++++++++++++++++++++
.../results/positive/kudu_complex_queries.q.out | 26 +++++++-------
.../org/apache/hadoop/hive/ql/metadata/Table.java | 19 ++++++++++
.../hadoop/hive/ql/parse/CalcitePlanner.java | 26 +++++---------
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 8 +----
9 files changed, 124 insertions(+), 55 deletions(-)
diff --git a/hbase-handler/src/test/results/positive/hbase_queries.q.out
b/hbase-handler/src/test/results/positive/hbase_queries.q.out
index 21e28305697..9eb409dc515 100644
--- a/hbase-handler/src/test/results/positive/hbase_queries.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_queries.q.out
@@ -151,14 +151,14 @@ STAGE PLANS:
predicate: UDFToDouble(key) is not null (type: boolean)
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE
Column stats: NONE
Select Operator
- expressions: key (type: int)
+ expressions: UDFToDouble(key) (type: double)
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE
Column stats: NONE
Reduce Output Operator
- key expressions: UDFToDouble(_col0) (type: double)
+ key expressions: _col0 (type: double)
null sort order: z
sort order: +
- Map-reduce partition columns: UDFToDouble(_col0) (type:
double)
+ Map-reduce partition columns: _col0 (type: double)
Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE
Column stats: NONE
TableScan
alias: src
@@ -168,29 +168,29 @@ STAGE PLANS:
predicate: UDFToDouble(key) is not null (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE
Column stats: COMPLETE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 89000 Basic stats:
COMPLETE Column stats: COMPLETE
+ expressions: key (type: string), value (type: string),
UDFToDouble(key) (type: double)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 93000 Basic stats:
COMPLETE Column stats: COMPLETE
Reduce Output Operator
- key expressions: UDFToDouble(_col0) (type: double)
+ key expressions: _col2 (type: double)
null sort order: z
sort order: +
- Map-reduce partition columns: UDFToDouble(_col0) (type:
double)
- Statistics: Num rows: 500 Data size: 89000 Basic stats:
COMPLETE Column stats: COMPLETE
+ Map-reduce partition columns: _col2 (type: double)
+ Statistics: Num rows: 500 Data size: 93000 Basic stats:
COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string), _col1 (type: string)
Reduce Operator Tree:
Join Operator
condition map:
Inner Join 0 to 1
keys:
- 0 UDFToDouble(_col0) (type: double)
- 1 UDFToDouble(_col0) (type: double)
+ 0 _col0 (type: double)
+ 1 _col2 (type: double)
outputColumnNames: _col1, _col2
- Statistics: Num rows: 550 Data size: 97900 Basic stats: COMPLETE
Column stats: NONE
+ Statistics: Num rows: 550 Data size: 102300 Basic stats: COMPLETE
Column stats: NONE
Select Operator
expressions: _col1 (type: string), _col2 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 550 Data size: 97900 Basic stats: COMPLETE
Column stats: NONE
+ Statistics: Num rows: 550 Data size: 102300 Basic stats: COMPLETE
Column stats: NONE
File Output Operator
compressed: false
table:
@@ -206,20 +206,20 @@ STAGE PLANS:
key expressions: _col0 (type: string), _col1 (type: string)
null sort order: zz
sort order: ++
- Statistics: Num rows: 550 Data size: 97900 Basic stats: COMPLETE
Column stats: NONE
+ Statistics: Num rows: 550 Data size: 102300 Basic stats:
COMPLETE Column stats: NONE
TopN Hash Memory Usage: 0.1
Execution mode: vectorized
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1
(type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 550 Data size: 97900 Basic stats: COMPLETE
Column stats: NONE
+ Statistics: Num rows: 550 Data size: 102300 Basic stats: COMPLETE
Column stats: NONE
Limit
Number of rows: 20
- Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE
Column stats: NONE
+ Statistics: Num rows: 20 Data size: 3720 Basic stats: COMPLETE
Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 20 Data size: 3560 Basic stats: COMPLETE
Column stats: NONE
+ Statistics: Num rows: 20 Data size: 3720 Basic stats: COMPLETE
Column stats: NONE
table:
input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_explain_formatted.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_explain_formatted.q
new file mode 100644
index 00000000000..8150ddbba6e
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_explain_formatted.q
@@ -0,0 +1,3 @@
+-- A simple explain formatted test for an iceberg table to check virtual
columns in the JSON output.
+create external table test (a int, b int) stored by iceberg;
+explain formatted select * from test;
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_multiple_lateral_views.q
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_multiple_lateral_views.q
new file mode 100644
index 00000000000..5d7a959efde
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_multiple_lateral_views.q
@@ -0,0 +1,6 @@
+create external table test(id int, arr array<string>) stored by iceberg;
+insert into test values (1, array("a", "b")), (2, array("c", "d")), (3,
array("e", "f"));
+
+select * from test
+lateral view explode(arr) tbl1 as name
+lateral view explode(arr) tbl2 as name1;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out
new file mode 100644
index 00000000000..ee00aa71a4e
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_explain_formatted.q.out
@@ -0,0 +1,17 @@
+PREHOOK: query: create external table test (a int, b int) stored by iceberg
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test
+POSTHOOK: query: create external table test (a int, b int) stored by iceberg
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test
+PREHOOK: query: explain formatted select * from test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: explain formatted select * from test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\":
\"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n
\"table\": [\n \"default\",\n \"test\"\n ],\n
\"table:alias\": \"test\",\n \"inputs\": [],\n \"rowCount\": 1.0,\n
\"avgRowSize\": 8.0,\n \"rowType\": {\n \"fields\": [\n
{\n \"type\": \"INTEGER\",\n \"nullable\": true,\n
\"name\": \"a\"\n [...]
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/iceberg_multiple_lateral_views.q.out
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_multiple_lateral_views.q.out
new file mode 100644
index 00000000000..0cb7d2c3ab7
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/results/positive/iceberg_multiple_lateral_views.q.out
@@ -0,0 +1,40 @@
+PREHOOK: query: create external table test(id int, arr array<string>) stored
by iceberg
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test
+POSTHOOK: query: create external table test(id int, arr array<string>) stored
by iceberg
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test
+PREHOOK: query: insert into test values (1, array("a", "b")), (2, array("c",
"d")), (3, array("e", "f"))
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test
+POSTHOOK: query: insert into test values (1, array("a", "b")), (2, array("c",
"d")), (3, array("e", "f"))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test
+PREHOOK: query: select * from test
+lateral view explode(arr) tbl1 as name
+lateral view explode(arr) tbl2 as name1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from test
+lateral view explode(arr) tbl1 as name
+lateral view explode(arr) tbl2 as name1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 ["a","b"] a a
+1 ["a","b"] a b
+1 ["a","b"] b a
+1 ["a","b"] b b
+2 ["c","d"] c c
+2 ["c","d"] c d
+2 ["c","d"] d c
+2 ["c","d"] d d
+3 ["e","f"] e e
+3 ["e","f"] e f
+3 ["e","f"] f e
+3 ["e","f"] f f
diff --git a/kudu-handler/src/test/results/positive/kudu_complex_queries.q.out
b/kudu-handler/src/test/results/positive/kudu_complex_queries.q.out
index 29942035010..4cd80f53e75 100644
--- a/kudu-handler/src/test/results/positive/kudu_complex_queries.q.out
+++ b/kudu-handler/src/test/results/positive/kudu_complex_queries.q.out
@@ -91,15 +91,15 @@ STAGE PLANS:
predicate: UDFToDouble(key) is not null (type: boolean)
Statistics: Num rows: 309 Data size: 1236 Basic stats:
COMPLETE Column stats: COMPLETE
Select Operator
- expressions: key (type: int)
+ expressions: UDFToDouble(key) (type: double)
outputColumnNames: _col0
- Statistics: Num rows: 309 Data size: 1236 Basic stats:
COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 309 Data size: 2472 Basic stats:
COMPLETE Column stats: COMPLETE
Reduce Output Operator
- key expressions: UDFToDouble(_col0) (type: double)
+ key expressions: _col0 (type: double)
null sort order: z
sort order: +
- Map-reduce partition columns: UDFToDouble(_col0)
(type: double)
- Statistics: Num rows: 309 Data size: 1236 Basic stats:
COMPLETE Column stats: COMPLETE
+ Map-reduce partition columns: _col0 (type: double)
+ Statistics: Num rows: 309 Data size: 2472 Basic stats:
COMPLETE Column stats: COMPLETE
Execution mode: vectorized
Map 4
Map Operator Tree:
@@ -111,15 +111,15 @@ STAGE PLANS:
predicate: UDFToDouble(key) is not null (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats:
COMPLETE Column stats: COMPLETE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 500 Data size: 89000 Basic stats:
COMPLETE Column stats: COMPLETE
+ expressions: key (type: string), value (type: string),
UDFToDouble(key) (type: double)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 93000 Basic stats:
COMPLETE Column stats: COMPLETE
Reduce Output Operator
- key expressions: UDFToDouble(_col0) (type: double)
+ key expressions: _col2 (type: double)
null sort order: z
sort order: +
- Map-reduce partition columns: UDFToDouble(_col0)
(type: double)
- Statistics: Num rows: 500 Data size: 89000 Basic
stats: COMPLETE Column stats: COMPLETE
+ Map-reduce partition columns: _col2 (type: double)
+ Statistics: Num rows: 500 Data size: 93000 Basic
stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string), _col1 (type:
string)
Execution mode: vectorized
Reducer 2
@@ -128,8 +128,8 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 UDFToDouble(_col0) (type: double)
- 1 UDFToDouble(_col0) (type: double)
+ 0 _col0 (type: double)
+ 1 _col2 (type: double)
outputColumnNames: _col1, _col2
Statistics: Num rows: 488 Data size: 86864 Basic stats:
COMPLETE Column stats: COMPLETE
Top N Key Operator
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
index 705082d7caf..a346eb76cc3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
@@ -61,6 +61,7 @@
import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
import org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec;
@@ -85,6 +86,8 @@
import com.google.common.base.Preconditions;
+import static org.apache.commons.lang3.StringUtils.isBlank;
+
/**
* A Hive Table: is a fundamental unit of data in Hive that shares a common
schema/DDL.
*
@@ -1364,4 +1367,20 @@ public SourceTable createSourceTable() {
sourceTable.setDeletedCount(0L);
return sourceTable;
}
+
+ public List<VirtualColumn> getVirtualColumns(HiveConf conf) {
+ List<VirtualColumn> virtualColumns = new ArrayList<>();
+ if (!isNonNative()) {
+ virtualColumns.addAll(VirtualColumn.getRegistry(conf));
+ }
+ if (isNonNative() && AcidUtils.isNonNativeAcidTable(this)) {
+ virtualColumns.addAll(getStorageHandler().acidVirtualColumns());
+ }
+ if (isNonNative() && getStorageHandler().areSnapshotsSupported() &&
+ isBlank(getMetaTable())) {
+ virtualColumns.add(VirtualColumn.SNAPSHOT_ID);
+ }
+
+ return virtualColumns;
+ }
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index cca4ac3c6b8..bd5c8012ca1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -366,7 +366,6 @@
import javax.sql.DataSource;
-import static org.apache.commons.lang3.StringUtils.isBlank;
import static
org.apache.hadoop.hive.ql.optimizer.calcite.HiveMaterializedViewASTSubQueryRewriteShuttle.getMaterializedViewByAST;
import static org.apache.hadoop.hive.ql.metadata.RewriteAlgorithm.ANY;
@@ -3094,23 +3093,14 @@ private RelNode genTableLogicalPlan(String tableAlias,
QB qb) throws SemanticExc
final TableType tableType = obtainTableType(tabMetaData);
// 3.3 Add column info corresponding to virtual columns
- List<VirtualColumn> virtualCols = new ArrayList<>();
- if (tableType == TableType.NATIVE) {
- virtualCols = VirtualColumn.getRegistry(conf);
- if (AcidUtils.isNonNativeAcidTable(tabMetaData)) {
-
virtualCols.addAll(tabMetaData.getStorageHandler().acidVirtualColumns());
- }
- if (tabMetaData.isNonNative() &&
tabMetaData.getStorageHandler().areSnapshotsSupported() &&
- isBlank(tabMetaData.getMetaTable())) {
- virtualCols.add(VirtualColumn.SNAPSHOT_ID);
- }
- for (VirtualColumn vc : virtualCols) {
- colInfo = new ColumnInfo(vc.getName(), vc.getTypeInfo(),
tableAlias, true,
- vc.getIsHidden());
- rr.put(tableAlias, vc.getName().toLowerCase(), colInfo);
- cInfoLst.add(colInfo);
- }
- }
+ List<VirtualColumn> virtualCols = tabMetaData.getVirtualColumns(conf);
+
+ virtualCols
+ .forEach(vc ->
+ rr.put(tableAlias, vc.getName().toLowerCase(),
+ new ColumnInfo(vc.getName(), vc.getTypeInfo(), tableAlias,
true, vc.getIsHidden())
+ )
+ );
// 4. Build operator
Map<String, String> tabPropsFromQuery =
qb.getTabPropsForAlias(tableAlias);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 908f06ab60d..3532c75ecb8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -11988,13 +11988,7 @@ private Operator genTablePlan(String alias, QB qb)
throws SemanticException {
}
// put virtual columns into RowResolver.
- List<VirtualColumn> vcList = new ArrayList<>();
- if (!tab.isNonNative()) {
- vcList.addAll(VirtualColumn.getRegistry(conf));
- }
- if (tab.isNonNative() && AcidUtils.isNonNativeAcidTable(tab)) {
- vcList.addAll(tab.getStorageHandler().acidVirtualColumns());
- }
+ List<VirtualColumn> vcList = tab.getVirtualColumns(conf);
vcList.forEach(vc -> rwsch.put(alias, vc.getName().toLowerCase(), new
ColumnInfo(vc.getName(),
vc.getTypeInfo(), alias, true, vc.getIsHidden()