This is an automated email from the ASF dual-hosted git repository.
kasakrisz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new d1282396679 HIVE-29786: Support SELECT * EXCLUDE syntax for column
omission (#6665)
d1282396679 is described below
commit d1282396679bc07d4dc45ca27677155d18b32a0c
Author: Raghav Aggarwal <[email protected]>
AuthorDate: Wed Aug 26 13:08:04 2026 +0530
HIVE-29786: Support SELECT * EXCLUDE syntax for column omission (#6665)
---
.../apache/hadoop/hive/ql/parse/FromClauseParser.g | 8 +-
.../apache/hadoop/hive/ql/parse/HiveLexerParent.g | 1 +
.../hadoop/hive/ql/parse/IdentifiersParser.g | 1 +
.../hadoop/hive/ql/parse/CalcitePlanner.java | 7 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 59 ++++-
.../test/queries/clientpositive/select_exclude.q | 39 +++
.../clientpositive/llap/select_exclude.q.out | 283 +++++++++++++++++++++
7 files changed, 389 insertions(+), 9 deletions(-)
diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
b/parser/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
index ad34eddbc1a..2822c66d1c1 100644
--- a/parser/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
+++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
@@ -48,10 +48,10 @@ catch (RecognitionException e) {
//-----------------------------------------------------------------------------------
tableAllColumns
- : STAR
- -> ^(TOK_ALLCOLREF)
- | tableName DOT STAR
- -> ^(TOK_ALLCOLREF tableName)
+ : STAR (KW_EXCLUDE LPAREN columnNameList RPAREN)?
+ -> ^(TOK_ALLCOLREF columnNameList?)
+ | tableName DOT STAR (KW_EXCLUDE LPAREN columnNameList RPAREN)?
+ -> ^(TOK_ALLCOLREF tableName columnNameList?)
;
// (table|column)
diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g
b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g
index a96812d4969..d784056d783 100644
--- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g
+++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveLexerParent.g
@@ -399,6 +399,7 @@ KW_SYSTEM_TIME: 'SYSTEM_TIME';
KW_SYSTEM_VERSION: 'SYSTEM_VERSION';
KW_EXPIRE_SNAPSHOTS: 'EXPIRE_SNAPSHOTS';
KW_REWRITE_MANIFESTS: 'REWRITE_MANIFESTS';
+KW_EXCLUDE: 'EXCLUDE';
KW_SET_CURRENT_SNAPSHOT: 'SET_CURRENT_SNAPSHOT';
KW_BRANCH: 'BRANCH';
KW_SNAPSHOTS: 'SNAPSHOTS';
diff --git
a/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
b/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
index 37fd6187d16..c7e63cd286a 100644
--- a/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
+++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
@@ -1026,6 +1026,7 @@ nonReserved
| KW_TRIM
| KW_SPEC
| KW_SYSTEM_TIME | KW_SYSTEM_VERSION
+ | KW_EXCLUDE
| KW_EXPIRE_SNAPSHOTS
| KW_REWRITE_MANIFESTS
| KW_SET_CURRENT_SNAPSHOT
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 819ff93dbb5..6c9f412a1a5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -4367,8 +4367,11 @@ private Pair<RelNode, RowResolver>
internalGenSelectLogicalPlan(QB qb, RelNode s
// 6.4 Build ExprNode corresponding to colums
if (expr.getType() == HiveParser.TOK_ALLCOLREF) {
- pos = genRexNodeRegex(".*",
- expr.getChildCount() == 0 ? null : getUnescapedName((ASTNode)
expr.getChild(0)).toLowerCase(),
+ // Parse SELECT * EXCLUDE columns and pass them to the Calcite
engine for exclusion
+ ExcludeResult excludeResult = processAllColRefAndExclude(expr,
inputRR);
+ String starTabAlias = excludeResult.tableAlias();
+ excludedColumns.addAll(excludeResult.excludedColumns());
+ pos = genRexNodeRegex(".*", starTabAlias,
expr, columnList, excludedColumns, inputRR, starRR, pos,
outputRR, qb.getAliases(), true);
} else if (expr.getType() == HiveParser.TOK_TABLE_OR_COL
&& !hasAsClause
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 6bcbd346b23..d74351bd590 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -4738,6 +4738,52 @@ static boolean isRegex(String pattern, HiveConf conf) {
return false;
}
+ /**
+ * Helper method to parse the excluded columns from an EXCLUDE AST node.
Returns an unmodifiable
+ * set to ensure the caller cannot accidentally mutate the result.
+ */
+ private Set<ColumnInfo> processExcludeColumns(
+ ASTNode excludeNode, String starTabAlias, RowResolver inputRR) throws
SemanticException {
+ Set<ColumnInfo> localExcluded = new HashSet<>();
+ for (int e = 0; e < excludeNode.getChildCount(); e++) {
+ String excludeColName =
unescapeIdentifier(excludeNode.getChild(e).getText()).toLowerCase();
+ ColumnInfo colInfo = inputRR.get(starTabAlias, excludeColName);
+ if (colInfo != null) {
+ localExcluded.add(colInfo);
+ }
+ }
+ return Collections.unmodifiableSet(localExcluded);
+ }
+
+ protected record ExcludeResult(String tableAlias, Set<ColumnInfo>
excludedColumns) {}
+
+ /**
+ * Parses a TOK_ALLCOLREF node (e.g. `*` or `t.* EXCLUDE (a)`) to extract
the table alias and the
+ * set of columns to be excluded.
+ */
+ protected ExcludeResult processAllColRefAndExclude(ASTNode expr, RowResolver
inputRR)
+ throws SemanticException {
+
+ String starTabAlias = null;
+
+ // Zero-allocation initialization for queries that don't use EXCLUDE.
+ Set<ColumnInfo> excludedColumns = Set.of();
+
+ if (expr.getChildren() != null) {
+ for (Node childNode : expr.getChildren()) {
+ ASTNode child = (ASTNode) childNode;
+ switch (child.getType()) {
+ case HiveParser.TOK_TABNAME -> starTabAlias =
getUnescapedName(child).toLowerCase();
+ case HiveParser.TOK_TABCOLNAME ->
+ excludedColumns = processExcludeColumns(child, starTabAlias,
inputRR);
+ default ->
+ throw new SemanticException(
+ "Unexpected node type in TOK_ALLCOLREF: " + child.getType());
+ }
+ }
+ }
+ return new ExcludeResult(starTabAlias, excludedColumns);
+ }
private Operator<?> genSelectPlan(String dest, QB qb, Operator<?> input,
Operator<?> inputForSelectStar) throws
SemanticException {
@@ -4915,9 +4961,16 @@ private Operator<?> genSelectPlan(String dest, ASTNode
selExprList, QB qb, Opera
// The real expression
if (expr.getType() == HiveParser.TOK_ALLCOLREF) {
int initPos = pos;
- pos = genExprNodeDescRegex(".*", expr.getChildCount() == 0 ? null
- : getUnescapedName((ASTNode) expr.getChild(0)).toLowerCase(),
- expr, colList, null, inputRR, starRR, pos, out_rwsch,
qb.getAliases(), false);
+
+ ExcludeResult excludeResult = processAllColRefAndExclude(expr,
inputRR);
+ String starTabAlias = excludeResult.tableAlias();
+ Set<ColumnInfo> excludeCols = excludeResult.excludedColumns();
+ if (excludeCols.isEmpty()) {
+ excludeCols = null;
+ }
+
+ pos = genExprNodeDescRegex(".*", starTabAlias,
+ expr, colList, excludeCols, inputRR, starRR, pos, out_rwsch,
qb.getAliases(), false);
if (unparseTranslator.isEnabled()) {
offset += pos - initPos - 1;
}
diff --git a/ql/src/test/queries/clientpositive/select_exclude.q
b/ql/src/test/queries/clientpositive/select_exclude.q
new file mode 100644
index 00000000000..3159ea299fc
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/select_exclude.q
@@ -0,0 +1,39 @@
+set hive.cli.print.header=true;
+
+CREATE TABLE test_exclude (
+ id INT,
+ name STRING,
+ email STRING,
+ address STRING,
+ phone STRING
+);
+
+INSERT INTO test_exclude VALUES
+(1, 'Alice', '[email protected]', '123 Apple St', '555-0100'),
+(2, 'Bob', '[email protected]', '456 Banana Ave', '555-0200');
+
+-- Exclude a single column
+EXPLAIN SELECT * EXCLUDE (email) FROM test_exclude;
+SELECT * EXCLUDE (email) FROM test_exclude;
+
+-- Exclude multiple columns
+EXPLAIN SELECT * EXCLUDE (email, address, phone) FROM test_exclude;
+SELECT * EXCLUDE (email, address, phone) FROM test_exclude;
+
+-- Exclude with table alias
+EXPLAIN SELECT t.* EXCLUDE (id, phone) FROM test_exclude t;
+SELECT t.* EXCLUDE (id, phone) FROM test_exclude t;
+
+-- Exclude with JOIN
+CREATE TABLE test_exclude_join (
+ id INT,
+ department STRING
+);
+INSERT INTO test_exclude_join VALUES (1, 'Engineering'), (2, 'Sales');
+
+EXPLAIN
+SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id)
+FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id;
+
+SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id)
+FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id;
diff --git a/ql/src/test/results/clientpositive/llap/select_exclude.q.out
b/ql/src/test/results/clientpositive/llap/select_exclude.q.out
new file mode 100644
index 00000000000..0bb6d242227
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/select_exclude.q.out
@@ -0,0 +1,283 @@
+PREHOOK: query: CREATE TABLE test_exclude (
+ id INT,
+ name STRING,
+ email STRING,
+ address STRING,
+ phone STRING
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test_exclude
+POSTHOOK: query: CREATE TABLE test_exclude (
+ id INT,
+ name STRING,
+ email STRING,
+ address STRING,
+ phone STRING
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test_exclude
+PREHOOK: query: INSERT INTO test_exclude VALUES
+(1, 'Alice', '[email protected]', '123 Apple St', '555-0100'),
+(2, 'Bob', '[email protected]', '456 Banana Ave', '555-0200')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_exclude
+POSTHOOK: query: INSERT INTO test_exclude VALUES
+(1, 'Alice', '[email protected]', '123 Apple St', '555-0100'),
+(2, 'Bob', '[email protected]', '456 Banana Ave', '555-0200')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_exclude
+POSTHOOK: Lineage: test_exclude.address SCRIPT []
+POSTHOOK: Lineage: test_exclude.email SCRIPT []
+POSTHOOK: Lineage: test_exclude.id SCRIPT []
+POSTHOOK: Lineage: test_exclude.name SCRIPT []
+POSTHOOK: Lineage: test_exclude.phone SCRIPT []
+col1 col2 col3 col4 col5
+PREHOOK: query: EXPLAIN SELECT * EXCLUDE (email) FROM test_exclude
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_exclude
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN SELECT * EXCLUDE (email) FROM test_exclude
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_exclude
+#### A masked pattern was here ####
+Explain
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: test_exclude
+ Select Operator
+ expressions: id (type: int), name (type: string), address (type:
string), phone (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ ListSink
+
+PREHOOK: query: SELECT * EXCLUDE (email) FROM test_exclude
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_exclude
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * EXCLUDE (email) FROM test_exclude
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_exclude
+#### A masked pattern was here ####
+test_exclude.id test_exclude.name test_exclude.address
test_exclude.phone
+1 Alice 123 Apple St 555-0100
+2 Bob 456 Banana Ave 555-0200
+PREHOOK: query: EXPLAIN SELECT * EXCLUDE (email, address, phone) FROM
test_exclude
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_exclude
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN SELECT * EXCLUDE (email, address, phone) FROM
test_exclude
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_exclude
+#### A masked pattern was here ####
+Explain
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: test_exclude
+ Select Operator
+ expressions: id (type: int), name (type: string)
+ outputColumnNames: _col0, _col1
+ ListSink
+
+PREHOOK: query: SELECT * EXCLUDE (email, address, phone) FROM test_exclude
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_exclude
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * EXCLUDE (email, address, phone) FROM test_exclude
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_exclude
+#### A masked pattern was here ####
+test_exclude.id test_exclude.name
+1 Alice
+2 Bob
+PREHOOK: query: EXPLAIN SELECT t.* EXCLUDE (id, phone) FROM test_exclude t
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_exclude
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN SELECT t.* EXCLUDE (id, phone) FROM test_exclude t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_exclude
+#### A masked pattern was here ####
+Explain
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: t
+ Select Operator
+ expressions: name (type: string), email (type: string), address
(type: string)
+ outputColumnNames: _col0, _col1, _col2
+ ListSink
+
+PREHOOK: query: SELECT t.* EXCLUDE (id, phone) FROM test_exclude t
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_exclude
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT t.* EXCLUDE (id, phone) FROM test_exclude t
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_exclude
+#### A masked pattern was here ####
+t.name t.email t.address
+Alice [email protected] 123 Apple St
+Bob [email protected] 456 Banana Ave
+PREHOOK: query: CREATE TABLE test_exclude_join (
+ id INT,
+ department STRING
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@test_exclude_join
+POSTHOOK: query: CREATE TABLE test_exclude_join (
+ id INT,
+ department STRING
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test_exclude_join
+PREHOOK: query: INSERT INTO test_exclude_join VALUES (1, 'Engineering'), (2,
'Sales')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@test_exclude_join
+POSTHOOK: query: INSERT INTO test_exclude_join VALUES (1, 'Engineering'), (2,
'Sales')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@test_exclude_join
+POSTHOOK: Lineage: test_exclude_join.department SCRIPT []
+POSTHOOK: Lineage: test_exclude_join.id SCRIPT []
+col1 col2
+PREHOOK: query: EXPLAIN
+SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id)
+FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_exclude
+PREHOOK: Input: default@test_exclude_join
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN
+SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id)
+FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_exclude
+POSTHOOK: Input: default@test_exclude_join
+#### A masked pattern was here ####
+Explain
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ filterExpr: id is not null (type: boolean)
+ Statistics: Num rows: 2 Data size: 378 Basic stats: COMPLETE
Column stats: COMPLETE
+ Filter Operator
+ predicate: id is not null (type: boolean)
+ Statistics: Num rows: 2 Data size: 378 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: id (type: int), name (type: string), email
(type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 378 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 2 Data size: 378 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string), _col2 (type:
string)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: b
+ filterExpr: id is not null (type: boolean)
+ Statistics: Num rows: 2 Data size: 192 Basic stats: COMPLETE
Column stats: COMPLETE
+ Filter Operator
+ predicate: id is not null (type: boolean)
+ Statistics: Num rows: 2 Data size: 192 Basic stats:
COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: id (type: int), department (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 192 Basic stats:
COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 2 Data size: 192 Basic stats:
COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: llap
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col4
+ Statistics: Num rows: 2 Data size: 562 Basic stats: COMPLETE
Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: string), _col2
(type: string), _col4 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 2 Data size: 562 Basic stats: COMPLETE
Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 562 Basic stats:
COMPLETE Column stats: COMPLETE
+ table:
+ input format:
org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde:
org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id)
+FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_exclude
+PREHOOK: Input: default@test_exclude_join
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT a.* EXCLUDE (address, phone), b.* EXCLUDE (id)
+FROM test_exclude a JOIN test_exclude_join b ON a.id = b.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_exclude
+POSTHOOK: Input: default@test_exclude_join
+#### A masked pattern was here ####
+a.id a.name a.email b.department
+1 Alice [email protected] Engineering
+2 Bob [email protected] Sales
