This is an automated email from the ASF dual-hosted git repository.
dengzhhu653 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 9e843dd82b6 HIVE-29807: Materialized CTE inputs not fully propagated
to query authorization (#6700)
9e843dd82b6 is described below
commit 9e843dd82b615bb467d29dd4ba88adc6b3e575b0
Author: dengzh <[email protected]>
AuthorDate: Thu Aug 27 15:52:51 2026 +0800
HIVE-29807: Materialized CTE inputs not fully propagated to query
authorization (#6700)
---
.../apache/hadoop/hive/ql/exec/ExplainTask.java | 2 +-
.../hadoop/hive/ql/parse/BaseSemanticAnalyzer.java | 2 +-
.../hadoop/hive/ql/parse/ColumnAccessAnalyzer.java | 18 ++-
.../hadoop/hive/ql/parse/ColumnAccessInfo.java | 10 ++
.../hive/ql/parse/DeleteSemanticAnalyzer.java | 2 +-
.../hive/ql/parse/ExplainSemanticAnalyzer.java | 2 +-
.../parse/HiveSemanticAnalyzerHookContextImpl.java | 2 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 17 ++-
.../apache/hadoop/hive/ql/plan/ExplainWork.java | 2 +-
.../authorization/command/CommandAuthorizer.java | 5 +-
.../hive/ql/parse/TestMaterializedCTEInputs.java | 125 +++++++++++++++++++++
.../test/results/clientpositive/llap/cte_3.q.out | 8 ++
.../results/clientpositive/llap/cte_mat_11.q.out | 64 +++++++++++
.../results/clientpositive/llap/cte_mat_3.q.out | 2 +
.../results/clientpositive/llap/cte_mat_4.q.out | 4 +
.../results/clientpositive/llap/cte_mat_5.q.out | 2 +
.../results/clientpositive/llap/cte_mat_9.q.out | 2 +
.../results/clientpositive/llap/cte_mat_type.q.out | 4 +
18 files changed, 255 insertions(+), 18 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
index 1db21728499..6130feac975 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
@@ -666,7 +666,7 @@ JSONObject collectAuthRelatedEntities(PrintStream out,
ExplainWork work)
HiveOperation operation = queryState.getHiveOperation();
JSONObject object = new JSONObject(new LinkedHashMap<>());
- Object jsonInput = toJson("INPUTS", toString(analyzer.getInputs()), out,
work);
+ Object jsonInput = toJson("INPUTS", toString(analyzer.getAllInputs()),
out, work);
if (work.isFormatted()) {
object.put("INPUTS", jsonInput);
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
index bbac9da2e47..0862d345582 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
@@ -1650,7 +1650,7 @@ public boolean hasTransactionalInQuery() {
public boolean isRequiresOpenTransaction() {
return hasTransactionalInQuery() || getAcidDdlDesc() != null ||
- Stream.of(getInputs(), getOutputs()).flatMap(Collection::stream)
+ Stream.of(getAllInputs(), getOutputs()).flatMap(Collection::stream)
.filter(entity -> entity.getType() == Entity.Type.TABLE ||
entity.getType() == Entity.Type.PARTITION)
.flatMap(entity -> {
Table tbl = entity.getTable();
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java
index e5ba8643f2b..2ce0dd78a73 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessAnalyzer.java
@@ -35,7 +35,8 @@ public ColumnAccessAnalyzer(ParseContext pactx) {
pGraphContext = pactx;
}
- public ColumnAccessInfo analyzeColumnAccess(ColumnAccessInfo
columnAccessInfo) throws SemanticException {
+ public ColumnAccessInfo analyzeColumnAccess(SemanticAnalyzer analyzer)
throws SemanticException {
+ ColumnAccessInfo columnAccessInfo = analyzer.getColumnAccessInfo();
if (columnAccessInfo == null) {
columnAccessInfo = new ColumnAccessInfo();
}
@@ -59,6 +60,21 @@ public ColumnAccessInfo analyzeColumnAccess(ColumnAccessInfo
columnAccessInfo) t
}
}
}
+
columnAccessInfo.merge(getMaterializedCteColumnAccessInfo(analyzer.rootClause.asExecutionOrder()));
+ return columnAccessInfo;
+ }
+
+ /**
+ * Merge column access recorded by materialized CTE sub-analyzers into this
analyzer's
+ * column access info so authorization sees base-table columns, not only the
temp CTE table.
+ */
+ private ColumnAccessInfo
getMaterializedCteColumnAccessInfo(List<SemanticAnalyzer.CTEClause> cteClauses)
{
+ ColumnAccessInfo columnAccessInfo = new ColumnAccessInfo();
+ for (SemanticAnalyzer.CTEClause cte : cteClauses) {
+ if (cte.source != null && cte.source.getColumnAccessInfo() != null) {
+ columnAccessInfo.merge(cte.source.getColumnAccessInfo());
+ }
+ }
return columnAccessInfo;
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java
index d43ffb77064..d9a3dd840eb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ColumnAccessInfo.java
@@ -93,6 +93,16 @@ public Map<String, List<String>>
getTableToColumnAllAccessMap() {
return mapping;
}
+ /**
+ * Merge column accesses from another ColumnAccessInfo into this one.
+ */
+ public void merge(ColumnAccessInfo other) {
+ if (other == null) {
+ return;
+ }
+ tableToColumnAccessMap.putAll(other.tableToColumnAccessMap);
+ }
+
/**
* Strip a virtual column out of the set of columns. This is useful in
cases where we do not
* want to be checking against the user reading virtual columns, namely
update and delete.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/DeleteSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/DeleteSemanticAnalyzer.java
index 5f4847093dd..12386371627 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DeleteSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DeleteSemanticAnalyzer.java
@@ -134,7 +134,7 @@ private boolean tryMetadataUpdate(Table table, ASTNode
tabNameNode, ASTNode wher
DDLWork ddlWork = createDDLWorkOfMetadataUpdate(tableName, sarg);
rootTasks = Collections.singletonList(TaskFactory.get(ddlWork));
- inputs = sem.getInputs();
+ inputs = sem.getAllInputs();
outputs = sem.getOutputs();
updateOutputs(table);
return true;
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
index f28e606aecc..226f033e909 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
@@ -187,7 +187,7 @@ public void analyzeInternal(ASTNode ast) throws
SemanticException {
BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(queryState, input);
sem.analyze(input, ctx);
sem.validate();
- inputs = sem.getInputs();
+ inputs = sem.getAllInputs();
outputs = sem.getOutputs();
ctx.setResFile(ctx.getLocalTmpPath());
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java
index 448f3bdc514..ea9c5097693 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveSemanticAnalyzerHookContextImpl.java
@@ -57,7 +57,7 @@ public void setConf(Configuration conf) {
@Override
public void update(BaseSemanticAnalyzer sem) {
- this.inputs = sem.getInputs();
+ this.inputs = sem.getAllInputs();
this.outputs = sem.getOutputs();
this.commandType = sem.getQueryState().getHiveOperation();
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index d74351bd590..4bef39b98b7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -1407,7 +1407,7 @@ private void addCTEAsSubQuery(QB qb, String cteName,
String cteAlias)
qb.rewriteCTEToSubq(cteAlias, cteName, cteQBExpr);
}
- private final CTEClause rootClause = new CTEClause(null, null, null);
+ final CTEClause rootClause = new CTEClause(null, null, null);
@Override
public List<Task<?>> getAllRootTasks() {
@@ -1422,10 +1422,10 @@ public List<Task<?>> getAllRootTasks() {
@Override
public Set<ReadEntity> getAllInputs() {
- Set<ReadEntity> readEntities = new HashSet<ReadEntity>(getInputs());
+ Set<ReadEntity> readEntities = new LinkedHashSet<>(getInputs());
for (CTEClause cte : rootClause.asExecutionOrder()) {
if (cte.source != null) {
- readEntities.addAll(cte.source.getInputs());
+ readEntities.addAll(cte.source.getAllInputs());
}
}
return readEntities;
@@ -1436,7 +1436,7 @@ public Set<WriteEntity> getAllOutputs() {
Set<WriteEntity> writeEntities = new HashSet<WriteEntity>(getOutputs());
for (CTEClause cte : rootClause.asExecutionOrder()) {
if (cte.source != null) {
- writeEntities.addAll(cte.source.getOutputs());
+ writeEntities.addAll(cte.source.getAllOutputs());
}
}
return writeEntities;
@@ -1596,9 +1596,8 @@ Table materializeCTE(String cteName, CTEClause cte)
throws HiveException {
LOG.info("{} will be materialized into {}", cteName, location);
cte.source = analyzer;
-
+
ctx.addMaterializedTable(cteName, table,
getMaterializedTableStats(analyzer.getSinkOp()));
-
return table;
}
@@ -13441,7 +13440,7 @@ void analyzeInternal(ASTNode ast,
Supplier<PlannerContext> pcf) throws SemanticE
|| HiveConf.getBoolVar(this.conf,
HiveConf.ConfVars.HIVE_STATS_COLLECT_SCANCOLS)) {
ColumnAccessAnalyzer columnAccessAnalyzer = new
ColumnAccessAnalyzer(pCtx);
// view column access info is carried by this.getColumnAccessInfo().
-
setColumnAccessInfo(columnAccessAnalyzer.analyzeColumnAccess(this.getColumnAccessInfo()));
+ setColumnAccessInfo(columnAccessAnalyzer.analyzeColumnAccess(this));
}
}
perfLogger.perfLogEnd(this.getClass().getName(),
PerfLogger.LOGICAL_OPTIMIZATION);
@@ -13480,7 +13479,7 @@ void analyzeInternal(ASTNode ast,
Supplier<PlannerContext> pcf) throws SemanticE
// 11. put accessed columns to readEntity
if (HiveConf.getBoolVar(this.conf,
HiveConf.ConfVars.HIVE_STATS_COLLECT_SCANCOLS)) {
- putAccessedColumnsToReadEntity(inputs, columnAccessInfo);
+ putAccessedColumnsToReadEntity(getAllInputs(), columnAccessInfo);
}
if (isCacheEnabled && lookupInfo != null) {
@@ -15342,7 +15341,7 @@ private void
useCachedResult(QueryResultsCache.CacheEntry cacheEntry, boolean ne
private QueryResultsCache.QueryInfo
createCacheQueryInfoForQuery(QueryResultsCache.LookupInfo lookupInfo) {
long queryTime =
SessionState.get().getQueryCurrentTimestamp().toEpochMilli();
return new QueryResultsCache.QueryInfo(queryTime, lookupInfo,
queryState.getHiveOperation(),
- resultSchema, getTableAccessInfo(), getColumnAccessInfo(), inputs);
+ resultSchema, getTableAccessInfo(), getColumnAccessInfo(),
getAllInputs());
}
/**
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
index 2faab2160dd..0fb126e1878 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
@@ -82,7 +82,7 @@ public ExplainWork(Path resFile,
}
this.analyzer = analyzer;
if (analyzer != null) {
- this.inputs = analyzer.getInputs();
+ this.inputs = analyzer.getAllInputs();
}
if (analyzer != null) {
this.outputs = analyzer.getAllOutputs();
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizer.java
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizer.java
index fbfc77e953d..8247a33c361 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/command/CommandAuthorizer.java
@@ -76,14 +76,15 @@ private static boolean skip(HiveOperation op,
BaseSemanticAnalyzer sem) throws H
private static Set<ReadEntity> getInputs(BaseSemanticAnalyzer sem) {
Set<ReadEntity> additionalInputs = new HashSet<ReadEntity>();
- for (Entity e : sem.getInputs()) {
+ for (Entity e : sem.getAllInputs()) {
if (e.getType() == Entity.Type.PARTITION) {
additionalInputs.add(new ReadEntity(e.getTable()));
}
}
+ // getAllInputs() includes tables read by materialized CTE sub-analyzers.
// Sets.union keeps the values from the first set if they are present in
both
- return Sets.union(sem.getInputs(), additionalInputs);
+ return Sets.union(sem.getAllInputs(), additionalInputs);
}
private static Set<WriteEntity> getOutputs(BaseSemanticAnalyzer sem) {
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMaterializedCTEInputs.java
b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMaterializedCTEInputs.java
new file mode 100644
index 00000000000..6a888cb3131
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestMaterializedCTEInputs.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.parse;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Stream;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConfForTest;
+import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.ddl.database.drop.DropDatabaseDesc;
+import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestMaterializedCTEInputs {
+ private static final String DB_NAME = "test_materialized_cte";
+ private static final String TABLE_FQ_NAME = DB_NAME + "@src";
+
+ private static Hive db;
+ private static HiveConf conf;
+
+ @BeforeAll
+ public static void beforeClass() throws Exception {
+ conf = new HiveConfForTest(TestMaterializedCTEInputs.class);
+ conf.set("hive.security.authorization.enabled", "false");
+ conf.set("hive.security.authorization.manager",
+
"org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdConfOnlyAuthorizerFactory");
+ conf.setIntVar(HiveConf.ConfVars.HIVE_CTE_MATERIALIZE_THRESHOLD, 1);
+
conf.setBoolVar(HiveConf.ConfVars.HIVE_CTE_MATERIALIZE_FULL_AGGREGATE_ONLY,
false);
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_STATS_COLLECT_SCANCOLS, true);
+ db = Hive.get(conf);
+ SessionState.start(conf);
+ new DatabaseBuilder().setName(DB_NAME).create(db.getMSC(), conf);
+ SessionState.get().setCurrentDatabase(DB_NAME);
+ new TableBuilder().setDbName(DB_NAME).setTableName("src")
+ .addCol("key", "string")
+ .addCol("value", "string")
+ .addCol("col1", "int")
+ .create(db.getMSC(), conf);
+ }
+
+ public static Stream<Arguments> casesForMaterializedCteInputs() {
+ return Stream.of(
+ Arguments.of("chain cte", "with q1 as ( select key from q2 where key =
'5'),"
+ + "q2 as ( select key from test_materialized_cte.src where key =
'5') "
+ + "select * from (select key from q1) a", Set.of("key")),
+ Arguments.of("nested cte", "WITH q1 AS ("
+ + "WITH q2 AS (SELECT key, value FROM test_materialized_cte.src
WHERE key = '4') "
+ + "SELECT * FROM q2 UNION ALL SELECT * FROM q2) "
+ + "SELECT * FROM q1 t1 JOIN q1 t2 ON t1.key = t2.key",
Set.of("key", "value")),
+ Arguments.of("merge columns", "WITH q1 AS ("
+ + "WITH q2 AS (SELECT key FROM test_materialized_cte.src WHERE
key = '4') "
+ + "SELECT * FROM q2 UNION ALL SELECT * FROM q2) "
+ + "SELECT * FROM q1 t1 JOIN test_materialized_cte.src t2 ON
t1.key = t2.key", Set.of("key", "value"))
+ );
+ }
+
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("casesForMaterializedCteInputs")
+ void testMaterializedCteInputs(String type, String query, Set<String>
expectedCols) throws Exception {
+ HiveConf testConf = new HiveConf(conf);
+ Context ctx = new Context(testConf);
+ ASTNode astNode = ParseUtils.parse(query, ctx);
+ QueryState queryState = new
QueryState.Builder().withHiveConf(testConf).build();
+ SemanticAnalyzer analyzer = (SemanticAnalyzer)
SemanticAnalyzerFactory.get(queryState, astNode);
+ analyzer.initCtx(ctx);
+ analyzer.analyze(astNode, ctx);
+
+ Set<ReadEntity> allInputs = analyzer.getAllInputs();
+
+ assertTrue(allInputs.stream().anyMatch(e -> isTableNamed(e, "src")),
+ "Materialized CTE base table must appear in getAllInputs");
+
+ ColumnAccessInfo columnAccessInfo = analyzer.getColumnAccessInfo();
+ assertNotNull(columnAccessInfo);
+ List<String> srcCols =
columnAccessInfo.getTableToColumnAccessMap().get(TABLE_FQ_NAME);
+ assertNotNull(srcCols, "Column must include materialized CTE base table");
+ assertTrue(new HashSet<>(srcCols).containsAll(expectedCols),
+ () -> "Expected columns " + expectedCols + " but got " + srcCols);
+ ctx.clear();
+ }
+
+ private static boolean isTableNamed(ReadEntity entity, String tableName) {
+ return entity.getTable() != null &&
tableName.equals(entity.getTable().getTableName());
+ }
+
+ @AfterAll
+ public static void afterClass() throws Exception {
+ try {
+ db.dropDatabase(new DropDatabaseDesc(DB_NAME, DB_NAME, true, true,
true));
+ } catch (Exception ignored) {
+ }
+ db.close(true);
+ }
+}
diff --git a/ql/src/test/results/clientpositive/llap/cte_3.q.out
b/ql/src/test/results/clientpositive/llap/cte_3.q.out
index ee1fe65efff..1a7f931ce03 100644
--- a/ql/src/test/results/clientpositive/llap/cte_3.q.out
+++ b/ql/src/test/results/clientpositive/llap/cte_3.q.out
@@ -4,6 +4,7 @@ select *
from q1
PREHOOK: type: QUERY
PREHOOK: Input: default@q1
+PREHOOK: Input: default@src
#### A masked pattern was here ####
POSTHOOK: query: explain
with q1 as ( select key from src where key = '5')
@@ -11,6 +12,7 @@ select *
from q1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@q1
+POSTHOOK: Input: default@src
#### A masked pattern was here ####
Plan optimized by CBO.
@@ -63,12 +65,14 @@ with q1 as ( select key from src where key = '5')
select * from (select key from q1) a
PREHOOK: type: QUERY
PREHOOK: Input: default@q1
+PREHOOK: Input: default@src
#### A masked pattern was here ####
POSTHOOK: query: explain
with q1 as ( select key from src where key = '5')
select * from (select key from q1) a
POSTHOOK: type: QUERY
POSTHOOK: Input: default@q1
+POSTHOOK: Input: default@src
#### A masked pattern was here ####
Plan optimized by CBO.
@@ -120,6 +124,8 @@ q2 as ( select key from src where key = '5')
select * from (select key from q1) a
PREHOOK: type: QUERY
PREHOOK: Input: default@q1
+PREHOOK: Input: default@q2
+PREHOOK: Input: default@src
#### A masked pattern was here ####
POSTHOOK: query: explain
with q1 as ( select key from q2 where key = '5'),
@@ -127,6 +133,8 @@ q2 as ( select key from src where key = '5')
select * from (select key from q1) a
POSTHOOK: type: QUERY
POSTHOOK: Input: default@q1
+POSTHOOK: Input: default@q2
+POSTHOOK: Input: default@src
#### A masked pattern was here ####
Plan optimized by CBO.
diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_11.q.out
b/ql/src/test/results/clientpositive/llap/cte_mat_11.q.out
index f4cadc024bf..809fadf4042 100644
--- a/ql/src/test/results/clientpositive/llap/cte_mat_11.q.out
+++ b/ql/src/test/results/clientpositive/llap/cte_mat_11.q.out
@@ -48,7 +48,9 @@ SELECT a.key
FROM materialized_cte2 a
JOIN materialized_cte2 b ON (a.key = b.key)
PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte1
PREHOOK: Input: default@materialized_cte2
+PREHOOK: Input: default@src
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS (
SELECT * FROM src
@@ -62,7 +64,9 @@ SELECT a.key
FROM materialized_cte2 a
JOIN materialized_cte2 b ON (a.key = b.key)
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte1
POSTHOOK: Input: default@materialized_cte2
+POSTHOOK: Input: default@src
#### A masked pattern was here ####
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -272,7 +276,9 @@ SELECT a.key
FROM materialized_cte2 a
JOIN materialized_cte2 b ON (a.key = b.key)
PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte1
PREHOOK: Input: default@materialized_cte2
+PREHOOK: Input: default@src
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
SELECT * FROM src
@@ -286,7 +292,9 @@ SELECT a.key
FROM materialized_cte2 a
JOIN materialized_cte2 b ON (a.key = b.key)
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte1
POSTHOOK: Input: default@materialized_cte2
+POSTHOOK: Input: default@src
#### A masked pattern was here ####
CBO PLAN:
HiveProject(a.key=[$0])
@@ -310,7 +318,9 @@ SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2
PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte1
PREHOOK: Input: default@materialized_cte2
+PREHOOK: Input: default@src
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS (
SELECT * FROM src
@@ -324,7 +334,9 @@ SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte1
POSTHOOK: Input: default@materialized_cte2
+POSTHOOK: Input: default@src
#### A masked pattern was here ####
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -493,7 +505,9 @@ SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2
PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte1
PREHOOK: Input: default@materialized_cte2
+PREHOOK: Input: default@src
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
SELECT * FROM src
@@ -507,7 +521,9 @@ SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte1
POSTHOOK: Input: default@materialized_cte2
+POSTHOOK: Input: default@src
#### A masked pattern was here ####
CBO PLAN:
HiveProject(key=[$0], value=[$1])
@@ -577,7 +593,9 @@ SELECT a.key
FROM materialized_cte2 a
JOIN materialized_cte2 b ON (a.key = b.key)
PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte1
PREHOOK: Input: default@materialized_cte2
+PREHOOK: Input: default@src_no_stats
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS (
SELECT * FROM src_no_stats
@@ -591,7 +609,9 @@ SELECT a.key
FROM materialized_cte2 a
JOIN materialized_cte2 b ON (a.key = b.key)
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte1
POSTHOOK: Input: default@materialized_cte2
+POSTHOOK: Input: default@src_no_stats
#### A masked pattern was here ####
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -801,7 +821,9 @@ SELECT a.key
FROM materialized_cte2 a
JOIN materialized_cte2 b ON (a.key = b.key)
PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte1
PREHOOK: Input: default@materialized_cte2
+PREHOOK: Input: default@src_no_stats
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
SELECT * FROM src_no_stats
@@ -815,7 +837,9 @@ SELECT a.key
FROM materialized_cte2 a
JOIN materialized_cte2 b ON (a.key = b.key)
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte1
POSTHOOK: Input: default@materialized_cte2
+POSTHOOK: Input: default@src_no_stats
#### A masked pattern was here ####
CBO PLAN:
HiveProject(a.key=[$0])
@@ -839,7 +863,9 @@ SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2
PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte1
PREHOOK: Input: default@materialized_cte2
+PREHOOK: Input: default@src_no_stats
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS (
SELECT * FROM src_no_stats
@@ -853,7 +879,9 @@ SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte1
POSTHOOK: Input: default@materialized_cte2
+POSTHOOK: Input: default@src_no_stats
#### A masked pattern was here ####
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -1022,7 +1050,9 @@ SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2
PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte1
PREHOOK: Input: default@materialized_cte2
+PREHOOK: Input: default@src_no_stats
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
SELECT * FROM src_no_stats
@@ -1036,7 +1066,9 @@ SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte1
POSTHOOK: Input: default@materialized_cte2
+POSTHOOK: Input: default@src_no_stats
#### A masked pattern was here ####
CBO PLAN:
HiveProject(key=[$0], value=[$1])
@@ -1118,7 +1150,9 @@ SELECT a.key
FROM materialized_cte2 a
JOIN materialized_cte2 b ON (a.key = b.key)
PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte1
PREHOOK: Input: default@materialized_cte2
+PREHOOK: Input: default@src_partial_stats_key
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS (
SELECT * FROM src_partial_stats_key
@@ -1132,7 +1166,9 @@ SELECT a.key
FROM materialized_cte2 a
JOIN materialized_cte2 b ON (a.key = b.key)
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte1
POSTHOOK: Input: default@materialized_cte2
+POSTHOOK: Input: default@src_partial_stats_key
#### A masked pattern was here ####
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -1342,7 +1378,9 @@ SELECT a.key
FROM materialized_cte2 a
JOIN materialized_cte2 b ON (a.key = b.key)
PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte1
PREHOOK: Input: default@materialized_cte2
+PREHOOK: Input: default@src_partial_stats_key
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
SELECT * FROM src_partial_stats_key
@@ -1356,7 +1394,9 @@ SELECT a.key
FROM materialized_cte2 a
JOIN materialized_cte2 b ON (a.key = b.key)
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte1
POSTHOOK: Input: default@materialized_cte2
+POSTHOOK: Input: default@src_partial_stats_key
#### A masked pattern was here ####
CBO PLAN:
HiveProject(a.key=[$0])
@@ -1380,7 +1420,9 @@ SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2
PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte1
PREHOOK: Input: default@materialized_cte2
+PREHOOK: Input: default@src_partial_stats_key
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS (
SELECT * FROM src_partial_stats_key
@@ -1394,7 +1436,9 @@ SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte1
POSTHOOK: Input: default@materialized_cte2
+POSTHOOK: Input: default@src_partial_stats_key
#### A masked pattern was here ####
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -1563,7 +1607,9 @@ SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2
PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte1
PREHOOK: Input: default@materialized_cte2
+PREHOOK: Input: default@src_partial_stats_key
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
SELECT * FROM src_partial_stats_key
@@ -1577,7 +1623,9 @@ SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte1
POSTHOOK: Input: default@materialized_cte2
+POSTHOOK: Input: default@src_partial_stats_key
#### A masked pattern was here ####
CBO PLAN:
HiveProject(key=[$0], value=[$1])
@@ -1659,7 +1707,9 @@ SELECT a.key
FROM materialized_cte2 a
JOIN materialized_cte2 b ON (a.key = b.key)
PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte1
PREHOOK: Input: default@materialized_cte2
+PREHOOK: Input: default@src_partial_stats_value
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS (
SELECT * FROM src_partial_stats_value
@@ -1673,7 +1723,9 @@ SELECT a.key
FROM materialized_cte2 a
JOIN materialized_cte2 b ON (a.key = b.key)
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte1
POSTHOOK: Input: default@materialized_cte2
+POSTHOOK: Input: default@src_partial_stats_value
#### A masked pattern was here ####
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -1883,7 +1935,9 @@ SELECT a.key
FROM materialized_cte2 a
JOIN materialized_cte2 b ON (a.key = b.key)
PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte1
PREHOOK: Input: default@materialized_cte2
+PREHOOK: Input: default@src_partial_stats_value
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
SELECT * FROM src_partial_stats_value
@@ -1897,7 +1951,9 @@ SELECT a.key
FROM materialized_cte2 a
JOIN materialized_cte2 b ON (a.key = b.key)
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte1
POSTHOOK: Input: default@materialized_cte2
+POSTHOOK: Input: default@src_partial_stats_value
#### A masked pattern was here ####
CBO PLAN:
HiveProject(a.key=[$0])
@@ -1921,7 +1977,9 @@ SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2
PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte1
PREHOOK: Input: default@materialized_cte2
+PREHOOK: Input: default@src_partial_stats_value
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN WITH materialized_cte1 AS (
SELECT * FROM src_partial_stats_value
@@ -1935,7 +1993,9 @@ SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte1
POSTHOOK: Input: default@materialized_cte2
+POSTHOOK: Input: default@src_partial_stats_value
#### A masked pattern was here ####
STAGE DEPENDENCIES:
Stage-1 is a root stage
@@ -2104,7 +2164,9 @@ SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2
PREHOOK: type: QUERY
+PREHOOK: Input: default@materialized_cte1
PREHOOK: Input: default@materialized_cte2
+PREHOOK: Input: default@src_partial_stats_value
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN CBO WITH materialized_cte1 AS (
SELECT * FROM src_partial_stats_value
@@ -2118,7 +2180,9 @@ SELECT * FROM materialized_cte2
UNION ALL
SELECT * FROM materialized_cte2
POSTHOOK: type: QUERY
+POSTHOOK: Input: default@materialized_cte1
POSTHOOK: Input: default@materialized_cte2
+POSTHOOK: Input: default@src_partial_stats_value
#### A masked pattern was here ####
CBO PLAN:
HiveProject(key=[$0], value=[$1])
diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_3.q.out
b/ql/src/test/results/clientpositive/llap/cte_mat_3.q.out
index d63d003dc26..955bd177902 100644
--- a/ql/src/test/results/clientpositive/llap/cte_mat_3.q.out
+++ b/ql/src/test/results/clientpositive/llap/cte_mat_3.q.out
@@ -5,6 +5,7 @@ from q1 a join q1 b
on a.key=b.key
PREHOOK: type: QUERY
PREHOOK: Input: default@q1
+PREHOOK: Input: default@src
#### A masked pattern was here ####
POSTHOOK: query: explain
with q1 as (select * from src where key= '5')
@@ -13,6 +14,7 @@ from q1 a join q1 b
on a.key=b.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@q1
+POSTHOOK: Input: default@src
#### A masked pattern was here ####
Plan optimized by CBO.
diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_4.q.out
b/ql/src/test/results/clientpositive/llap/cte_mat_4.q.out
index 4a1d6e0b06a..0c6a5384f66 100644
--- a/ql/src/test/results/clientpositive/llap/cte_mat_4.q.out
+++ b/ql/src/test/results/clientpositive/llap/cte_mat_4.q.out
@@ -30,6 +30,7 @@ from q1 a join q1 b
on a.key=b.key
PREHOOK: type: QUERY
PREHOOK: Input: default@q1
+PREHOOK: Input: default@src
#### A masked pattern was here ####
POSTHOOK: query: explain
with q1 as (select * from src where key= '5')
@@ -38,6 +39,7 @@ from q1 a join q1 b
on a.key=b.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@q1
+POSTHOOK: Input: default@src
#### A masked pattern was here ####
Plan optimized by CBO.
@@ -154,6 +156,7 @@ from q1 a join q1 b
on a.key=b.key
PREHOOK: type: QUERY
PREHOOK: Input: default@q1
+PREHOOK: Input: default@src
#### A masked pattern was here ####
POSTHOOK: query: explain
with q1 as (select * from src where key= '5')
@@ -162,6 +165,7 @@ from q1 a join q1 b
on a.key=b.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@q1
+POSTHOOK: Input: default@src
#### A masked pattern was here ####
Plan optimized by CBO.
diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_5.q.out
b/ql/src/test/results/clientpositive/llap/cte_mat_5.q.out
index 59d2e04afb5..f45d4c2cfb7 100644
--- a/ql/src/test/results/clientpositive/llap/cte_mat_5.q.out
+++ b/ql/src/test/results/clientpositive/llap/cte_mat_5.q.out
@@ -48,6 +48,7 @@ from mydb.q1 as a join q1 as b
on a.colnum=b.key
PREHOOK: type: QUERY
PREHOOK: Input: default@q1
+PREHOOK: Input: default@src
PREHOOK: Input: mydb@q1
#### A masked pattern was here ####
POSTHOOK: query: explain
@@ -57,6 +58,7 @@ from mydb.q1 as a join q1 as b
on a.colnum=b.key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@q1
+POSTHOOK: Input: default@src
POSTHOOK: Input: mydb@q1
#### A masked pattern was here ####
Plan optimized by CBO.
diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_9.q.out
b/ql/src/test/results/clientpositive/llap/cte_mat_9.q.out
index a5064535c42..363b96322fa 100644
--- a/ql/src/test/results/clientpositive/llap/cte_mat_9.q.out
+++ b/ql/src/test/results/clientpositive/llap/cte_mat_9.q.out
@@ -76,6 +76,7 @@ PREHOOK: type: QUERY
PREHOOK: Input: default@b1
PREHOOK: Input: default@b2
PREHOOK: Input: default@cte_mat_9_a
+PREHOOK: Input: default@cte_mat_9_b
#### A masked pattern was here ####
POSTHOOK: query: explain with a0 AS (
select id, 'a0' as tag from cte_mat_9_a
@@ -108,6 +109,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@b1
POSTHOOK: Input: default@b2
POSTHOOK: Input: default@cte_mat_9_a
+POSTHOOK: Input: default@cte_mat_9_b
#### A masked pattern was here ####
STAGE DEPENDENCIES:
Stage-1 is a root stage
diff --git a/ql/src/test/results/clientpositive/llap/cte_mat_type.q.out
b/ql/src/test/results/clientpositive/llap/cte_mat_type.q.out
index 0bf4bea932b..eb5bc18d555 100644
--- a/ql/src/test/results/clientpositive/llap/cte_mat_type.q.out
+++ b/ql/src/test/results/clientpositive/llap/cte_mat_type.q.out
@@ -50,6 +50,8 @@ FROM dept_avg da
WHERE da.avg_salary < 100000
PREHOOK: type: QUERY
PREHOOK: Input: default@dept_avg
+PREHOOK: Input: default@depts
+PREHOOK: Input: default@emps
#### A masked pattern was here ####
POSTHOOK: query: EXPLAIN CBO
WITH dept_avg AS (
@@ -67,6 +69,8 @@ FROM dept_avg da
WHERE da.avg_salary < 100000
POSTHOOK: type: QUERY
POSTHOOK: Input: default@dept_avg
+POSTHOOK: Input: default@depts
+POSTHOOK: Input: default@emps
#### A masked pattern was here ####
CBO PLAN:
HiveProject(d_name=[$0], _c1=[$1])
