This is an automated email from the ASF dual-hosted git repository.
rameshkumar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new bb0509f916d HIVE-28333: Skip CBO for UNIQUEJOIN and TABLESAMPLE (#5307)
bb0509f916d is described below
commit bb0509f916d2edfad104e8af95ea3e304f920e6e
Author: Soumyakanti Das <[email protected]>
AuthorDate: Fri Sep 6 10:17:46 2024 -0700
HIVE-28333: Skip CBO for UNIQUEJOIN and TABLESAMPLE (#5307)
* Skip CBO for UNIQUEJOIN and TABLESAMPLE
* Skip CBO for UNIONTYPE
* resolve test failure
* identify the CBO feature that is not supported while checking AST and QB
* Revert "identify the CBO feature that is not supported while checking AST
and QB"
This reverts commit 8c91a82014218b7093fcf895b7a4e89e28eb285f.
* Revert "resolve test failure"
This reverts commit e23bd4391f48ccf637be503a17dbe81fbcf29433.
* Revert "Skip CBO for UNIONTYPE"
This reverts commit 30d82aa38f176344899796b79cf57aa956c16b79.
* identify the CBO feature that is not supported while checking AST and QB
* fix failing tests
---
.../hive/ql/optimizer/calcite/HiveCalciteUtil.java | 18 +++++++++++++-----
.../apache/hadoop/hive/ql/parse/CalcitePlanner.java | 18 ++++++++----------
.../org/apache/hadoop/hive/ql/parse/ParseUtils.java | 21 ++++++++++++++-------
ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java | 14 ++++++++++++++
.../test/queries/clientnegative/bad_sample_clause.q | 2 ++
ql/src/test/queries/clientnegative/sample.q | 1 +
ql/src/test/queries/clientnegative/uniquejoin2.q | 2 ++
.../clientpositive/archive_excludeHadoop20.q | 1 +
ql/src/test/queries/clientpositive/bucket2.q | 1 +
ql/src/test/queries/clientpositive/bucket3.q | 1 +
ql/src/test/queries/clientpositive/bucket4.q | 1 +
ql/src/test/queries/clientpositive/bucket5.q | 1 +
ql/src/test/queries/clientpositive/bucket_many.q | 1 +
.../clientpositive/bucketsortoptimize_insert_1.q | 1 +
.../clientpositive/bucketsortoptimize_insert_2.q | 1 +
.../clientpositive/bucketsortoptimize_insert_3.q | 1 +
.../clientpositive/bucketsortoptimize_insert_4.q | 1 +
.../clientpositive/bucketsortoptimize_insert_5.q | 1 +
.../clientpositive/bucketsortoptimize_insert_6.q | 1 +
.../clientpositive/bucketsortoptimize_insert_7.q | 1 +
.../clientpositive/bucketsortoptimize_insert_8.q | 1 +
ql/src/test/queries/clientpositive/combine3.q | 1 +
ql/src/test/queries/clientpositive/create_view.q | 1 +
ql/src/test/queries/clientpositive/database.q | 1 +
.../clientpositive/disable_merge_for_bucketing.q | 1 +
.../test/queries/clientpositive/explainanalyze_2.q | 1 +
ql/src/test/queries/clientpositive/explainuser_2.q | 1 +
.../infer_bucket_sort_bucketed_table.q | 1 +
ql/src/test/queries/clientpositive/masking_5.q | 1 +
ql/src/test/queries/clientpositive/mm_buckets.q | 1 +
ql/src/test/queries/clientpositive/nonmr_fetch.q | 1 +
ql/src/test/queries/clientpositive/sample1.q | 2 ++
ql/src/test/queries/clientpositive/sample10.q | 2 +-
ql/src/test/queries/clientpositive/sample10_mm.q | 1 +
ql/src/test/queries/clientpositive/sample3.q | 2 ++
ql/src/test/queries/clientpositive/sample5.q | 2 ++
ql/src/test/queries/clientpositive/sample6.q | 1 +
ql/src/test/queries/clientpositive/sample7.q | 1 +
ql/src/test/queries/clientpositive/sample8.q | 1 +
ql/src/test/queries/clientpositive/sample9.q | 2 ++
ql/src/test/queries/clientpositive/smb_mapjoin_18.q | 1 +
ql/src/test/queries/clientpositive/smb_mapjoin_19.q | 1 +
ql/src/test/queries/clientpositive/smb_mapjoin_20.q | 1 +
ql/src/test/queries/clientpositive/smb_mapjoin_22.q | 1 +
ql/src/test/queries/clientpositive/stats10.q | 1 +
ql/src/test/queries/clientpositive/uniquejoin.q | 2 ++
.../clientpositive/llap/explainanalyze_2.q.out | 2 +-
.../results/clientpositive/llap/explainuser_2.q.out | 2 +-
48 files changed, 100 insertions(+), 25 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
index c944af8f6b2..fc48a623864 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
@@ -82,6 +82,7 @@ import
org.apache.hadoop.hive.ql.optimizer.calcite.translator.TypeConverter;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.ParseUtils;
+import org.apache.hadoop.hive.ql.parse.QB;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
@@ -101,12 +102,19 @@ import com.google.common.collect.Sets;
public class HiveCalciteUtil {
- public static boolean validateASTForUnsupportedTokens(ASTNode ast) {
- if (ParseUtils.containsTokenOfType(ast, HiveParser.TOK_CHARSETLITERAL,
HiveParser.TOK_TABLESPLITSAMPLE)) {
- return false;
- } else {
- return true;
+ public static Pair<Boolean, String>
unsupportedFeaturesPresentInASTorQB(ASTNode ast, QB qb) {
+ Pair<Boolean, String> containsToken =
+ ParseUtils.containsTokenOfType(ast, HiveParser.TOK_CHARSETLITERAL,
HiveParser.TOK_TABLESPLITSAMPLE,
+ HiveParser.TOK_UNIQUEJOIN, HiveParser.TOK_TABLEBUCKETSAMPLE);
+
+ if (Boolean.TRUE.equals(containsToken.getKey())) {
+ return containsToken;
}
+ if (qb.hasTableSampleRecursive()) {
+ return Pair.of(true, "TOK_TABLEBUCKETSAMPLE");
+ }
+
+ return Pair.of(false, null);
}
public static List<RexNode> getProjsFromBelowAsInputRef(final RelNode rel) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 24680c113f5..e9f5fde1a22 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -927,8 +927,8 @@ public class CalcitePlanner extends SemanticAnalyzer {
boolean isSupportedType = (qb.getIsQuery())
|| qb.isCTAS() || qb.isMaterializedView() || cboCtx.type ==
PreCboCtx.Type.INSERT
|| cboCtx.type == PreCboCtx.Type.MULTI_INSERT;
- boolean noBadTokens = HiveCalciteUtil.validateASTForUnsupportedTokens(ast);
- boolean result = isSupportedRoot && isSupportedType && noBadTokens;
+ Pair<Boolean, String> unsupportedFeatures =
HiveCalciteUtil.unsupportedFeaturesPresentInASTorQB(ast, qb);
+ boolean result = isSupportedRoot && isSupportedType &&
Boolean.FALSE.equals(unsupportedFeatures.getKey());
String msg = "";
if (!result) {
@@ -939,11 +939,8 @@ public class CalcitePlanner extends SemanticAnalyzer {
msg += "is not a query with at least one source table "
+ " or there is a subquery without a source table, or CTAS, or
insert; ";
}
- if (!noBadTokens) {
- msg += "has unsupported tokens; ";
- }
- if (msg.isEmpty()) {
- msg += "has some unspecified limitations; ";
+ if (Boolean.TRUE.equals(unsupportedFeatures.getKey())) {
+ msg += "has unsupported feature [" + unsupportedFeatures.getValue() +
"]; ";
}
msg = msg.substring(0, msg.length() - 2);
if (needToLogMessage) {
@@ -4639,11 +4636,12 @@ public class CalcitePlanner extends SemanticAnalyzer {
unescapeIdentifier(expr.getChild(0).getChild(0).getText().toLowerCase()),
expr, columnList, excludedColumns, inputRR, starRR, pos,
outputRR, qb.getAliases(), true);
- } else if (ParseUtils.containsTokenOfType(expr,
HiveParser.TOK_FUNCTIONDI) &&
- !ParseUtils.containsTokenOfType(expr, HiveParser.TOK_WINDOWSPEC)
&&
+ } else if (
+ Boolean.TRUE.equals(ParseUtils.containsTokenOfType(expr,
HiveParser.TOK_FUNCTIONDI).getKey()) &&
+ Boolean.FALSE.equals(ParseUtils.containsTokenOfType(expr,
HiveParser.TOK_WINDOWSPEC).getKey()) &&
!(srcRel instanceof HiveAggregate ||
(srcRel.getInputs().size() == 1 && srcRel.getInput(0) instanceof
HiveAggregate))) {
- // Likely a malformed query eg, select hash(distinct c1) from t1;
+ // Likely a malformed query eg, select hash(distinct c1) from t1
throw new CalciteSemanticException("Distinct without an
aggregation.",
UnsupportedFeature.Distinct_without_an_aggregation);
} else {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
index a4280ce2398..9cbd47123a0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
@@ -24,6 +24,7 @@ import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.ArrayDeque;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -37,6 +38,7 @@ import com.google.common.collect.Lists;
import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.Tree;
import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.util.Pair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -278,18 +280,23 @@ public final class ParseUtils {
return className;
}
- public static boolean containsTokenOfType(ASTNode root, Integer ... tokens) {
- final Set<Integer> tokensToMatch = new HashSet<Integer>();
- for (Integer tokenTypeToMatch : tokens) {
- tokensToMatch.add(tokenTypeToMatch);
- }
+ public static Pair<Boolean, String> containsTokenOfType(ASTNode root,
Integer ... tokens) {
+ final Set<Integer> tokensToMatch = new HashSet<>(Arrays.asList(tokens));
+ final String[] matched = {null};
- return ParseUtils.containsTokenOfType(root, new
PTFUtils.Predicate<ASTNode>() {
+ boolean check = ParseUtils.containsTokenOfType(root, new
PTFUtils.Predicate<ASTNode>() {
@Override
public boolean apply(ASTNode node) {
- return tokensToMatch.contains(node.getType());
+ if (tokensToMatch.contains(node.getType())) {
+ matched[0] = node.getText();
+ return true;
+ }
+
+ return false;
}
});
+
+ return Pair.of(check, matched[0]);
}
public static boolean containsTokenOfType(ASTNode root,
PTFUtils.Predicate<ASTNode> predicate) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java
index ca347878220..fa18ab82c52 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/QB.java
@@ -316,6 +316,20 @@ public class QB {
public boolean hasTableSample(String alias) {
return qbp.getTabSample(alias) != null;
}
+
+ public boolean hasTableSampleRecursive() {
+ if (!qbp.getNameToSample().isEmpty()) {
+ return true;
+ }
+
+ for (QBExpr qbExpr: aliasToSubq.values()) {
+ if (qbExpr.getQB() != null && qbExpr.getQB().hasTableSampleRecursive()) {
+ return true;
+ }
+ }
+
+ return false;
+ }
public CreateTableDesc getTableDesc() {
return tblDesc;
diff --git a/ql/src/test/queries/clientnegative/bad_sample_clause.q
b/ql/src/test/queries/clientnegative/bad_sample_clause.q
index c06bb84af95..ba4d794d31e 100644
--- a/ql/src/test/queries/clientnegative/bad_sample_clause.q
+++ b/ql/src/test/queries/clientnegative/bad_sample_clause.q
@@ -1,4 +1,6 @@
--! qt:dataset:srcpart
+set hive.cbo.fallback.strategy=NEVER;
+
CREATE TABLE dest1(key INT, value STRING, dt STRING, hr STRING) STORED AS
TEXTFILE;
INSERT OVERWRITE TABLE dest1 SELECT s.*
diff --git a/ql/src/test/queries/clientnegative/sample.q
b/ql/src/test/queries/clientnegative/sample.q
index 7f51f0bcdd5..f561797cb90 100644
--- a/ql/src/test/queries/clientnegative/sample.q
+++ b/ql/src/test/queries/clientnegative/sample.q
@@ -1,2 +1,3 @@
--! qt:dataset:srcbucket
+set hive.cbo.fallback.strategy=NEVER;
explain extended SELECT s.* FROM srcbucket TABLESAMPLE (BUCKET 5 OUT OF 4 on
key) s
\ No newline at end of file
diff --git a/ql/src/test/queries/clientnegative/uniquejoin2.q
b/ql/src/test/queries/clientnegative/uniquejoin2.q
index f14f38eaf43..29687d143cc 100644
--- a/ql/src/test/queries/clientnegative/uniquejoin2.q
+++ b/ql/src/test/queries/clientnegative/uniquejoin2.q
@@ -1,4 +1,6 @@
--! qt:dataset:src
+set hive.cbo.fallback.strategy=NEVER;
+
FROM UNIQUEJOIN src a (a.key), PRESERVE src b (b.key, b.val)
SELECT a.key, b.key;
diff --git a/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q
b/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q
index 6f77fa8a005..fe13555b673 100644
--- a/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q
+++ b/ql/src/test/queries/clientpositive/archive_excludeHadoop20.q
@@ -4,6 +4,7 @@ set hive.mapred.mode=nonstrict;
set hive.archive.enabled = true;
set hive.exec.submitviachild=false;
set hive.exec.submit.local.task.via.child=false;
+set hive.cbo.fallback.strategy=NEVER;
drop table tstsrc_n2;
drop table tstsrcpart_n2;
diff --git a/ql/src/test/queries/clientpositive/bucket2.q
b/ql/src/test/queries/clientpositive/bucket2.q
index c7ecb6e678d..45490f4cf66 100644
--- a/ql/src/test/queries/clientpositive/bucket2.q
+++ b/ql/src/test/queries/clientpositive/bucket2.q
@@ -2,6 +2,7 @@
set hive.explain.user=false;
;
set hive.exec.reducers.max = 1;
+set hive.cbo.fallback.strategy=NEVER;
-- SORT_QUERY_RESULTS
diff --git a/ql/src/test/queries/clientpositive/bucket3.q
b/ql/src/test/queries/clientpositive/bucket3.q
index fd8b2e374a9..c4628e285e1 100644
--- a/ql/src/test/queries/clientpositive/bucket3.q
+++ b/ql/src/test/queries/clientpositive/bucket3.q
@@ -2,6 +2,7 @@
set hive.explain.user=false;
;
set hive.exec.reducers.max = 1;
+set hive.cbo.fallback.strategy=NEVER;
-- SORT_QUERY_RESULTS
diff --git a/ql/src/test/queries/clientpositive/bucket4.q
b/ql/src/test/queries/clientpositive/bucket4.q
index e3fe6605577..fa28083323c 100644
--- a/ql/src/test/queries/clientpositive/bucket4.q
+++ b/ql/src/test/queries/clientpositive/bucket4.q
@@ -4,6 +4,7 @@ set
hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
;
set hive.exec.reducers.max = 1;
+set hive.cbo.fallback.strategy=NEVER;
CREATE TABLE bucket4_1(key int, value string) CLUSTERED BY (key) SORTED BY
(key) INTO 2 BUCKETS;
diff --git a/ql/src/test/queries/clientpositive/bucket5.q
b/ql/src/test/queries/clientpositive/bucket5.q
index 7c3da98ea10..bf95edfd824 100644
--- a/ql/src/test/queries/clientpositive/bucket5.q
+++ b/ql/src/test/queries/clientpositive/bucket5.q
@@ -7,6 +7,7 @@ set hive.merge.mapfiles = true;
set hive.merge.mapredfiles = true;
set hive.merge.tezfiles = true;
set mapred.reduce.tasks = 2;
+set hive.cbo.fallback.strategy=NEVER;
-- Tests that when a multi insert inserts into a bucketed table and a table
which is not bucketed
-- the bucketed table is not merged and the table which is not bucketed is
diff --git a/ql/src/test/queries/clientpositive/bucket_many.q
b/ql/src/test/queries/clientpositive/bucket_many.q
index 8abcdc746b7..6a2ed55ffd5 100644
--- a/ql/src/test/queries/clientpositive/bucket_many.q
+++ b/ql/src/test/queries/clientpositive/bucket_many.q
@@ -1,6 +1,7 @@
--! qt:dataset:src
;
set mapred.reduce.tasks = 16;
+set hive.cbo.fallback.strategy=NEVER;
create table bucket_many(key int, value string) clustered by (key) into 256
buckets;
diff --git a/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_1.q
b/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_1.q
index 5bfdf48fad8..2832ffd9f23 100644
--- a/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_1.q
+++ b/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_1.q
@@ -9,6 +9,7 @@ set hive.merge.mapfiles=false;
set hive.merge.mapredfiles=false;
set hive.auto.convert.sortmerge.join.to.mapjoin=true;
+set hive.cbo.fallback.strategy=NEVER;
-- Create two bucketed and sorted tables
CREATE TABLE test_table1_n5 (key INT, value STRING) PARTITIONED BY (ds STRING)
diff --git a/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_2.q
b/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_2.q
index 0b913caa8e0..6a134837087 100644
--- a/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_2.q
+++ b/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_2.q
@@ -15,6 +15,7 @@ set
hive.auto.convert.sortmerge.join.bigtable.selection.policy=org.apache.hadoop
set hive.auto.convert.sortmerge.join.to.mapjoin=true;
-- disable hash joins
set hive.auto.convert.join.noconditionaltask.size=10;
+set hive.cbo.fallback.strategy=NEVER;
-- Create two bucketed and sorted tables
CREATE TABLE test_table1_n0 (key INT, value STRING) PARTITIONED BY (ds STRING)
diff --git a/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_3.q
b/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_3.q
index 289d10ec1c7..7dd15a8967a 100644
--- a/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_3.q
+++ b/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_3.q
@@ -9,6 +9,7 @@ set hive.merge.mapfiles=false;
set hive.merge.mapredfiles=false;
set hive.auto.convert.sortmerge.join.to.mapjoin=true;
+set hive.cbo.fallback.strategy=NEVER;
-- Create two bucketed and sorted tables
CREATE TABLE test_table1_n19 (key INT, value STRING) PARTITIONED BY (ds STRING)
diff --git a/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_4.q
b/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_4.q
index 25bb8727c46..01eb6eb1d37 100644
--- a/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_4.q
+++ b/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_4.q
@@ -13,6 +13,7 @@ set hive.merge.mapredfiles=false;
set
hive.auto.convert.sortmerge.join.bigtable.selection.policy=org.apache.hadoop.hive.ql.optimizer.LeftmostBigTableSelectorForAutoSMJ;
set hive.auto.convert.sortmerge.join.to.mapjoin=true;
+set hive.cbo.fallback.strategy=NEVER;
-- Create two bucketed and sorted tables
CREATE TABLE test_table1_n16 (key INT, value STRING) PARTITIONED BY (ds STRING)
diff --git a/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_5.q
b/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_5.q
index 038c03c0662..8cb3b68723d 100644
--- a/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_5.q
+++ b/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_5.q
@@ -13,6 +13,7 @@ set hive.merge.mapredfiles=false;
set
hive.auto.convert.sortmerge.join.bigtable.selection.policy=org.apache.hadoop.hive.ql.optimizer.LeftmostBigTableSelectorForAutoSMJ;
set hive.auto.convert.sortmerge.join.to.mapjoin=true;
+set hive.cbo.fallback.strategy=NEVER;
-- Create two bucketed and sorted tables
CREATE TABLE test_table1_n8 (key INT, value STRING) PARTITIONED BY (ds STRING)
diff --git a/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_6.q
b/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_6.q
index 2d4907c5cf1..2f82a38fce3 100644
--- a/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_6.q
+++ b/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_6.q
@@ -13,6 +13,7 @@ set hive.merge.mapredfiles=false;
set
hive.auto.convert.sortmerge.join.bigtable.selection.policy=org.apache.hadoop.hive.ql.optimizer.LeftmostBigTableSelectorForAutoSMJ;
set hive.auto.convert.sortmerge.join.to.mapjoin=true;
+set hive.cbo.fallback.strategy=NEVER;
-- Create two bucketed and sorted tables
CREATE TABLE test_table1_n3 (key INT, key2 INT, value STRING) PARTITIONED BY
(ds STRING)
diff --git a/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_7.q
b/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_7.q
index bef48b2d0a4..ba300deabe7 100644
--- a/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_7.q
+++ b/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_7.q
@@ -15,6 +15,7 @@ set
hive.auto.convert.sortmerge.join.bigtable.selection.policy=org.apache.hadoop
set hive.auto.convert.sortmerge.join.to.mapjoin=true;
-- disable hash joins
set hive.auto.convert.join.noconditionaltask.size=10;
+set hive.cbo.fallback.strategy=NEVER;
-- Create two bucketed and sorted tables
CREATE TABLE test_table1_n20 (key INT, value STRING) PARTITIONED BY (ds STRING)
diff --git a/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_8.q
b/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_8.q
index 1f66e2035f8..2570b796d25 100644
--- a/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_8.q
+++ b/ql/src/test/queries/clientpositive/bucketsortoptimize_insert_8.q
@@ -13,6 +13,7 @@ set hive.merge.mapredfiles=false;
set
hive.auto.convert.sortmerge.join.bigtable.selection.policy=org.apache.hadoop.hive.ql.optimizer.LeftmostBigTableSelectorForAutoSMJ;
set hive.auto.convert.sortmerge.join.to.mapjoin=true;
+set hive.cbo.fallback.strategy=NEVER;
-- Create two bucketed and sorted tables
CREATE TABLE test_table1_n2 (key INT, value STRING) PARTITIONED BY (ds STRING)
diff --git a/ql/src/test/queries/clientpositive/combine3.q
b/ql/src/test/queries/clientpositive/combine3.q
index 479108f7da3..47d833137b6 100644
--- a/ql/src/test/queries/clientpositive/combine3.q
+++ b/ql/src/test/queries/clientpositive/combine3.q
@@ -5,6 +5,7 @@ set mapred.min.split.size=256;
set mapred.min.split.size.per.node=256;
set mapred.min.split.size.per.rack=256;
set mapred.max.split.size=256;
+set hive.cbo.fallback.strategy=NEVER;
drop table combine_3_srcpart_seq_rc;
diff --git a/ql/src/test/queries/clientpositive/create_view.q
b/ql/src/test/queries/clientpositive/create_view.q
index b712c2e8994..36a350479b1 100644
--- a/ql/src/test/queries/clientpositive/create_view.q
+++ b/ql/src/test/queries/clientpositive/create_view.q
@@ -2,6 +2,7 @@
--! qt:dataset:src1
--! qt:dataset:src
set hive.mapred.mode=nonstrict;
+set hive.cbo.fallback.strategy=NEVER;
DROP VIEW view1;
DROP VIEW view2;
DROP VIEW view3;
diff --git a/ql/src/test/queries/clientpositive/database.q
b/ql/src/test/queries/clientpositive/database.q
index bfa9d1aad32..ce1be8501d5 100644
--- a/ql/src/test/queries/clientpositive/database.q
+++ b/ql/src/test/queries/clientpositive/database.q
@@ -3,6 +3,7 @@
--! qt:dataset:src
set hive.mapred.mode=nonstrict;
set hive.support.concurrency = true;
+set hive.cbo.fallback.strategy=NEVER;
-- SORT_QUERY_RESULTS
diff --git a/ql/src/test/queries/clientpositive/disable_merge_for_bucketing.q
b/ql/src/test/queries/clientpositive/disable_merge_for_bucketing.q
index eedbf71e046..40f6e6eb268 100644
--- a/ql/src/test/queries/clientpositive/disable_merge_for_bucketing.q
+++ b/ql/src/test/queries/clientpositive/disable_merge_for_bucketing.q
@@ -5,6 +5,7 @@ set
hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
;
set hive.exec.reducers.max = 1;
set hive.merge.mapredfiles=true;
+set hive.cbo.fallback.strategy=NEVER;
CREATE TABLE bucket2_1_n0(key int, value string) CLUSTERED BY (key) INTO 2
BUCKETS;
diff --git a/ql/src/test/queries/clientpositive/explainanalyze_2.q
b/ql/src/test/queries/clientpositive/explainanalyze_2.q
index 7e48a8a2b81..ad0f5751d72 100644
--- a/ql/src/test/queries/clientpositive/explainanalyze_2.q
+++ b/ql/src/test/queries/clientpositive/explainanalyze_2.q
@@ -6,6 +6,7 @@ set hive.map.aggr=false;
set hive.strict.checks.bucketing=false;
set hive.explain.user=true;
+set hive.cbo.fallback.strategy=NEVER;
explain analyze
SELECT x.key, z.value, y.value
diff --git a/ql/src/test/queries/clientpositive/explainuser_2.q
b/ql/src/test/queries/clientpositive/explainuser_2.q
index bc795cfd315..1f905c594aa 100644
--- a/ql/src/test/queries/clientpositive/explainuser_2.q
+++ b/ql/src/test/queries/clientpositive/explainuser_2.q
@@ -5,6 +5,7 @@ set hive.strict.checks.bucketing=false;
set hive.explain.user=true;
set hive.metastore.aggregate.stats.cache.enabled=false;
+set hive.cbo.fallback.strategy=NEVER;
-- SORT_QUERY_RESULTS
diff --git
a/ql/src/test/queries/clientpositive/infer_bucket_sort_bucketed_table.q
b/ql/src/test/queries/clientpositive/infer_bucket_sort_bucketed_table.q
index 0d4fcedc972..5cbb6338a08 100644
--- a/ql/src/test/queries/clientpositive/infer_bucket_sort_bucketed_table.q
+++ b/ql/src/test/queries/clientpositive/infer_bucket_sort_bucketed_table.q
@@ -3,6 +3,7 @@
set hive.exec.infer.bucket.sort=true;
+set hive.cbo.fallback.strategy=NEVER;
-- Test writing to a bucketed table, the output should be bucketed by the
bucketing key into the
-- a number of files equal to the number of buckets
diff --git a/ql/src/test/queries/clientpositive/masking_5.q
b/ql/src/test/queries/clientpositive/masking_5.q
index 903afdbb4d9..cf5a22d80fd 100644
--- a/ql/src/test/queries/clientpositive/masking_5.q
+++ b/ql/src/test/queries/clientpositive/masking_5.q
@@ -2,6 +2,7 @@
set hive.mapred.mode=nonstrict;
set hive.security.authorization.enabled=true;
set
hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
+set hive.cbo.fallback.strategy=NEVER;
create table masking_test_n6 as select cast(key as int) as key, value from src;
diff --git a/ql/src/test/queries/clientpositive/mm_buckets.q
b/ql/src/test/queries/clientpositive/mm_buckets.q
index 5686c8e11e2..c0cd8d103c0 100644
--- a/ql/src/test/queries/clientpositive/mm_buckets.q
+++ b/ql/src/test/queries/clientpositive/mm_buckets.q
@@ -8,6 +8,7 @@ set tez.grouping.min-size=1;
set tez.grouping.max-size=2;
set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.cbo.fallback.strategy=NEVER;
-- Bucketing tests are slow and some tablesample ones don't work w/o MM
diff --git a/ql/src/test/queries/clientpositive/nonmr_fetch.q
b/ql/src/test/queries/clientpositive/nonmr_fetch.q
index 5b7e61ec005..10419a8a37d 100644
--- a/ql/src/test/queries/clientpositive/nonmr_fetch.q
+++ b/ql/src/test/queries/clientpositive/nonmr_fetch.q
@@ -4,6 +4,7 @@ set hive.mapred.mode=nonstrict;
set hive.fetch.task.conversion=minimal;
set hive.exec.submitviachild=false;
set hive.exec.submit.local.task.via.child=false;
+set hive.cbo.fallback.strategy=NEVER;
-- backward compatible (minimal)
explain select * from src limit 10;
diff --git a/ql/src/test/queries/clientpositive/sample1.q
b/ql/src/test/queries/clientpositive/sample1.q
index c7dcbbaa656..dc52a7af811 100644
--- a/ql/src/test/queries/clientpositive/sample1.q
+++ b/ql/src/test/queries/clientpositive/sample1.q
@@ -1,5 +1,7 @@
--! qt:dataset:srcpart
--! qt:dataset:srcbucket
+set hive.cbo.fallback.strategy=NEVER;
+
CREATE TABLE dest1_n89(key INT, value STRING, dt STRING, hr STRING) STORED AS
TEXTFILE;
-- no input pruning, no sample filter
diff --git a/ql/src/test/queries/clientpositive/sample10.q
b/ql/src/test/queries/clientpositive/sample10.q
index 914763c9363..c9b36281f06 100644
--- a/ql/src/test/queries/clientpositive/sample10.q
+++ b/ql/src/test/queries/clientpositive/sample10.q
@@ -8,7 +8,7 @@ set hive.exec.reducers.max=4;
set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
set hive.default.fileformat=RCFILE;
set hive.exec.pre.hooks =
org.apache.hadoop.hive.ql.hooks.PreExecutePrinter,org.apache.hadoop.hive.ql.hooks.EnforceReadOnlyTables,org.apache.hadoop.hive.ql.hooks.UpdateInputAccessTimeHook$PreExec;
-
+set hive.cbo.fallback.strategy=NEVER;
create table srcpartbucket (key string, value string) partitioned by (ds
string, hr string) clustered by (key) into 4 buckets;
diff --git a/ql/src/test/queries/clientpositive/sample10_mm.q
b/ql/src/test/queries/clientpositive/sample10_mm.q
index 04e3d4b1fad..e54a3dc42ff 100644
--- a/ql/src/test/queries/clientpositive/sample10_mm.q
+++ b/ql/src/test/queries/clientpositive/sample10_mm.q
@@ -11,6 +11,7 @@ set hive.exec.pre.hooks =
org.apache.hadoop.hive.ql.hooks.PreExecutePrinter,org.
set hive.support.concurrency=true;
set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.cbo.fallback.strategy=NEVER;
create table srcpartbucket (key string, value string) partitioned by (ds
string, hr string) clustered by (key) into 4 buckets stored as orc
tblproperties ("transactional"="true",
"transactional_properties"="insert_only");
diff --git a/ql/src/test/queries/clientpositive/sample3.q
b/ql/src/test/queries/clientpositive/sample3.q
index 0318e64c225..f1eba902064 100644
--- a/ql/src/test/queries/clientpositive/sample3.q
+++ b/ql/src/test/queries/clientpositive/sample3.q
@@ -2,6 +2,8 @@
-- SORT_QUERY_RESULTS
-- no input pruning, sample filter
+set hive.cbo.fallback.strategy=NEVER;
+
EXPLAIN
SELECT s.key
FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 5 on key) s;
diff --git a/ql/src/test/queries/clientpositive/sample5.q
b/ql/src/test/queries/clientpositive/sample5.q
index a659f2b5d55..ebba7061cc9 100644
--- a/ql/src/test/queries/clientpositive/sample5.q
+++ b/ql/src/test/queries/clientpositive/sample5.q
@@ -1,4 +1,6 @@
--! qt:dataset:srcbucket
+set hive.cbo.fallback.strategy=NEVER;
+
CREATE TABLE dest1_n69(key INT, value STRING) STORED AS TEXTFILE;
-- SORT_QUERY_RESULTS
diff --git a/ql/src/test/queries/clientpositive/sample6.q
b/ql/src/test/queries/clientpositive/sample6.q
index 292338cb0ba..d1e8eba1dba 100644
--- a/ql/src/test/queries/clientpositive/sample6.q
+++ b/ql/src/test/queries/clientpositive/sample6.q
@@ -1,6 +1,7 @@
--! qt:dataset:srcbucket2
--! qt:dataset:srcbucket
set hive.mapred.mode=nonstrict;
+set hive.cbo.fallback.strategy=NEVER;
CREATE TABLE dest1_n27(key INT, value STRING) STORED AS TEXTFILE;
-- both input pruning and sample filter
diff --git a/ql/src/test/queries/clientpositive/sample7.q
b/ql/src/test/queries/clientpositive/sample7.q
index dbbd26292ce..307b610f202 100644
--- a/ql/src/test/queries/clientpositive/sample7.q
+++ b/ql/src/test/queries/clientpositive/sample7.q
@@ -1,4 +1,5 @@
--! qt:dataset:srcbucket
+set hive.cbo.fallback.strategy=NEVER;
CREATE TABLE dest1_n160(key INT, value STRING) STORED AS TEXTFILE;
-- both input pruning and sample filter
diff --git a/ql/src/test/queries/clientpositive/sample8.q
b/ql/src/test/queries/clientpositive/sample8.q
index 33304f4c737..67b0e2b844c 100644
--- a/ql/src/test/queries/clientpositive/sample8.q
+++ b/ql/src/test/queries/clientpositive/sample8.q
@@ -4,6 +4,7 @@
set hive.mapred.mode=nonstrict;
-- sampling with join and alias
-- SORT_QUERY_RESULTS
+set hive.cbo.fallback.strategy=NEVER;
EXPLAIN EXTENDED
SELECT s.*
diff --git a/ql/src/test/queries/clientpositive/sample9.q
b/ql/src/test/queries/clientpositive/sample9.q
index 46949cd282f..d5837432a69 100644
--- a/ql/src/test/queries/clientpositive/sample9.q
+++ b/ql/src/test/queries/clientpositive/sample9.q
@@ -1,4 +1,6 @@
--! qt:dataset:srcbucket
+set hive.cbo.fallback.strategy=NEVER;
+
EXPLAIN EXTENDED
SELECT s.*
FROM (SELECT a.* FROM srcbucket TABLESAMPLE (BUCKET 1 OUT OF 2 on key) a) s;
diff --git a/ql/src/test/queries/clientpositive/smb_mapjoin_18.q
b/ql/src/test/queries/clientpositive/smb_mapjoin_18.q
index 64e4b29755c..19ef5462ce5 100644
--- a/ql/src/test/queries/clientpositive/smb_mapjoin_18.q
+++ b/ql/src/test/queries/clientpositive/smb_mapjoin_18.q
@@ -6,6 +6,7 @@ set hive.optimize.bucketmapjoin.sortedmerge = true;
set hive.exec.reducers.max = 1;
set hive.merge.mapfiles=false;
set hive.merge.mapredfiles=false;
+set hive.cbo.fallback.strategy=NEVER;
-- Create two bucketed and sorted tables
CREATE TABLE test_table1_n7 (key INT, value STRING) PARTITIONED BY (ds STRING)
diff --git a/ql/src/test/queries/clientpositive/smb_mapjoin_19.q
b/ql/src/test/queries/clientpositive/smb_mapjoin_19.q
index 8b9c3982e26..f0caab8f816 100644
--- a/ql/src/test/queries/clientpositive/smb_mapjoin_19.q
+++ b/ql/src/test/queries/clientpositive/smb_mapjoin_19.q
@@ -8,6 +8,7 @@ set hive.merge.mapfiles=false;
set hive.merge.mapredfiles=false;
set hive.metastore.aggregate.stats.cache.enabled=false;
+set hive.cbo.fallback.strategy=NEVER;
-- Create two bucketed and sorted tables
CREATE TABLE test_table1_n11 (key INT, value STRING) PARTITIONED BY (ds STRING)
diff --git a/ql/src/test/queries/clientpositive/smb_mapjoin_20.q
b/ql/src/test/queries/clientpositive/smb_mapjoin_20.q
index 94842ba7177..6b012b90dd7 100644
--- a/ql/src/test/queries/clientpositive/smb_mapjoin_20.q
+++ b/ql/src/test/queries/clientpositive/smb_mapjoin_20.q
@@ -8,6 +8,7 @@ set hive.optimize.bucketmapjoin.sortedmerge = true;
set hive.exec.reducers.max = 1;
set hive.merge.mapfiles=false;
set hive.merge.mapredfiles=false;
+set hive.cbo.fallback.strategy=NEVER;
-- Create two bucketed and sorted tables
CREATE TABLE test_table1 (key int, value STRING) PARTITIONED BY (ds STRING)
diff --git a/ql/src/test/queries/clientpositive/smb_mapjoin_22.q
b/ql/src/test/queries/clientpositive/smb_mapjoin_22.q
index c7fc2d3743f..dac70cc8089 100644
--- a/ql/src/test/queries/clientpositive/smb_mapjoin_22.q
+++ b/ql/src/test/queries/clientpositive/smb_mapjoin_22.q
@@ -6,6 +6,7 @@ set hive.optimize.bucketmapjoin.sortedmerge = true;
set hive.exec.reducers.max = 1;
set hive.merge.mapfiles=false;
set hive.merge.mapredfiles=false;
+set hive.cbo.fallback.strategy=NEVER;
-- Create two bucketed and sorted tables
CREATE TABLE test_table1_n10 (key INT, value STRING)
diff --git a/ql/src/test/queries/clientpositive/stats10.q
b/ql/src/test/queries/clientpositive/stats10.q
index 0fd7054e4b7..9276bc5433e 100644
--- a/ql/src/test/queries/clientpositive/stats10.q
+++ b/ql/src/test/queries/clientpositive/stats10.q
@@ -4,6 +4,7 @@ set datanucleus.cache.collections=false;
set hive.stats.autogather=true;
;
set hive.exec.reducers.max = 1;
+set hive.cbo.fallback.strategy=NEVER;
CREATE TABLE bucket3_1_n0(key int, value string) partitioned by (ds string)
CLUSTERED BY (key) INTO 2 BUCKETS;
diff --git a/ql/src/test/queries/clientpositive/uniquejoin.q
b/ql/src/test/queries/clientpositive/uniquejoin.q
index e365896f618..d3892dfe80c 100644
--- a/ql/src/test/queries/clientpositive/uniquejoin.q
+++ b/ql/src/test/queries/clientpositive/uniquejoin.q
@@ -1,3 +1,5 @@
+set hive.cbo.fallback.strategy=NEVER;
+
CREATE TABLE T1_n1(key STRING, val STRING) STORED AS TEXTFILE;
CREATE TABLE T2_n1(key STRING, val STRING) STORED AS TEXTFILE;
CREATE TABLE T3_n0(key STRING, val STRING) STORED AS TEXTFILE;
diff --git a/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out
b/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out
index 4ca869cd77e..dfa7cdb7e24 100644
--- a/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/explainanalyze_2.q.out
@@ -2012,7 +2012,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
#### A masked pattern was here ####
-Plan not optimized by CBO due to missing feature [Unique_join].
+Plan not optimized by CBO because the statement has unsupported feature
[TOK_UNIQUEJOIN]
Vertex dependency in root stage
Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)
diff --git a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
index a078d68a155..00c18780bff 100644
--- a/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
+++ b/ql/src/test/results/clientpositive/llap/explainuser_2.q.out
@@ -3633,7 +3633,7 @@ POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
POSTHOOK: Output: hdfs://### HDFS PATH ###
-Plan not optimized by CBO due to missing feature [Unique_join].
+Plan not optimized by CBO because the statement has unsupported feature
[TOK_UNIQUEJOIN]
Vertex dependency in root stage
Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 3 (SIMPLE_EDGE), Map 4 (SIMPLE_EDGE)