This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 36df716299c HIVE-28193: Skip CBO fallback when EXCEPT, INTERSECT, or
QUALIFY are used (Shohei Okumiya, reviewed by Stamatis Zampetakis, Denys
Kuzmenko)
36df716299c is described below
commit 36df716299cd377210b13aca5286a4678293ca90
Author: okumin <[email protected]>
AuthorDate: Mon Sep 16 00:24:33 2024 +0900
HIVE-28193: Skip CBO fallback when EXCEPT, INTERSECT, or QUALIFY are used
(Shohei Okumiya, reviewed by Stamatis Zampetakis, Denys Kuzmenko)
Closes #5312
---
.../org/apache/hadoop/hive/ql/QueryProperties.java | 30 ++++++
.../hadoop/hive/ql/parse/CalcitePlanner.java | 5 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 5 +
.../hadoop/hive/ql/parse/TestQueryProperties.java | 113 +++++++++++++++++++++
.../clientnegative/cbo_fallback_skip_with_except.q | 2 +
.../cbo_fallback_skip_with_except.q.out | 1 +
6 files changed, 155 insertions(+), 1 deletion(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
index eec7f548b19..1cf65994c54 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
@@ -49,6 +49,9 @@ public class QueryProperties {
boolean hasJoinFollowedByGroupBy = false;
boolean hasPTF = false;
boolean hasWindowing = false;
+ boolean hasQualify = false;
+ boolean hasExcept = false;
+ boolean hasIntersect = false;
// does the query have a using clause
boolean usesScript = false;
@@ -242,6 +245,30 @@ public class QueryProperties {
this.hasWindowing = hasWindowing;
}
+ public boolean hasQualify() {
+ return hasQualify;
+ }
+
+ public void setHasQualify(boolean hasQualify) {
+ this.hasQualify = hasQualify;
+ }
+
+ public boolean hasExcept() {
+ return hasExcept;
+ }
+
+ public void setHasExcept(boolean hasExcept) {
+ this.hasExcept = hasExcept;
+ }
+
+ public boolean hasIntersect() {
+ return hasIntersect;
+ }
+
+ public void setHasIntersect(boolean hasIntersect) {
+ this.hasIntersect = hasIntersect;
+ }
+
public boolean isMapJoinRemoved() {
return mapJoinRemoved;
}
@@ -311,6 +338,9 @@ public class QueryProperties {
hasJoinFollowedByGroupBy = false;
hasPTF = false;
hasWindowing = false;
+ hasQualify = false;
+ hasExcept = false;
+ hasIntersect = false;
// does the query have a using clause
usesScript = false;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 3e97b7d6331..72c195aaefe 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -689,7 +689,10 @@ public class CalcitePlanner extends SemanticAnalyzer {
this.ctx.setCboInfo(cboMsg);
// Determine if we should re-throw the exception OR if we try to
mark the query to retry as non-CBO.
- if (fallbackStrategy.isFatal(e)) {
+ final boolean requiresCBO = queryProperties.hasQualify()
+ || queryProperties.hasExcept()
+ || queryProperties.hasIntersect();
+ if (requiresCBO || fallbackStrategy.isFatal(e)) {
if (e instanceof RuntimeException || e instanceof
SemanticException) {
// These types of exceptions do not need wrapped
throw e;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 506a431ac3d..a966ec8d0d9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -683,15 +683,19 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
qbexpr.setOpcode(QBExpr.Opcode.UNION);
break;
case HiveParser.TOK_INTERSECTALL:
+ queryProperties.setHasIntersect(true);
qbexpr.setOpcode(QBExpr.Opcode.INTERSECTALL);
break;
case HiveParser.TOK_INTERSECTDISTINCT:
+ queryProperties.setHasIntersect(true);
qbexpr.setOpcode(QBExpr.Opcode.INTERSECT);
break;
case HiveParser.TOK_EXCEPTALL:
+ queryProperties.setHasExcept(true);
qbexpr.setOpcode(QBExpr.Opcode.EXCEPTALL);
break;
case HiveParser.TOK_EXCEPTDISTINCT:
+ queryProperties.setHasExcept(true);
qbexpr.setOpcode(QBExpr.Opcode.EXCEPT);
break;
default:
@@ -1988,6 +1992,7 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
break;
case HiveParser.TOK_QUALIFY:
+ queryProperties.setHasQualify(true);
qbp.setQualifyExprForClause(ctx_1.dest, ast);
qbp.addAggregationExprsForClause(ctx_1.dest,
doPhase1GetAggregationsFromSelect(ast, qb, ctx_1.dest));
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQueryProperties.java
b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQueryProperties.java
new file mode 100644
index 00000000000..2948f744439
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQueryProperties.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.parse;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.conf.HiveConfForTest;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.Context;
+import org.apache.hadoop.hive.ql.QueryProperties;
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestQueryProperties {
+ private static HiveConf conf;
+ private static CalcitePlanner analyzer;
+
+ @BeforeClass
+ public static void setUpAll() throws Exception {
+ conf = new HiveConfForTest(TestSemanticAnalyzer.class);
+ conf.setVar(ConfVars.HIVE_AUTHENTICATOR_MANAGER,
+ "org.apache.hadoop.hive.ql.security.HadoopDefaultAuthenticator");
+ conf.setVar(ConfVars.HIVE_AUTHORIZATION_MANAGER,
+
"org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
+
+ SessionState ss = new SessionState(conf);
+ SessionState.setCurrentSessionState(ss);
+ SessionState.start(ss);
+
+ Table testTable = new Table("test_db", "test_table");
+ testTable.getSd().setLocation("dummy");
+ List<FieldSchema> columns = testTable.getCols();
+ columns.add(new FieldSchema("t1", "string", ""));
+ columns.add(new FieldSchema("t2", "string", ""));
+ Map<String, String> parameters = testTable.getParameters();
+ parameters.put(StatsSetupConst.ROW_COUNT, "10");
+ parameters.put(StatsSetupConst.RAW_DATA_SIZE, "10");
+ parameters.put(StatsSetupConst.TOTAL_SIZE, "10");
+ ss.getTempTables().put("test_db", Collections.singletonMap("test_table",
testTable));
+
+ QueryState state = new QueryState.Builder().build();
+ analyzer = new CalcitePlanner(state);
+ }
+
+ private QueryProperties analyze(String sql) throws Exception {
+ Context context = new Context(conf);
+ ASTNode node = ParseUtils.parse(sql, context);
+ analyzer.initCtx(context);
+ analyzer.init(false);
+ // CalcitePlanner#genOPTree is invoked in analyzeInternal, and it cleans
up QueryProperties
+ // That's why we call only genResolvedParseTree here
+ analyzer.genResolvedParseTree(node, null);
+ return analyzer.getQueryProperties();
+ }
+
+ @Test
+ public void testExceptAll() throws Exception {
+ QueryProperties properties = analyze(
+ "SELECT t1 FROM test_db.test_table EXCEPT ALL SELECT t2 FROM
test_db.test_table");
+ Assert.assertTrue(properties.hasExcept());
+ }
+
+ @Test
+ public void testExceptDistinct() throws Exception {
+ QueryProperties properties = analyze(
+ "SELECT t1 FROM test_db.test_table EXCEPT DISTINCT SELECT t2 FROM
test_db.test_table");
+ Assert.assertTrue(properties.hasExcept());
+ }
+
+ @Test
+ public void testIntersectAll() throws Exception {
+ QueryProperties properties = analyze(
+ "SELECT t1 FROM test_db.test_table INTERSECT ALL SELECT t2 FROM
test_db.test_table");
+ Assert.assertTrue(properties.hasIntersect());
+ }
+
+ @Test
+ public void testIntersectDistinct() throws Exception {
+ QueryProperties properties = analyze(
+ "SELECT t1 FROM test_db.test_table INTERSECT DISTINCT SELECT t2 FROM
test_db.test_table");
+ Assert.assertTrue(properties.hasIntersect());
+ }
+
+ @Test
+ public void testQualify() throws Exception {
+ QueryProperties properties = analyze(
+ "SELECT t1 FROM test_db.test_table QUALIFY row_number() OVER
(PARTITION BY t1 ORDER BY t2) = 1");
+ Assert.assertTrue(properties.hasQualify());
+ }
+}
diff --git a/ql/src/test/queries/clientnegative/cbo_fallback_skip_with_except.q
b/ql/src/test/queries/clientnegative/cbo_fallback_skip_with_except.q
new file mode 100644
index 00000000000..8592c3f29b2
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/cbo_fallback_skip_with_except.q
@@ -0,0 +1,2 @@
+set hive.cbo.fallback.strategy=CONSERVATIVE;
+select cast(0 as bigint) = '1' except select cast(1 as bigint) = '1';
diff --git
a/ql/src/test/results/clientnegative/cbo_fallback_skip_with_except.q.out
b/ql/src/test/results/clientnegative/cbo_fallback_skip_with_except.q.out
new file mode 100644
index 00000000000..01ea35cd212
--- /dev/null
+++ b/ql/src/test/results/clientnegative/cbo_fallback_skip_with_except.q.out
@@ -0,0 +1 @@
+FAILED: SemanticException Line 0:-1 Wrong arguments ''1'': Unsafe compares
between different types are disabled for safety reasons. If you know what you
are doing, please set hive.strict.checks.type.safety to false and make sure
that hive.mapred.mode is not set to 'strict' to proceed. Note that you may get
errors or incorrect results if you make a mistake while using some of the
unsafe features.