This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4b089c2a48b HIVE-28264: OOM/slow compilation when query contains
SELECT clauses with nested expressions (Krisztian Kasa, reviewed by Stamatis
Zampetakis)
4b089c2a48b is described below
commit 4b089c2a48b8d36e5084ce06945722b0ae3f0836
Author: Krisztian Kasa <[email protected]>
AuthorDate: Wed Sep 11 12:54:16 2024 +0200
HIVE-28264: OOM/slow compilation when query contains SELECT clauses with
nested expressions (Krisztian Kasa, reviewed by Stamatis Zampetakis)
---
.../hadoop/hive/ql/optimizer/calcite/Bug.java | 5 ++
.../hive/ql/optimizer/calcite/HiveRelOptUtil.java | 14 +++++
.../rules/HiveFilterProjectTSTransposeRule.java | 17 ++++--
.../rules/HiveFilterProjectTransposeRule.java | 63 ++++++++++++++++------
.../clientpositive/subquery_nested_expressions.q | 10 ++++
.../llap/subquery_nested_expressions.q.out | 36 +++++++++++++
6 files changed, 123 insertions(+), 22 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java
index 8d5c207c688..ab79129ae33 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java
@@ -93,4 +93,9 @@ public final class Bug {
* Whether <a
href="https://issues.apache.org/jira/browse/CALCITE-5669">CALCITE-5985</a> is
fixed.
*/
public static final boolean CALCITE_5985_FIXED = false;
+
+ /**
+ * Whether <a
href="https://issues.apache.org/jira/browse/CALCITE-6513">CALCITE-6513</a> is
fixed.
+ */
+ public static final boolean CALCITE_6513_FIXED = false;
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
index efd0e67b73e..91f2f488d11 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
@@ -727,6 +727,20 @@ public class HiveRelOptUtil extends RelOptUtil {
return cannotExtract;
}
+ public static RexNode pushPastProjectUnlessBloat(RexNode node, Project
project, int bloat) {
+ if (Bug.CALCITE_6513_FIXED) {
+ throw new IllegalStateException("Method is redundant when the fix for
CALCITE-6513 is merged into Calcite. " +
+ "Use RelOptUtil.pushPastProjectUnlessBloat");
+ }
+
+ List<RexNode> newConditions =
+ RelOptUtil.pushPastProjectUnlessBloat(Collections.singletonList(node),
project, bloat);
+ if (newConditions == null || newConditions.size() != 1) {
+ return null;
+ }
+ return newConditions.get(0);
+ }
+
public static class PKFKJoinInfo {
public final boolean isPkFkJoin;
public final Pair<ImmutableBitSet, ImmutableBitSet> pkFkJoinColumns;
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
index a5d1c433759..6eace95a4a2 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
import org.apache.calcite.adapter.druid.DruidQuery;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Project;
@@ -28,12 +27,14 @@ import org.apache.calcite.rel.core.RelFactories;
import org.apache.calcite.rel.core.RelFactories.FilterFactory;
import org.apache.calcite.rel.core.RelFactories.ProjectFactory;
import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.rules.ProjectMergeRule;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexOver;
import org.apache.calcite.rex.RexUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
import java.util.Collections;
@@ -44,22 +45,25 @@ public class HiveFilterProjectTSTransposeRule extends
RelOptRule {
public final static HiveFilterProjectTSTransposeRule INSTANCE =
new HiveFilterProjectTSTransposeRule(
Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY,
HiveProject.class,
- HiveRelFactories.HIVE_PROJECT_FACTORY, TableScan.class);
+ HiveRelFactories.HIVE_PROJECT_FACTORY, TableScan.class,
ProjectMergeRule.DEFAULT_BLOAT);
public final static HiveFilterProjectTSTransposeRule INSTANCE_DRUID =
new HiveFilterProjectTSTransposeRule(
Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY,
HiveProject.class,
- HiveRelFactories.HIVE_PROJECT_FACTORY, DruidQuery.class);
+ HiveRelFactories.HIVE_PROJECT_FACTORY, DruidQuery.class,
ProjectMergeRule.DEFAULT_BLOAT);
private final RelFactories.FilterFactory filterFactory;
private final RelFactories.ProjectFactory projectFactory;
+ private final int bloat;
private HiveFilterProjectTSTransposeRule(Class<? extends Filter> filterClass,
FilterFactory filterFactory, Class<? extends Project> projectClass,
- ProjectFactory projectFactory, Class<? extends RelNode> tsClass) {
+ ProjectFactory projectFactory, Class<? extends RelNode> tsClass,
+ int bloat) {
super(operand(filterClass, operand(projectClass, operand(tsClass,
none()))));
this.filterFactory = filterFactory;
this.projectFactory = projectFactory;
+ this.bloat = bloat;
}
@Override
@@ -104,7 +108,10 @@ public class HiveFilterProjectTSTransposeRule extends
RelOptRule {
}
// convert the filter to one that references the child of the project
- RexNode newCondition = RelOptUtil.pushPastProject(filter.getCondition(),
project);
+ RexNode newCondition =
HiveRelOptUtil.pushPastProjectUnlessBloat(filter.getCondition(), project,
bloat);
+ if (newCondition == null) {
+ return;
+ }
// Remove cast of BOOLEAN NOT NULL to BOOLEAN or vice versa. Filter accepts
// nullable and not-nullable conditions, but a CAST might get in the way of
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
index 62f0067fd1b..819231cb54d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
@@ -36,6 +36,7 @@ import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
+import org.apache.calcite.rel.rules.ProjectMergeRule;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
@@ -66,43 +67,47 @@ public class HiveFilterProjectTransposeRule extends
FilterProjectTransposeRule {
public static final HiveFilterProjectTransposeRule
DETERMINISTIC_WINDOWING_ON_NON_FILTERING_JOIN =
new HiveFilterProjectTransposeRule(
operand(Filter.class, operand(Project.class, operand(Join.class,
any()))),
- HiveRelFactories.HIVE_BUILDER, true, true);
+ HiveRelFactories.HIVE_BUILDER, true, true,
ProjectMergeRule.DEFAULT_BLOAT);
public static final HiveFilterProjectTransposeRule DETERMINISTIC_WINDOWING =
new HiveFilterProjectTransposeRule(Filter.class, HiveProject.class,
- HiveRelFactories.HIVE_BUILDER, true, true);
+ HiveRelFactories.HIVE_BUILDER, true, true,
ProjectMergeRule.DEFAULT_BLOAT);
public static final HiveFilterProjectTransposeRule
DETERMINISTIC_ON_NON_FILTERING_JOIN =
new HiveFilterProjectTransposeRule(
operand(Filter.class, operand(Project.class, operand(Join.class,
any()))),
- HiveRelFactories.HIVE_BUILDER, true, false);
+ HiveRelFactories.HIVE_BUILDER, true, false,
ProjectMergeRule.DEFAULT_BLOAT);
public static final HiveFilterProjectTransposeRule DETERMINISTIC =
new HiveFilterProjectTransposeRule(Filter.class, HiveProject.class,
- HiveRelFactories.HIVE_BUILDER, true, false);
+ HiveRelFactories.HIVE_BUILDER, true, false,
ProjectMergeRule.DEFAULT_BLOAT);
public static final HiveFilterProjectTransposeRule INSTANCE =
new HiveFilterProjectTransposeRule(Filter.class, HiveProject.class,
- HiveRelFactories.HIVE_BUILDER, false, false);
+ HiveRelFactories.HIVE_BUILDER, false, false,
ProjectMergeRule.DEFAULT_BLOAT);
private final boolean onlyDeterministic;
private final boolean pushThroughWindowing;
+ private final int bloat;
+
private HiveFilterProjectTransposeRule(Class<? extends Filter> filterClass,
Class<? extends Project> projectClass, RelBuilderFactory
relBuilderFactory,
- boolean onlyDeterministic,boolean pushThroughWindowing) {
+ boolean onlyDeterministic,boolean pushThroughWindowing, int bloat) {
super(filterClass, projectClass, false, false, relBuilderFactory);
this.onlyDeterministic = onlyDeterministic;
this.pushThroughWindowing = pushThroughWindowing;
+ this.bloat = bloat;
}
private HiveFilterProjectTransposeRule(RelOptRuleOperand operand,
RelBuilderFactory relBuilderFactory,
- boolean onlyDeterministic, boolean pushThroughWindowing) {
+ boolean onlyDeterministic, boolean pushThroughWindowing, int bloat) {
super(operand, false, false, relBuilderFactory);
this.onlyDeterministic = onlyDeterministic;
this.pushThroughWindowing = pushThroughWindowing;
+ this.bloat = bloat;
}
@Override
@@ -113,7 +118,11 @@ public class HiveFilterProjectTransposeRule extends
FilterProjectTransposeRule {
// as part of the select list when a view is in play. But the condition
after the pushdown
// will resolve to using the udf from select list. The check here for
deterministic filters
// should be based on the resolved expression. Refer to test case
cbo_ppd_non_deterministic.q.
- RexNode condition = RelOptUtil.pushPastProject(filterRel.getCondition(),
call.rel(1));
+ RexNode condition = HiveRelOptUtil.pushPastProjectUnlessBloat(
+ filterRel.getCondition(), call.rel(1), bloat);
+ if (condition == null) {
+ return false;
+ }
if (this.onlyDeterministic && !HiveCalciteUtil.isDeterministic(condition))
{
return false;
@@ -131,6 +140,7 @@ public class HiveFilterProjectTransposeRule extends
FilterProjectTransposeRule {
return super.matches(call);
}
+ @Override
public void onMatch(RelOptRuleCall call) {
final Filter filter = call.rel(0);
final Project origproject = call.rel(1);
@@ -166,8 +176,8 @@ public class HiveFilterProjectTransposeRule extends
FilterProjectTransposeRule {
// from t1 where value < 10)t1)t2
if (!commonPartitionKeys.isEmpty()) {
for (RexNode ce : RelOptUtil.conjunctions(origFilterCond)) {
- RexNode newCondition = RelOptUtil.pushPastProject(ce, origproject);
- if
(HiveCalciteUtil.isDeterministicFuncWithSingleInputRef(newCondition,
+ RexNode newCondition =
HiveRelOptUtil.pushPastProjectUnlessBloat(ce, origproject, bloat);
+ if (newCondition != null &&
HiveCalciteUtil.isDeterministicFuncWithSingleInputRef(newCondition,
commonPartitionKeys)) {
newPartKeyFilConds.add(ce);
} else {
@@ -193,15 +203,21 @@ public class HiveFilterProjectTransposeRule extends
FilterProjectTransposeRule {
RelNode newProjRel = getNewProject(filterCondToPushBelowProj,
unPushedFilCondAboveProj, origproject, filter.getCluster()
.getTypeFactory(), call.builder());
- call.transformTo(newProjRel);
+ if (newProjRel != null) {
+ call.transformTo(newProjRel);
+ }
}
}
- private static RelNode getNewProject(RexNode filterCondToPushBelowProj,
RexNode unPushedFilCondAboveProj, Project oldProj,
+ private RelNode getNewProject(RexNode filterCondToPushBelowProj, RexNode
unPushedFilCondAboveProj, Project oldProj,
RelDataTypeFactory typeFactory, RelBuilder relBuilder) {
// convert the filter to one that references the child of the project.
- RexNode newPushedCondition =
RelOptUtil.pushPastProject(filterCondToPushBelowProj, oldProj);
+ RexNode newPushedCondition =
+ HiveRelOptUtil.pushPastProjectUnlessBloat(filterCondToPushBelowProj,
oldProj, bloat);
+ if (newPushedCondition == null) {
+ return null;
+ }
// Remove cast of BOOLEAN NOT NULL to BOOLEAN or vice versa. Filter accepts
// nullable and not-nullable conditions, but a CAST might get in the way of
@@ -262,7 +278,7 @@ public class HiveFilterProjectTransposeRule extends
FilterProjectTransposeRule {
// in cases when a filter using IS NOT NULL is applied to an input $i down
the subtree,
// creating another filter IS NOT NULL(FUNC_CALL+($i)) is of a doubtful
usefulness and
// might lead to infinite loops in predicate pull-up and push-down, like in
HIVE-25275
- private static boolean isRedundantIsNotNull(Project project, RexNode
newCondition) {
+ private boolean isRedundantIsNotNull(Project project, RexNode newCondition) {
if (!newCondition.isA(SqlKind.IS_NOT_NULL)) {
return false;
}
@@ -272,7 +288,7 @@ public class HiveFilterProjectTransposeRule extends
FilterProjectTransposeRule {
return false;
}
- RedundancyChecker redundancyChecker = new RedundancyChecker(newCondition);
+ RedundancyChecker redundancyChecker = new RedundancyChecker(newCondition,
bloat);
redundancyChecker.go(project);
return redundancyChecker.isRedundant;
}
@@ -282,9 +298,11 @@ public class HiveFilterProjectTransposeRule extends
FilterProjectTransposeRule {
final RexNode newCondition;
final Map<RelNode, RexNode> filter2newConditionMap = new HashMap<>();
+ private final int bloat;
- protected RedundancyChecker(RexNode newCondition) {
+ protected RedundancyChecker(RexNode newCondition, int bloat) {
this.newCondition = newCondition;
+ this.bloat = bloat;
}
@Override
@@ -305,7 +323,11 @@ public class HiveFilterProjectTransposeRule extends
FilterProjectTransposeRule {
if (node instanceof Filter) {
check((Filter) node);
} else if (node instanceof Project) {
- filterCondition = RelOptUtil.pushPastProject(filterCondition,
(Project) node);
+ RexNode condition = HiveRelOptUtil.pushPastProjectUnlessBloat(
+ filterCondition, (Project) node, bloat);
+ if (condition != null) {
+ filterCondition = condition;
+ }
} else {
// we do not support other operators for now
return;
@@ -371,18 +393,22 @@ public class HiveFilterProjectTransposeRule extends
FilterProjectTransposeRule {
this.includedPredDigest = includedPred.toString();
}
+ @Override
public Boolean visitInputRef(RexInputRef inputRef) {
return false;
}
+ @Override
public Boolean visitLiteral(RexLiteral literal) {
return false;
}
+ @Override
public Boolean visitCorrelVariable(RexCorrelVariable correlVariable) {
return false;
}
+ @Override
public Boolean visitCall(RexCall call) {
if (call.isA(SqlKind.AND)) {
return call.getOperands().stream().anyMatch(o -> o.accept(this));
@@ -392,14 +418,17 @@ public class HiveFilterProjectTransposeRule extends
FilterProjectTransposeRule {
return includedPredDigest.equals(call.toString());
}
+ @Override
public Boolean visitDynamicParam(RexDynamicParam dynamicParam) {
return false;
}
+ @Override
public Boolean visitRangeRef(RexRangeRef rangeRef) {
return false;
}
+ @Override
public Boolean visitFieldAccess(RexFieldAccess fieldAccess) {
return false;
}
diff --git a/ql/src/test/queries/clientpositive/subquery_nested_expressions.q
b/ql/src/test/queries/clientpositive/subquery_nested_expressions.q
new file mode 100644
index 00000000000..b47ebd56767
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/subquery_nested_expressions.q
@@ -0,0 +1,10 @@
+CREATE TABLE t0 (`title` string);
+
+explain cbo
+SELECT x4 from
+ (SELECT concat_ws('L4',x3, x3, x3, x3) as x4 from
+ (SELECT concat_ws('L3',x2, x2, x2, x2) as x3 from
+ (SELECT concat_ws('L2',x1, x1, x1, x1) as x2 from
+ (SELECT concat_ws('L1',x0, x0, x0, x0) as x1 from
+ (SELECT concat_ws('L0',title, title, title, title) as x0
from t0) t1) t2) t3) t4) t
+WHERE x4 = 'Something';
diff --git
a/ql/src/test/results/clientpositive/llap/subquery_nested_expressions.q.out
b/ql/src/test/results/clientpositive/llap/subquery_nested_expressions.q.out
new file mode 100644
index 00000000000..b60245997a9
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/subquery_nested_expressions.q.out
@@ -0,0 +1,36 @@
+PREHOOK: query: CREATE TABLE t0 (`title` string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t0
+POSTHOOK: query: CREATE TABLE t0 (`title` string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t0
+PREHOOK: query: explain cbo
+SELECT x4 from
+ (SELECT concat_ws('L4',x3, x3, x3, x3) as x4 from
+ (SELECT concat_ws('L3',x2, x2, x2, x2) as x3 from
+ (SELECT concat_ws('L2',x1, x1, x1, x1) as x2 from
+ (SELECT concat_ws('L1',x0, x0, x0, x0) as x1 from
+ (SELECT concat_ws('L0',title, title, title, title) as x0
from t0) t1) t2) t3) t4) t
+WHERE x4 = 'Something'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t0
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo
+SELECT x4 from
+ (SELECT concat_ws('L4',x3, x3, x3, x3) as x4 from
+ (SELECT concat_ws('L3',x2, x2, x2, x2) as x3 from
+ (SELECT concat_ws('L2',x1, x1, x1, x1) as x2 from
+ (SELECT concat_ws('L1',x0, x0, x0, x0) as x1 from
+ (SELECT concat_ws('L0',title, title, title, title) as x0
from t0) t1) t2) t3) t4) t
+WHERE x4 = 'Something'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t0
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(x4=[CAST(_UTF-16LE'Something':VARCHAR(2147483647) CHARACTER SET
"UTF-16LE"):VARCHAR(2147483647) CHARACTER SET "UTF-16LE"])
+ HiveFilter(condition=[=(concat_ws(_UTF-16LE'L4':VARCHAR(2147483647)
CHARACTER SET "UTF-16LE", concat_ws(_UTF-16LE'L3':VARCHAR(2147483647) CHARACTER
SET "UTF-16LE", concat_ws(_UTF-16LE'L2':VARCHAR(2147483647) CHARACTER SET
"UTF-16LE", $0, $0, $0, $0), concat_ws(_UTF-16LE'L2':VARCHAR(2147483647)
CHARACTER SET "UTF-16LE", $0, $0, $0, $0),
concat_ws(_UTF-16LE'L2':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", $0, $0,
$0, $0), concat_ws(_UTF-16LE'L2':VARCHAR(2147483647) CHARACTER SET "UTF-16 [...]
+ HiveProject(x1=[concat_ws(_UTF-16LE'L1':VARCHAR(2147483647) CHARACTER SET
"UTF-16LE", concat_ws(_UTF-16LE'L0':VARCHAR(2147483647) CHARACTER SET
"UTF-16LE", $0, $0, $0, $0), concat_ws(_UTF-16LE'L0':VARCHAR(2147483647)
CHARACTER SET "UTF-16LE", $0, $0, $0, $0),
concat_ws(_UTF-16LE'L0':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", $0, $0,
$0, $0), concat_ws(_UTF-16LE'L0':VARCHAR(2147483647) CHARACTER SET "UTF-16LE",
$0, $0, $0, $0))])
+ HiveTableScan(table=[[default, t0]], table:alias=[t0])
+