This is an automated email from the ASF dual-hosted git repository.

krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 4b089c2a48b HIVE-28264: OOM/slow compilation when query contains 
SELECT clauses with nested expressions (Krisztian Kasa, reviewed by Stamatis 
Zampetakis)
4b089c2a48b is described below

commit 4b089c2a48b8d36e5084ce06945722b0ae3f0836
Author: Krisztian Kasa <[email protected]>
AuthorDate: Wed Sep 11 12:54:16 2024 +0200

    HIVE-28264: OOM/slow compilation when query contains SELECT clauses with 
nested expressions (Krisztian Kasa, reviewed by Stamatis Zampetakis)
---
 .../hadoop/hive/ql/optimizer/calcite/Bug.java      |  5 ++
 .../hive/ql/optimizer/calcite/HiveRelOptUtil.java  | 14 +++++
 .../rules/HiveFilterProjectTSTransposeRule.java    | 17 ++++--
 .../rules/HiveFilterProjectTransposeRule.java      | 63 ++++++++++++++++------
 .../clientpositive/subquery_nested_expressions.q   | 10 ++++
 .../llap/subquery_nested_expressions.q.out         | 36 +++++++++++++
 6 files changed, 123 insertions(+), 22 deletions(-)

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java
index 8d5c207c688..ab79129ae33 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/Bug.java
@@ -93,4 +93,9 @@ public final class Bug {
    * Whether <a 
href="https://issues.apache.org/jira/browse/CALCITE-5669";>CALCITE-5985</a> is 
fixed.
    */
   public static final boolean CALCITE_5985_FIXED = false;
+
+  /**
+   * Whether <a 
href="https://issues.apache.org/jira/browse/CALCITE-6513";>CALCITE-6513</a> is 
fixed.
+   */
+  public static final boolean CALCITE_6513_FIXED = false;
 }
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
index efd0e67b73e..91f2f488d11 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelOptUtil.java
@@ -727,6 +727,20 @@ public class HiveRelOptUtil extends RelOptUtil {
     return cannotExtract;
   }
 
+  public static RexNode pushPastProjectUnlessBloat(RexNode node, Project 
project, int bloat) {
+    if (Bug.CALCITE_6513_FIXED) {
+      throw new IllegalStateException("Method is redundant when the fix for 
CALCITE-6513 is merged into Calcite. " +
+          "Use RelOptUtil.pushPastProjectUnlessBloat");
+    }
+
+    List<RexNode> newConditions =
+        RelOptUtil.pushPastProjectUnlessBloat(Collections.singletonList(node), 
project, bloat);
+    if (newConditions == null || newConditions.size() != 1) {
+      return null;
+    }
+    return newConditions.get(0);
+  }
+
   public static class PKFKJoinInfo {
     public final boolean isPkFkJoin;
     public final Pair<ImmutableBitSet, ImmutableBitSet> pkFkJoinColumns;
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
index a5d1c433759..6eace95a4a2 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTSTransposeRule.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
 import org.apache.calcite.adapter.druid.DruidQuery;
 import org.apache.calcite.plan.RelOptRule;
 import org.apache.calcite.plan.RelOptRuleCall;
-import org.apache.calcite.plan.RelOptUtil;
 import org.apache.calcite.rel.RelNode;
 import org.apache.calcite.rel.core.Filter;
 import org.apache.calcite.rel.core.Project;
@@ -28,12 +27,14 @@ import org.apache.calcite.rel.core.RelFactories;
 import org.apache.calcite.rel.core.RelFactories.FilterFactory;
 import org.apache.calcite.rel.core.RelFactories.ProjectFactory;
 import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.rules.ProjectMergeRule;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rex.RexCall;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexOver;
 import org.apache.calcite.rex.RexUtil;
 import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelFactories;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelOptUtil;
 import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
 
 import java.util.Collections;
@@ -44,22 +45,25 @@ public class HiveFilterProjectTSTransposeRule extends 
RelOptRule {
   public final static HiveFilterProjectTSTransposeRule INSTANCE =
       new HiveFilterProjectTSTransposeRule(
           Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, 
HiveProject.class,
-          HiveRelFactories.HIVE_PROJECT_FACTORY, TableScan.class);
+          HiveRelFactories.HIVE_PROJECT_FACTORY, TableScan.class, 
ProjectMergeRule.DEFAULT_BLOAT);
 
   public final static  HiveFilterProjectTSTransposeRule INSTANCE_DRUID =
       new HiveFilterProjectTSTransposeRule(
           Filter.class, HiveRelFactories.HIVE_FILTER_FACTORY, 
HiveProject.class,
-          HiveRelFactories.HIVE_PROJECT_FACTORY, DruidQuery.class);
+          HiveRelFactories.HIVE_PROJECT_FACTORY, DruidQuery.class, 
ProjectMergeRule.DEFAULT_BLOAT);
 
   private final RelFactories.FilterFactory  filterFactory;
   private final RelFactories.ProjectFactory projectFactory;
+  private final int bloat;
 
   private HiveFilterProjectTSTransposeRule(Class<? extends Filter> filterClass,
       FilterFactory filterFactory, Class<? extends Project> projectClass,
-      ProjectFactory projectFactory, Class<? extends RelNode> tsClass) {
+      ProjectFactory projectFactory, Class<? extends RelNode> tsClass,
+      int bloat) {
     super(operand(filterClass, operand(projectClass, operand(tsClass, 
none()))));
     this.filterFactory = filterFactory;
     this.projectFactory = projectFactory;
+    this.bloat = bloat;
   }
 
   @Override
@@ -104,7 +108,10 @@ public class HiveFilterProjectTSTransposeRule extends 
RelOptRule {
     }
 
     // convert the filter to one that references the child of the project
-    RexNode newCondition = RelOptUtil.pushPastProject(filter.getCondition(), 
project);
+    RexNode newCondition = 
HiveRelOptUtil.pushPastProjectUnlessBloat(filter.getCondition(), project, 
bloat);
+    if (newCondition == null) {
+      return;
+    }
 
     // Remove cast of BOOLEAN NOT NULL to BOOLEAN or vice versa. Filter accepts
     // nullable and not-nullable conditions, but a CAST might get in the way of
diff --git 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
index 62f0067fd1b..819231cb54d 100644
--- 
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
+++ 
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java
@@ -36,6 +36,7 @@ import org.apache.calcite.rel.core.Filter;
 import org.apache.calcite.rel.core.Join;
 import org.apache.calcite.rel.core.Project;
 import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
+import org.apache.calcite.rel.rules.ProjectMergeRule;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexCall;
@@ -66,43 +67,47 @@ public class HiveFilterProjectTransposeRule extends 
FilterProjectTransposeRule {
   public static final HiveFilterProjectTransposeRule 
DETERMINISTIC_WINDOWING_ON_NON_FILTERING_JOIN =
       new HiveFilterProjectTransposeRule(
           operand(Filter.class, operand(Project.class, operand(Join.class, 
any()))),
-          HiveRelFactories.HIVE_BUILDER, true, true);
+          HiveRelFactories.HIVE_BUILDER, true, true, 
ProjectMergeRule.DEFAULT_BLOAT);
 
   public static final HiveFilterProjectTransposeRule DETERMINISTIC_WINDOWING =
           new HiveFilterProjectTransposeRule(Filter.class, HiveProject.class,
-                  HiveRelFactories.HIVE_BUILDER, true, true);
+                  HiveRelFactories.HIVE_BUILDER, true, true, 
ProjectMergeRule.DEFAULT_BLOAT);
 
   public static final HiveFilterProjectTransposeRule 
DETERMINISTIC_ON_NON_FILTERING_JOIN =
       new HiveFilterProjectTransposeRule(
           operand(Filter.class, operand(Project.class, operand(Join.class, 
any()))),
-          HiveRelFactories.HIVE_BUILDER, true, false);
+          HiveRelFactories.HIVE_BUILDER, true, false, 
ProjectMergeRule.DEFAULT_BLOAT);
 
   public static final HiveFilterProjectTransposeRule DETERMINISTIC =
           new HiveFilterProjectTransposeRule(Filter.class, HiveProject.class,
-                  HiveRelFactories.HIVE_BUILDER, true, false);
+                  HiveRelFactories.HIVE_BUILDER, true, false, 
ProjectMergeRule.DEFAULT_BLOAT);
 
   public static final HiveFilterProjectTransposeRule INSTANCE =
           new HiveFilterProjectTransposeRule(Filter.class, HiveProject.class,
-                  HiveRelFactories.HIVE_BUILDER, false, false);
+                  HiveRelFactories.HIVE_BUILDER, false, false, 
ProjectMergeRule.DEFAULT_BLOAT);
 
   private final boolean onlyDeterministic;
 
   private final boolean pushThroughWindowing;
 
+  private final int bloat;
+
   private HiveFilterProjectTransposeRule(Class<? extends Filter> filterClass,
       Class<? extends Project> projectClass, RelBuilderFactory 
relBuilderFactory,
-      boolean onlyDeterministic,boolean pushThroughWindowing) {
+      boolean onlyDeterministic,boolean pushThroughWindowing, int bloat) {
     super(filterClass, projectClass, false, false, relBuilderFactory);
     this.onlyDeterministic = onlyDeterministic;
     this.pushThroughWindowing = pushThroughWindowing;
+    this.bloat = bloat;
   }
 
   private HiveFilterProjectTransposeRule(RelOptRuleOperand operand,
       RelBuilderFactory relBuilderFactory,
-      boolean onlyDeterministic, boolean pushThroughWindowing) {
+      boolean onlyDeterministic, boolean pushThroughWindowing, int bloat) {
     super(operand, false, false, relBuilderFactory);
     this.onlyDeterministic = onlyDeterministic;
     this.pushThroughWindowing = pushThroughWindowing;
+    this.bloat = bloat;
   }
 
   @Override
@@ -113,7 +118,11 @@ public class HiveFilterProjectTransposeRule extends 
FilterProjectTransposeRule {
     // as part of the select list when a view is in play.  But the condition 
after the pushdown
     // will resolve to using the udf from select list.  The check here for 
deterministic filters
     // should be based on the resolved expression.  Refer to test case 
cbo_ppd_non_deterministic.q.
-    RexNode condition = RelOptUtil.pushPastProject(filterRel.getCondition(), 
call.rel(1));
+    RexNode condition = HiveRelOptUtil.pushPastProjectUnlessBloat(
+        filterRel.getCondition(), call.rel(1), bloat);
+    if (condition == null) {
+      return false;
+    }
 
     if (this.onlyDeterministic && !HiveCalciteUtil.isDeterministic(condition)) 
{
       return false;
@@ -131,6 +140,7 @@ public class HiveFilterProjectTransposeRule extends 
FilterProjectTransposeRule {
     return super.matches(call);
   }
 
+  @Override
   public void onMatch(RelOptRuleCall call) {
     final Filter filter = call.rel(0);
     final Project origproject = call.rel(1);
@@ -166,8 +176,8 @@ public class HiveFilterProjectTransposeRule extends 
FilterProjectTransposeRule {
         // from t1 where value < 10)t1)t2
         if (!commonPartitionKeys.isEmpty()) {
           for (RexNode ce : RelOptUtil.conjunctions(origFilterCond)) {
-            RexNode newCondition = RelOptUtil.pushPastProject(ce, origproject);
-            if 
(HiveCalciteUtil.isDeterministicFuncWithSingleInputRef(newCondition,
+            RexNode newCondition = 
HiveRelOptUtil.pushPastProjectUnlessBloat(ce, origproject, bloat);
+            if (newCondition != null && 
HiveCalciteUtil.isDeterministicFuncWithSingleInputRef(newCondition,
                 commonPartitionKeys)) {
               newPartKeyFilConds.add(ce);
             } else {
@@ -193,15 +203,21 @@ public class HiveFilterProjectTransposeRule extends 
FilterProjectTransposeRule {
       RelNode newProjRel = getNewProject(filterCondToPushBelowProj, 
unPushedFilCondAboveProj, origproject, filter.getCluster()
           .getTypeFactory(), call.builder());
 
-      call.transformTo(newProjRel);
+      if (newProjRel != null) {
+        call.transformTo(newProjRel);
+      }
     }
   }
 
-  private static RelNode getNewProject(RexNode filterCondToPushBelowProj, 
RexNode unPushedFilCondAboveProj, Project oldProj,
+  private RelNode getNewProject(RexNode filterCondToPushBelowProj, RexNode 
unPushedFilCondAboveProj, Project oldProj,
       RelDataTypeFactory typeFactory, RelBuilder relBuilder) {
 
     // convert the filter to one that references the child of the project.
-    RexNode newPushedCondition = 
RelOptUtil.pushPastProject(filterCondToPushBelowProj, oldProj);
+    RexNode newPushedCondition =
+        HiveRelOptUtil.pushPastProjectUnlessBloat(filterCondToPushBelowProj, 
oldProj, bloat);
+    if (newPushedCondition == null) {
+      return null;
+    }
 
     // Remove cast of BOOLEAN NOT NULL to BOOLEAN or vice versa. Filter accepts
     // nullable and not-nullable conditions, but a CAST might get in the way of
@@ -262,7 +278,7 @@ public class HiveFilterProjectTransposeRule extends 
FilterProjectTransposeRule {
   // in cases when a filter using IS NOT NULL is applied to an input $i down 
the subtree,
   // creating another filter IS NOT NULL(FUNC_CALL+($i)) is of a doubtful 
usefulness and
   // might lead to infinite loops in predicate pull-up and push-down, like in 
HIVE-25275
-  private static boolean isRedundantIsNotNull(Project project, RexNode 
newCondition) {
+  private boolean isRedundantIsNotNull(Project project, RexNode newCondition) {
     if (!newCondition.isA(SqlKind.IS_NOT_NULL)) {
       return false;
     }
@@ -272,7 +288,7 @@ public class HiveFilterProjectTransposeRule extends 
FilterProjectTransposeRule {
       return false;
     }
 
-    RedundancyChecker redundancyChecker = new RedundancyChecker(newCondition);
+    RedundancyChecker redundancyChecker = new RedundancyChecker(newCondition, 
bloat);
     redundancyChecker.go(project);
     return redundancyChecker.isRedundant;
   }
@@ -282,9 +298,11 @@ public class HiveFilterProjectTransposeRule extends 
FilterProjectTransposeRule {
 
     final RexNode newCondition;
     final Map<RelNode, RexNode> filter2newConditionMap = new HashMap<>();
+    private final int bloat;
 
-    protected RedundancyChecker(RexNode newCondition) {
+    protected RedundancyChecker(RexNode newCondition, int bloat) {
       this.newCondition = newCondition;
+      this.bloat = bloat;
     }
 
     @Override
@@ -305,7 +323,11 @@ public class HiveFilterProjectTransposeRule extends 
FilterProjectTransposeRule {
         if (node instanceof Filter) {
           check((Filter) node);
         } else if (node instanceof Project) {
-          filterCondition = RelOptUtil.pushPastProject(filterCondition, 
(Project) node);
+          RexNode condition = HiveRelOptUtil.pushPastProjectUnlessBloat(
+              filterCondition, (Project) node, bloat);
+          if (condition != null) {
+            filterCondition = condition;
+          }
         } else {
           // we do not support other operators for now
           return;
@@ -371,18 +393,22 @@ public class HiveFilterProjectTransposeRule extends 
FilterProjectTransposeRule {
       this.includedPredDigest = includedPred.toString();
     }
 
+    @Override
     public Boolean visitInputRef(RexInputRef inputRef) {
       return false;
     }
 
+    @Override
     public Boolean visitLiteral(RexLiteral literal) {
       return false;
     }
 
+    @Override
     public Boolean visitCorrelVariable(RexCorrelVariable correlVariable) {
       return false;
     }
 
+    @Override
     public Boolean visitCall(RexCall call) {
       if (call.isA(SqlKind.AND)) {
         return call.getOperands().stream().anyMatch(o -> o.accept(this));
@@ -392,14 +418,17 @@ public class HiveFilterProjectTransposeRule extends 
FilterProjectTransposeRule {
       return includedPredDigest.equals(call.toString());
     }
 
+    @Override
     public Boolean visitDynamicParam(RexDynamicParam dynamicParam) {
       return false;
     }
 
+    @Override
     public Boolean visitRangeRef(RexRangeRef rangeRef) {
       return false;
     }
 
+    @Override
     public Boolean visitFieldAccess(RexFieldAccess fieldAccess) {
       return false;
     }
diff --git a/ql/src/test/queries/clientpositive/subquery_nested_expressions.q 
b/ql/src/test/queries/clientpositive/subquery_nested_expressions.q
new file mode 100644
index 00000000000..b47ebd56767
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/subquery_nested_expressions.q
@@ -0,0 +1,10 @@
+CREATE TABLE t0 (`title` string);
+
+explain cbo
+SELECT x4 from
+    (SELECT concat_ws('L4',x3, x3, x3, x3) as x4 from
+        (SELECT concat_ws('L3',x2, x2, x2, x2) as x3 from
+            (SELECT concat_ws('L2',x1, x1, x1, x1) as x2 from
+                (SELECT concat_ws('L1',x0, x0, x0, x0) as x1 from
+                    (SELECT concat_ws('L0',title, title, title, title) as x0 
from t0) t1) t2) t3) t4) t
+WHERE x4 = 'Something';
diff --git 
a/ql/src/test/results/clientpositive/llap/subquery_nested_expressions.q.out 
b/ql/src/test/results/clientpositive/llap/subquery_nested_expressions.q.out
new file mode 100644
index 00000000000..b60245997a9
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/subquery_nested_expressions.q.out
@@ -0,0 +1,36 @@
+PREHOOK: query: CREATE TABLE t0 (`title` string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t0
+POSTHOOK: query: CREATE TABLE t0 (`title` string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t0
+PREHOOK: query: explain cbo
+SELECT x4 from
+    (SELECT concat_ws('L4',x3, x3, x3, x3) as x4 from
+        (SELECT concat_ws('L3',x2, x2, x2, x2) as x3 from
+            (SELECT concat_ws('L2',x1, x1, x1, x1) as x2 from
+                (SELECT concat_ws('L1',x0, x0, x0, x0) as x1 from
+                    (SELECT concat_ws('L0',title, title, title, title) as x0 
from t0) t1) t2) t3) t4) t
+WHERE x4 = 'Something'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t0
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo
+SELECT x4 from
+    (SELECT concat_ws('L4',x3, x3, x3, x3) as x4 from
+        (SELECT concat_ws('L3',x2, x2, x2, x2) as x3 from
+            (SELECT concat_ws('L2',x1, x1, x1, x1) as x2 from
+                (SELECT concat_ws('L1',x0, x0, x0, x0) as x1 from
+                    (SELECT concat_ws('L0',title, title, title, title) as x0 
from t0) t1) t2) t3) t4) t
+WHERE x4 = 'Something'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t0
+#### A masked pattern was here ####
+CBO PLAN:
+HiveProject(x4=[CAST(_UTF-16LE'Something':VARCHAR(2147483647) CHARACTER SET 
"UTF-16LE"):VARCHAR(2147483647) CHARACTER SET "UTF-16LE"])
+  HiveFilter(condition=[=(concat_ws(_UTF-16LE'L4':VARCHAR(2147483647) 
CHARACTER SET "UTF-16LE", concat_ws(_UTF-16LE'L3':VARCHAR(2147483647) CHARACTER 
SET "UTF-16LE", concat_ws(_UTF-16LE'L2':VARCHAR(2147483647) CHARACTER SET 
"UTF-16LE", $0, $0, $0, $0), concat_ws(_UTF-16LE'L2':VARCHAR(2147483647) 
CHARACTER SET "UTF-16LE", $0, $0, $0, $0), 
concat_ws(_UTF-16LE'L2':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", $0, $0, 
$0, $0), concat_ws(_UTF-16LE'L2':VARCHAR(2147483647) CHARACTER SET "UTF-16 [...]
+    HiveProject(x1=[concat_ws(_UTF-16LE'L1':VARCHAR(2147483647) CHARACTER SET 
"UTF-16LE", concat_ws(_UTF-16LE'L0':VARCHAR(2147483647) CHARACTER SET 
"UTF-16LE", $0, $0, $0, $0), concat_ws(_UTF-16LE'L0':VARCHAR(2147483647) 
CHARACTER SET "UTF-16LE", $0, $0, $0, $0), 
concat_ws(_UTF-16LE'L0':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", $0, $0, 
$0, $0), concat_ws(_UTF-16LE'L0':VARCHAR(2147483647) CHARACTER SET "UTF-16LE", 
$0, $0, $0, $0))])
+      HiveTableScan(table=[[default, t0]], table:alias=[t0])
+

Reply via email to