Add support for non-equijoins in the presence of other equijoins between the same tables.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/63287658 Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/63287658 Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/63287658 Branch: refs/heads/master Commit: 63287658c0628ecfbc56d0d33618ad2219fef438 Parents: 4cfdb3b Author: Aman Sinha <[email protected]> Authored: Fri Apr 4 10:08:24 2014 -0700 Committer: Jacques Nadeau <[email protected]> Committed: Sat Apr 19 18:07:12 2014 -0700 ---------------------------------------------------------------------- .../exec/planner/common/DrillJoinRelBase.java | 4 +- .../exec/planner/logical/DrillJoinRel.java | 27 ++++++++- .../exec/planner/logical/DrillJoinRule.java | 63 +++++++++++++++++++- .../exec/planner/logical/DrillRuleSets.java | 4 +- .../exec/planner/physical/MergeJoinPrel.java | 2 +- 5 files changed, 91 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63287658/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java index e88faf9..b9c112d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java @@ -45,8 +45,8 @@ import com.google.common.collect.Lists; * Base class for logical and physical Joins implemented in Drill. */ public abstract class DrillJoinRelBase extends JoinRelBase implements DrillRelNode { - protected final List<Integer> leftKeys = new ArrayList<>(); - protected final List<Integer> rightKeys = new ArrayList<>(); + protected List<Integer> leftKeys = Lists.newArrayList(); + protected List<Integer> rightKeys = Lists.newArrayList() ; public DrillJoinRelBase(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, RexNode condition, JoinRelType joinType) throws InvalidRelException { http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63287658/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java index 31a8e6e..a3a4ac9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRel.java @@ -42,20 +42,41 @@ import org.eigenbase.rex.RexUtil; import org.eigenbase.sql.fun.SqlStdOperatorTable; import org.eigenbase.util.Pair; +import com.google.hive12.common.collect.Lists; + /** * Join implemented in Drill. */ public class DrillJoinRel extends DrillJoinRelBase implements DrillRel { + /** Creates a DrillJoinRel. */ public DrillJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, RexNode condition, JoinRelType joinType) throws InvalidRelException { super(cluster, traits, left, right, condition, joinType); RexNode remaining = RelOptUtil.splitJoinCondition(left, right, condition, leftKeys, rightKeys); - if (!remaining.isAlwaysTrue()) { + if (!remaining.isAlwaysTrue() && (leftKeys.size() == 0 || rightKeys.size() == 0)) { throw new InvalidRelException("DrillJoinRel only supports equi-join"); } } + + public DrillJoinRel(RelOptCluster cluster, RelTraitSet traits, RelNode left, RelNode right, RexNode condition, + JoinRelType joinType, List<Integer> leftKeys, List<Integer> rightKeys, boolean checkCartesian) throws InvalidRelException { + super(cluster, traits, left, right, condition, joinType); + + assert (leftKeys != null && rightKeys != null); + + if (checkCartesian) { + List<Integer> tmpLeftKeys = Lists.newArrayList(); + List<Integer> tmpRightKeys = Lists.newArrayList(); + RexNode remaining = RelOptUtil.splitJoinCondition(left, right, condition, tmpLeftKeys, tmpRightKeys); + if (!remaining.isAlwaysTrue() && (tmpLeftKeys.size() == 0 || tmpRightKeys.size() == 0)) { + throw new InvalidRelException("DrillJoinRel only supports equi-join"); + } + } + this.leftKeys = leftKeys; + this.rightKeys = rightKeys; + } @Override public DrillJoinRel copy(RelTraitSet traitSet, RexNode condition, RelNode left, RelNode right, JoinRelType joinType) { @@ -141,7 +162,9 @@ public class DrillJoinRel extends DrillJoinRelBase implements DrillRel { ); } RexNode rexCondition = RexUtil.composeConjunction(context.getRexBuilder(), joinConditions, false); - return new DrillJoinRel(context.getCluster(), context.getLogicalTraits(), left, right, rexCondition, join.getJoinType()); + DrillJoinRel joinRel = new DrillJoinRel(context.getCluster(), context.getLogicalTraits(), left, right, rexCondition, join.getJoinType()); + + return joinRel; } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63287658/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java index 2a895a5..60c4ae0 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillJoinRule.java @@ -21,8 +21,17 @@ import org.eigenbase.rel.InvalidRelException; import org.eigenbase.rel.JoinRel; import org.eigenbase.rel.RelNode; import org.eigenbase.relopt.*; +import org.eigenbase.reltype.RelDataTypeField; +import org.eigenbase.rex.RexBuilder; +import org.eigenbase.rex.RexNode; +import org.eigenbase.rex.RexUtil; +import org.eigenbase.sql.fun.SqlStdOperatorTable; import org.eigenbase.trace.EigenbaseTrace; +import com.beust.jcommander.internal.Lists; + +import java.util.ArrayList; +import java.util.List; import java.util.logging.Logger; /** @@ -47,9 +56,59 @@ public class DrillJoinRule extends RelOptRule { final RelNode convertedLeft = convert(left, traits); final RelNode convertedRight = convert(right, traits); + + List<Integer> leftKeys = Lists.newArrayList(); + List<Integer> rightKeys = Lists.newArrayList(); + int numLeftFields = convertedLeft.getRowType().getFieldCount(); + + boolean addFilter = false; + RexNode origJoinCondition = join.getCondition(); + RexNode newJoinCondition = origJoinCondition; + + RexNode remaining = RelOptUtil.splitJoinCondition(convertedLeft, convertedRight, origJoinCondition, leftKeys, rightKeys); + boolean hasEquijoins = (leftKeys.size() == rightKeys.size() && leftKeys.size() > 0) ? true : false; + + // If the join involves equijoins and non-equijoins, then we can process the non-equijoins through + // a filter right after the join + if (! remaining.isAlwaysTrue()) { + if (hasEquijoins) { + addFilter = true; + List<RexNode> equijoinList = Lists.newArrayList(); + List<RelDataTypeField> leftTypes = convertedLeft.getRowType().getFieldList(); + List<RelDataTypeField> rightTypes = convertedRight.getRowType().getFieldList(); + RexBuilder builder = join.getCluster().getRexBuilder(); + + for (int i=0; i < leftKeys.size(); i++) { + int leftKeyOrdinal = leftKeys.get(i).intValue(); + int rightKeyOrdinal = rightKeys.get(i).intValue(); + + equijoinList.add(builder.makeCall( + SqlStdOperatorTable.EQUALS, + builder.makeInputRef(leftTypes.get(leftKeyOrdinal).getType(), leftKeyOrdinal), + builder.makeInputRef(rightTypes.get(rightKeyOrdinal).getType(), rightKeyOrdinal + numLeftFields) + ) ); + } + newJoinCondition = RexUtil.composeConjunction(builder, equijoinList, false); + } else { + tracer.warning("Non-equijoins are only supported in the presence of an equijoin."); + return; + } + } + //else { + // + // return; + // } + try { - call.transformTo(new DrillJoinRel(join.getCluster(), traits, convertedLeft, convertedRight, join.getCondition(), - join.getJoinType())); + if (!addFilter) { + RelNode joinRel = new DrillJoinRel(join.getCluster(), traits, convertedLeft, convertedRight, origJoinCondition, + join.getJoinType(), leftKeys, rightKeys, false); + call.transformTo(joinRel); + } else { + RelNode joinRel = new DrillJoinRel(join.getCluster(), traits, convertedLeft, convertedRight, newJoinCondition, + join.getJoinType(), leftKeys, rightKeys, false); + call.transformTo(new DrillFilterRel(join.getCluster(), traits, joinRel, remaining)); + } } catch (InvalidRelException e) { tracer.warning(e.toString()); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63287658/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java index 2a1d3ff..e249af3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java @@ -90,8 +90,8 @@ public class DrillRuleSets { DrillLimitRule.INSTANCE, DrillSortRule.INSTANCE, DrillJoinRule.INSTANCE, - DrillUnionRule.INSTANCE - + DrillUnionRule.INSTANCE, + MergeProjectRule.INSTANCE )); public static final RuleSet DRILL_PHYSICAL_MEM = new DrillRuleSet(ImmutableSet.of( // http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/63287658/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MergeJoinPrel.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MergeJoinPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MergeJoinPrel.java index bfb2192..0ced205 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MergeJoinPrel.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/MergeJoinPrel.java @@ -55,7 +55,7 @@ public class MergeJoinPrel extends DrillJoinRelBase implements Prel { } RexNode remaining = RelOptUtil.splitJoinCondition(left, right, condition, leftKeys, rightKeys); - if (!remaining.isAlwaysTrue()) { + if (!remaining.isAlwaysTrue() && (leftKeys.size() == 0 || rightKeys.size() == 0)) { throw new InvalidRelException("MergeJoinPrel only supports equi-join"); } }
