This is an automated email from the ASF dual-hosted git repository. djaiswal pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new cdd8fa5 HIVE-21408 : Disable synthetic join predicates for non-equi joins for unintended cases (Deepak Jaiswal, reviewed by Vineet Garg) cdd8fa5 is described below commit cdd8fa58f4b7f6aa2973fb4a114bb891ed397416 Author: Deepak Jaiswal <djais...@apache.org> AuthorDate: Fri Mar 8 11:14:10 2019 -0800 HIVE-21408 : Disable synthetic join predicates for non-equi joins for unintended cases (Deepak Jaiswal, reviewed by Vineet Garg) --- .../DynamicPartitionPruningOptimization.java | 24 +++-- .../clientpositive/dynamic_semijoin_reduction.q | 3 + .../llap/dynamic_semijoin_reduction.q.out | 111 +++++++++++++++++++++ 3 files changed, 129 insertions(+), 9 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java index 439fb75..cc01034 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/DynamicPartitionPruningOptimization.java @@ -167,7 +167,8 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor { Table table = ts.getConf().getTableMetadata(); - if (table != null && table.isPartitionKey(column)) { + boolean nonEquiJoin = isNonEquiJoin(ctx.parent); + if (table != null && table.isPartitionKey(column) && !nonEquiJoin) { String columnType = table.getPartColByName(column).getType(); String alias = ts.getConf().getAlias(); PrunedPartitionList plist = parseContext.getPrunedPartitions(alias, ts); @@ -197,7 +198,7 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor { } else { // semijoin LOG.debug("Column " + column + " is not a partition column"); if (semiJoin && !disableSemiJoinOptDueToExternalTable(parseContext.getConf(), ts, ctx) - && ts.getConf().getFilterExpr() != null) { + && ts.getConf().getFilterExpr() != null && !nonEquiJoin) { LOG.debug("Initiate semijoin reduction for " + column + " (" + ts.getConf().getFilterExpr().getExprString()); @@ -434,6 +435,18 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor { } } + private boolean isNonEquiJoin(ExprNodeDesc predicate) { + Preconditions.checkArgument(predicate instanceof ExprNodeGenericFuncDesc); + + ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) predicate; + if (funcDesc.getGenericUDF() instanceof GenericUDFIn) { + return false; + } + + return true; + } + + private void generateEventOperatorPlan(DynamicListContext ctx, ParseContext parseContext, TableScanOperator ts, String column, String columnType) { @@ -530,13 +543,6 @@ public class DynamicPartitionPruningOptimization implements NodeProcessor { TableScanOperator ts, String keyBaseAlias, String internalColName, String colName, SemiJoinHint sjHint) throws SemanticException { - // Semijoin reduction for non-equi join not yet supported, check for it - ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) ctx.parent; - if (!(funcDesc.getGenericUDF() instanceof GenericUDFIn)) { - LOG.info("Semijoin reduction for non-equi joins is currently disabled."); - return false; - } - // we will put a fork in the plan at the source of the reduce sink Operator<? extends OperatorDesc> parentOfRS = ctx.generator.getParentOperators().get(0); diff --git a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q index 32c2854..f5a5824 100644 --- a/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q +++ b/ql/src/test/queries/clientpositive/dynamic_semijoin_reduction.q @@ -141,6 +141,9 @@ explain select * from alltypesorc_int_n1 join select srcpart_small_n3.key1 as key from srcpart_small_n3) unionsrc on (alltypesorc_int_n1.cstring = unionsrc.key); +-- Non equi joins : Should NOT create a semijoin branch +EXPLAIN select count(*) from srcpart_date_n7 join srcpart_small_n3 on (srcpart_date_n7.key = srcpart_small_n3.key1 and srcpart_date_n7.value > srcpart_small_n3.value1); + drop table srcpart_date_n7; drop table srcpart_small_n3; diff --git a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out index 74bb1a0..d5e4a2a 100644 --- a/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out +++ b/ql/src/test/results/clientpositive/llap/dynamic_semijoin_reduction.q.out @@ -4151,6 +4151,117 @@ STAGE PLANS: Processor Tree: ListSink +PREHOOK: query: EXPLAIN select count(*) from srcpart_date_n7 join srcpart_small_n3 on (srcpart_date_n7.key = srcpart_small_n3.key1 and srcpart_date_n7.value > srcpart_small_n3.value1) +PREHOOK: type: QUERY +PREHOOK: Input: default@srcpart_date_n7 +PREHOOK: Input: default@srcpart_date_n7@ds=2008-04-08 +PREHOOK: Input: default@srcpart_date_n7@ds=2008-04-09 +PREHOOK: Input: default@srcpart_small_n3 +PREHOOK: Input: default@srcpart_small_n3@ds=2008-04-08 +PREHOOK: Input: default@srcpart_small_n3@ds=2008-04-09 +#### A masked pattern was here #### +POSTHOOK: query: EXPLAIN select count(*) from srcpart_date_n7 join srcpart_small_n3 on (srcpart_date_n7.key = srcpart_small_n3.key1 and srcpart_date_n7.value > srcpart_small_n3.value1) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@srcpart_date_n7 +POSTHOOK: Input: default@srcpart_date_n7@ds=2008-04-08 +POSTHOOK: Input: default@srcpart_date_n7@ds=2008-04-09 +POSTHOOK: Input: default@srcpart_small_n3 +POSTHOOK: Input: default@srcpart_small_n3@ds=2008-04-08 +POSTHOOK: Input: default@srcpart_small_n3@ds=2008-04-09 +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Map 1 <- Map 3 (BROADCAST_EDGE) + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: srcpart_date_n7 + filterExpr: key is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Filter Operator + predicate: key is not null (type: boolean) + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: key (type: string), value (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 2000 Data size: 356000 Basic stats: COMPLETE Column stats: COMPLETE + Map Join Operator + condition map: + Inner Join 0 to 1 + keys: + 0 _col0 (type: string) + 1 _col0 (type: string) + outputColumnNames: _col1, _col3 + input vertices: + 1 Map 3 + residual filter predicates: {(_col1 > _col3)} + Statistics: Num rows: 733 Data size: 130474 Basic stats: PARTIAL Column stats: NONE + Select Operator + Statistics: Num rows: 733 Data size: 130474 Basic stats: PARTIAL Column stats: NONE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: all inputs + Map 3 + Map Operator Tree: + TableScan + alias: srcpart_small_n3 + filterExpr: key1 is not null (type: boolean) + Statistics: Num rows: 20 Data size: 3560 Basic stats: PARTIAL Column stats: PARTIAL + Filter Operator + predicate: key1 is not null (type: boolean) + Statistics: Num rows: 20 Data size: 3560 Basic stats: PARTIAL Column stats: PARTIAL + Select Operator + expressions: key1 (type: string), value1 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 20 Data size: 3560 Basic stats: PARTIAL Column stats: PARTIAL + Reduce Output Operator + key expressions: _col0 (type: string) + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 20 Data size: 3560 Basic stats: PARTIAL Column stats: PARTIAL + value expressions: _col1 (type: string) + Execution mode: vectorized, llap + LLAP IO: all inputs + Reducer 2 + Execution mode: vectorized, llap + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + PREHOOK: query: drop table srcpart_date_n7 PREHOOK: type: DROPTABLE PREHOOK: Input: default@srcpart_date_n7