[hive] branch master updated (351bfe2 -> fffb31f)
This is an automated email from the ASF dual-hosted git repository. sankarh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 351bfe2 HIVE-25818: Values query with order by position clause fails (Krisztian Kasa, reviewed by Zoltan Haindrich) add fffb31f Revert "HIVE-25653: Incorrect results returned by STDDEV, STDDEV_SAMP, STDDEV_POP for floating point data types (Ashish Sharma, reviewed by Adesh Rao, Sankar Hariappan)" No new revisions were added by this update. Summary of changes: .../hadoop/hive/ql/udf/generic/GenericUDAFStd.java | 8 +- .../hive/ql/udf/generic/GenericUDAFVariance.java | 29 ++ ql/src/test/queries/clientpositive/stddev.q| 14 --- .../clientpositive/llap/cbo_rp_windowing_2.q.out | 42 - .../test/results/clientpositive/llap/stddev.q.out | 102 - .../clientpositive/llap/vector_windowing.q.out | 42 - .../results/clientpositive/llap/windowing.q.out| 42 - 7 files changed, 74 insertions(+), 205 deletions(-) delete mode 100644 ql/src/test/queries/clientpositive/stddev.q delete mode 100644 ql/src/test/results/clientpositive/llap/stddev.q.out
[hive] branch master updated: HIVE-25818: Values query with order by position clause fails (Krisztian Kasa, reviewed by Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository. krisztiankasa pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new 351bfe2 HIVE-25818: Values query with order by position clause fails (Krisztian Kasa, reviewed by Zoltan Haindrich) 351bfe2 is described below commit 351bfe2ffc4abce52801769743e109260d06f9be Author: Krisztian Kasa AuthorDate: Wed Dec 22 05:44:01 2021 +0100 HIVE-25818: Values query with order by position clause fails (Krisztian Kasa, reviewed by Zoltan Haindrich) --- .../org/apache/hadoop/hive/ql/parse/CalcitePlanner.java| 14 ++ 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 65391d3..12dcebd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -4777,7 +4777,7 @@ public class CalcitePlanner extends SemanticAnalyzer { // need to remember the expressions and the alias. // In OP return path, we need to generate a SEL and then a UDTF // following old semantic analyzer. -outputRel = genUDTFPlan(genericUDTF, genericUDTFName, udtfTableAlias, udtfColAliases, qb, +return genUDTFPlan(genericUDTF, genericUDTFName, udtfTableAlias, udtfColAliases, qb, columnList, outputRR, srcRel); } else { String dest = qbp.getClauseNames().iterator().next(); @@ -4869,8 +4869,14 @@ public class CalcitePlanner extends SemanticAnalyzer { return i; } -private RelNode genUDTFPlan(GenericUDTF genericUDTF, String genericUDTFName, String outputTableAlias, -ArrayList colAliases, QB qb, List selectColLst, RowResolver selectRR, RelNode input) throws SemanticException { +private Pair genUDTFPlan(GenericUDTF genericUDTF, + String genericUDTFName, + String outputTableAlias, + ArrayList colAliases, + QB qb, + List selectColLst, + RowResolver selectRR, + RelNode input) throws SemanticException { // No GROUP BY / DISTRIBUTE BY / SORT BY / CLUSTER BY QBParseInfo qbp = qb.getParseInfo(); @@ -4968,7 +4974,7 @@ public class CalcitePlanner extends SemanticAnalyzer { relToHiveColNameCalcitePosMap.put(udtf, buildHiveToCalciteColumnMap(outputRR)); relToHiveRR.put(udtf, outputRR); - return udtf; + return new Pair<>(udtf, outputRR); } private Pair genGBSelectDistinctPlan(Pair srcNodeRR)
[hive] branch master updated: HIVE-25805: Wrong result when rebuilding MV with count(col) incrementally (Krisztian Kasa, reviewed by Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository. krisztiankasa pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new 8572c12 HIVE-25805: Wrong result when rebuilding MV with count(col) incrementally (Krisztian Kasa, reviewed by Zoltan Haindrich) 8572c12 is described below commit 8572c1201e1d483eb03c7e413f4ff7f9b6f4a3d2 Author: Krisztian Kasa AuthorDate: Tue Dec 21 14:37:34 2021 +0100 HIVE-25805: Wrong result when rebuilding MV with count(col) incrementally (Krisztian Kasa, reviewed by Zoltan Haindrich) --- ...regateInsertDeleteIncrementalRewritingRule.java | 18 +- .../materialized_view_create_rewrite_6.q | 14 +- .../llap/materialized_view_create_rewrite_6.q.out | 208 ++--- 3 files changed, 118 insertions(+), 122 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAggregateInsertDeleteIncrementalRewritingRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAggregateInsertDeleteIncrementalRewritingRule.java index d4d4bdd..601cbfc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAggregateInsertDeleteIncrementalRewritingRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAggregateInsertDeleteIncrementalRewritingRule.java @@ -139,7 +139,14 @@ public class HiveAggregateInsertDeleteIncrementalRewritingRule extends HiveAggre switch (aggregateCall.getAggregation().getKind()) { case COUNT: aggFunction = SqlStdOperatorTable.SUM; - argument = relBuilder.literal(1); + + // count(*) + if (aggregateCall.getArgList().isEmpty()) { +argument = relBuilder.literal(1); + } else { +// count() +argument = genArgumentForCountColumn(relBuilder, aggInput, aggregateCall.getArgList().get(0)); + } break; case SUM: aggFunction = SqlStdOperatorTable.SUM; @@ -171,6 +178,15 @@ public class HiveAggregateInsertDeleteIncrementalRewritingRule extends HiveAggre countIdx); } + private RexNode genArgumentForCountColumn(RelBuilder relBuilder, RelNode aggInput, int argumentIdx) { +RexBuilder rexBuilder = relBuilder.getRexBuilder(); +RexNode countArg = rexBuilder.makeInputRef( +aggInput.getRowType().getFieldList().get(argumentIdx).getType(), argumentIdx); +RexNode isNull = rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL, countArg); +return rexBuilder.makeCall( +SqlStdOperatorTable.CASE, isNull, relBuilder.literal(0), relBuilder.literal(1)); + } + /** * Extend {@link HiveAggregateIncrementalRewritingRuleBase.IncrementalComputePlan} with countStarIndex. It is * required for building the filter condition that distribute the rows to the insert/update/delete branches. diff --git a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_6.q b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_6.q index e7d3449..6024657 100644 --- a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_6.q +++ b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_6.q @@ -21,13 +21,13 @@ insert into t2(a, b) values ('sum0', 0); -- Aggregate with count(*): incremental rebuild should be triggered even if there were deletes from source table create materialized view mat1 stored as orc TBLPROPERTIES ('transactional'='true') as -select t1.a, sum(t1.b), count(*) from t1 +select t1.a, sum(t1.b), count(t1.b), avg(t1.b), count(*) from t1 join t2 on (t1.a = t2.a) group by t1.a; explain cbo -select t1.a, sum(t1.b) from t1 +select t1.a, sum(t1.b), count(t1.b), avg(t1.b), count(*) from t1 join t2 on (t1.a = t2.a) group by t1.a; @@ -54,7 +54,7 @@ delete from t1 where a like '%add/remove'; -- view can not be used explain cbo -select t1.a, sum(t1.b) from t1 +select t1.a, sum(t1.b), count(t1.b), avg(t1.b), count(*) from t1 join t2 on (t1.a = t2.a) group by t1.a; @@ -68,16 +68,12 @@ alter materialized view mat1 rebuild; -- the view should be up to date and used explain cbo -select t1.a, sum(t1.b) from t1 +select t1.a, sum(t1.b), count(t1.b), avg(t1.b), count(*) from t1 join t2 on (t1.a = t2.a) group by t1.a; -select t1.a, sum(t1.b) from t1 +select t1.a, sum(t1.b), count(t1.b), avg(t1.b), count(*) from t1 join t2 on (t1.a = t2.a) group by t1.a; drop materialized view mat1; - -select t1.a, sum(t1.b) from t1 -join t2 on (t1.a = t2.a) -group by t1.a; diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6.q.out index 19f810b..f299bbd 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6.q.out +++ b/q
[hive] branch master updated: HIVE-25822: Unexpected result rows in case of outer join contains conditions only affecting one side (#2891) (Zoltan Haindrich reviewed by Stamatis Zampetakis, Aman Sinha
This is an automated email from the ASF dual-hosted git repository. kgyrtkirk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new 621bfd1 HIVE-25822: Unexpected result rows in case of outer join contains conditions only affecting one side (#2891) (Zoltan Haindrich reviewed by Stamatis Zampetakis, Aman Sinha) 621bfd1 is described below commit 621bfd164f018063fe5e03d9f7a7d990ce22691a Author: Zoltan Haindrich AuthorDate: Tue Dec 21 11:50:45 2021 +0100 HIVE-25822: Unexpected result rows in case of outer join contains conditions only affecting one side (#2891) (Zoltan Haindrich reviewed by Stamatis Zampetakis, Aman Sinha) --- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 2 + .../hive/ql/exec/CommonMergeJoinOperator.java | 80 -- .../clientpositive/outer_join_unexpected_rows.q| 17 .../test/results/clientpositive/llap/join46.q.out | 4 +- .../results/clientpositive/llap/join_1to1.q.out| 64 +-- .../clientpositive/llap/join_emit_interval.q.out | 6 +- .../results/clientpositive/llap/mapjoin2.q.out | 2 +- .../llap/outer_join_unexpected_rows.q.out | 95 ++ .../llap/vector_full_outer_join2.q.out | 18 ++-- 9 files changed, 208 insertions(+), 80 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index fd95070..b0b9b4f 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1836,6 +1836,8 @@ public class HiveConf extends Configuration { HIVEALIAS("hive.alias", "", ""), HIVEMAPSIDEAGGREGATE("hive.map.aggr", true, "Whether to use map-side aggregation in Hive Group By queries"), HIVEGROUPBYSKEW("hive.groupby.skewindata", false, "Whether there is skew in data to optimize group by queries"), +HIVE_JOIN_SHORTCUT_UNMATCHED_ROWS("hive.join.shortcut.unmatched.rows", true, +"Enables to shortcut processing of known filtered rows in merge joins. internal use only. may affect correctness"), HIVEJOINEMITINTERVAL("hive.join.emit.interval", 1000, "How many rows in the right-most join operand Hive should buffer before emitting the join result."), HIVEJOINCACHESIZE("hive.join.cache.size", 25000, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java index e574fb9..0044a04 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java @@ -42,12 +42,12 @@ import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.plan.CommonMergeJoinDesc; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.apache.hadoop.hive.ql.plan.JoinCondDesc; +import org.apache.hadoop.hive.ql.plan.JoinDesc; import org.apache.hadoop.hive.ql.plan.OperatorDesc; import org.apache.hadoop.hive.ql.plan.api.OperatorType; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; -import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.io.WritableComparator; /* @@ -74,6 +74,7 @@ public class CommonMergeJoinOperator extends AbstractMapJoinOperator[] nextKeyWritables; transient RowContainer>[] nextGroupStorage; transient RowContainer>[] candidateStorage; + transient RowContainer>[] unmatchedStorage; transient String[] tagToAlias; private transient boolean[] fetchDone; @@ -94,6 +95,7 @@ public class CommonMergeJoinOperator extends AbstractMapJoinOperator> unmatchedRC = JoinUtil.getRowContainer(hconf, + rowContainerStandardObjectInspectors[pos], pos, bucketSize, spillTableDesc, conf, !hasFilter(pos), reporter); + unmatchedStorage[pos] = unmatchedRC; } for (byte pos = 0; pos < order.length; pos++) { @@ -240,6 +248,18 @@ public class CommonMergeJoinOperator extends AbstractMapJoinOperator value = getFilteredValue(alias, row); + +if (isOuterJoinUnmatchedRow(tag, value)) { + int type = condn[0].getType(); + if (tag == 0 && (type == JoinDesc.LEFT_OUTER_JOIN || type == JoinDesc.FULL_OUTER_JOIN)) { +unmatchedStorage[tag].addRow(value); + } + if (tag == 1 && (type == JoinDesc.RIGHT_OUTER_JOIN || type == JoinDesc.FULL_OUTER_JOIN)) { +unmatchedStorage[tag].addRow(value); + } + emitUnmatchedRows(tag, false); + return; +} // compute keys and values as StandardObjects List key = mergeJoinComputeKeys(row, alias); // Fetch the first group for all small
[hive] branch master updated (3a12055 -> bb26fac)
This is an automated email from the ASF dual-hosted git repository. pvary pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 3a12055 HIVE-25786: Auto-close browser window/tab after successful auth with SSO (Saihemanth Gantasala via Naveen Gangam) add bb26fac HIVE-25792: Recompile the query if CBO has failed (Peter Vary reviewed by Stamatis Zampetakis and Zoltan Haindrich)(#2865) No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/hive/conf/HiveConf.java | 7 +- .../TestReExecuteKilledTezAMQueryPlugin.java | 2 +- .../java/org/apache/hadoop/hive/ql/Compiler.java | 19 +- ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 9 +- .../org/apache/hadoop/hive/ql/DriverFactory.java | 9 +- .../java/org/apache/hadoop/hive/ql/Executor.java | 6 +- .../java/org/apache/hadoop/hive/ql/HookRunner.java | 27 ++- .../apache/hadoop/hive/ql/hooks/HookContext.java | 6 +- .../hadoop/hive/ql/hooks/PrivateHookContext.java | 12 ++ .../hadoop/hive/ql/parse/CBOFallbackStrategy.java | 24 ++- .../hadoop/hive/ql/parse/CalcitePlanner.java | 26 +-- .../hive/ql/parse/ExplainSemanticAnalyzer.java | 7 +- .../hadoop/hive/ql/reexec/IReExecutionPlugin.java | 62 ++- .../hadoop/hive/ql/reexec/ReCompileException.java | 18 +- .../hive/ql/reexec/ReCompileWithoutCBOPlugin.java | 85 + .../apache/hadoop/hive/ql/reexec/ReExecDriver.java | 196 - .../hive/ql/reexec/ReExecuteLostAMQueryPlugin.java | 14 +- .../hive/ql/reexec/ReExecutionDagSubmitPlugin.java | 14 +- .../hive/ql/reexec/ReExecutionOverlayPlugin.java | 11 +- .../hadoop/hive/ql/reexec/ReOptimizePlugin.java| 2 +- .../cbo_fallback_wrong_configuration_exception.q | 6 + ql/src/test/queries/clientpositive/retry_failure.q | 2 +- .../queries/clientpositive/retry_failure_oom.q | 2 +- .../queries/clientpositive/retry_failure_reorder.q | 2 +- .../clientpositive/retry_failure_stat_changes.q| 2 +- .../queries/clientpositive/runtime_stats_hs2.q | 2 +- .../queries/clientpositive/vector_retry_failure.q | 2 +- ...bo_fallback_wrong_configuration_exception.q.out | 1 + 28 files changed, 374 insertions(+), 201 deletions(-) copy itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/TestZookeeperTokenStorePlain.java => ql/src/java/org/apache/hadoop/hive/ql/reexec/ReCompileException.java (64%) create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/reexec/ReCompileWithoutCBOPlugin.java create mode 100644 ql/src/test/queries/clientnegative/cbo_fallback_wrong_configuration_exception.q create mode 100644 ql/src/test/results/clientnegative/cbo_fallback_wrong_configuration_exception.q.out