[hive] branch master updated (351bfe2 -> fffb31f)
This is an automated email from the ASF dual-hosted git repository. sankarh pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 351bfe2 HIVE-25818: Values query with order by position clause fails (Krisztian Kasa, reviewed by Zoltan Haindrich) add fffb31f Revert "HIVE-25653: Incorrect results returned by STDDEV, STDDEV_SAMP, STDDEV_POP for floating point data types (Ashish Sharma, reviewed by Adesh Rao, Sankar Hariappan)" No new revisions were added by this update. Summary of changes: .../hadoop/hive/ql/udf/generic/GenericUDAFStd.java | 8 +- .../hive/ql/udf/generic/GenericUDAFVariance.java | 29 ++ ql/src/test/queries/clientpositive/stddev.q| 14 --- .../clientpositive/llap/cbo_rp_windowing_2.q.out | 42 - .../test/results/clientpositive/llap/stddev.q.out | 102 - .../clientpositive/llap/vector_windowing.q.out | 42 - .../results/clientpositive/llap/windowing.q.out| 42 - 7 files changed, 74 insertions(+), 205 deletions(-) delete mode 100644 ql/src/test/queries/clientpositive/stddev.q delete mode 100644 ql/src/test/results/clientpositive/llap/stddev.q.out
[hive] branch master updated: HIVE-25818: Values query with order by position clause fails (Krisztian Kasa, reviewed by Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 351bfe2 HIVE-25818: Values query with order by position clause fails
(Krisztian Kasa, reviewed by Zoltan Haindrich)
351bfe2 is described below
commit 351bfe2ffc4abce52801769743e109260d06f9be
Author: Krisztian Kasa
AuthorDate: Wed Dec 22 05:44:01 2021 +0100
HIVE-25818: Values query with order by position clause fails (Krisztian
Kasa, reviewed by Zoltan Haindrich)
---
.../org/apache/hadoop/hive/ql/parse/CalcitePlanner.java| 14 ++
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 65391d3..12dcebd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -4777,7 +4777,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
// need to remember the expressions and the alias.
// In OP return path, we need to generate a SEL and then a UDTF
// following old semantic analyzer.
-outputRel = genUDTFPlan(genericUDTF, genericUDTFName, udtfTableAlias,
udtfColAliases, qb,
+return genUDTFPlan(genericUDTF, genericUDTFName, udtfTableAlias,
udtfColAliases, qb,
columnList, outputRR, srcRel);
} else {
String dest = qbp.getClauseNames().iterator().next();
@@ -4869,8 +4869,14 @@ public class CalcitePlanner extends SemanticAnalyzer {
return i;
}
-private RelNode genUDTFPlan(GenericUDTF genericUDTF, String
genericUDTFName, String outputTableAlias,
-ArrayList colAliases, QB qb, List selectColLst,
RowResolver selectRR, RelNode input) throws SemanticException {
+private Pair genUDTFPlan(GenericUDTF genericUDTF,
+ String genericUDTFName,
+ String outputTableAlias,
+ ArrayList
colAliases,
+ QB qb,
+ List selectColLst,
+ RowResolver selectRR,
+ RelNode input) throws
SemanticException {
// No GROUP BY / DISTRIBUTE BY / SORT BY / CLUSTER BY
QBParseInfo qbp = qb.getParseInfo();
@@ -4968,7 +4974,7 @@ public class CalcitePlanner extends SemanticAnalyzer {
relToHiveColNameCalcitePosMap.put(udtf,
buildHiveToCalciteColumnMap(outputRR));
relToHiveRR.put(udtf, outputRR);
- return udtf;
+ return new Pair<>(udtf, outputRR);
}
private Pair genGBSelectDistinctPlan(Pair srcNodeRR)
[hive] branch master updated: HIVE-25805: Wrong result when rebuilding MV with count(col) incrementally (Krisztian Kasa, reviewed by Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8572c12 HIVE-25805: Wrong result when rebuilding MV with count(col)
incrementally (Krisztian Kasa, reviewed by Zoltan Haindrich)
8572c12 is described below
commit 8572c1201e1d483eb03c7e413f4ff7f9b6f4a3d2
Author: Krisztian Kasa
AuthorDate: Tue Dec 21 14:37:34 2021 +0100
HIVE-25805: Wrong result when rebuilding MV with count(col) incrementally
(Krisztian Kasa, reviewed by Zoltan Haindrich)
---
...regateInsertDeleteIncrementalRewritingRule.java | 18 +-
.../materialized_view_create_rewrite_6.q | 14 +-
.../llap/materialized_view_create_rewrite_6.q.out | 208 ++---
3 files changed, 118 insertions(+), 122 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAggregateInsertDeleteIncrementalRewritingRule.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAggregateInsertDeleteIncrementalRewritingRule.java
index d4d4bdd..601cbfc 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAggregateInsertDeleteIncrementalRewritingRule.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/views/HiveAggregateInsertDeleteIncrementalRewritingRule.java
@@ -139,7 +139,14 @@ public class
HiveAggregateInsertDeleteIncrementalRewritingRule extends HiveAggre
switch (aggregateCall.getAggregation().getKind()) {
case COUNT:
aggFunction = SqlStdOperatorTable.SUM;
- argument = relBuilder.literal(1);
+
+ // count(*)
+ if (aggregateCall.getArgList().isEmpty()) {
+argument = relBuilder.literal(1);
+ } else {
+// count()
+argument = genArgumentForCountColumn(relBuilder, aggInput,
aggregateCall.getArgList().get(0));
+ }
break;
case SUM:
aggFunction = SqlStdOperatorTable.SUM;
@@ -171,6 +178,15 @@ public class
HiveAggregateInsertDeleteIncrementalRewritingRule extends HiveAggre
countIdx);
}
+ private RexNode genArgumentForCountColumn(RelBuilder relBuilder, RelNode
aggInput, int argumentIdx) {
+RexBuilder rexBuilder = relBuilder.getRexBuilder();
+RexNode countArg = rexBuilder.makeInputRef(
+aggInput.getRowType().getFieldList().get(argumentIdx).getType(),
argumentIdx);
+RexNode isNull = rexBuilder.makeCall(SqlStdOperatorTable.IS_NULL,
countArg);
+return rexBuilder.makeCall(
+SqlStdOperatorTable.CASE, isNull, relBuilder.literal(0),
relBuilder.literal(1));
+ }
+
/**
* Extend {@link
HiveAggregateIncrementalRewritingRuleBase.IncrementalComputePlan} with
countStarIndex. It is
* required for building the filter condition that distribute the rows to
the insert/update/delete branches.
diff --git
a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_6.q
b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_6.q
index e7d3449..6024657 100644
--- a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_6.q
+++ b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_6.q
@@ -21,13 +21,13 @@ insert into t2(a, b) values ('sum0', 0);
-- Aggregate with count(*): incremental rebuild should be triggered even if
there were deletes from source table
create materialized view mat1 stored as orc TBLPROPERTIES
('transactional'='true') as
-select t1.a, sum(t1.b), count(*) from t1
+select t1.a, sum(t1.b), count(t1.b), avg(t1.b), count(*) from t1
join t2 on (t1.a = t2.a)
group by t1.a;
explain cbo
-select t1.a, sum(t1.b) from t1
+select t1.a, sum(t1.b), count(t1.b), avg(t1.b), count(*) from t1
join t2 on (t1.a = t2.a)
group by t1.a;
@@ -54,7 +54,7 @@ delete from t1 where a like '%add/remove';
-- view can not be used
explain cbo
-select t1.a, sum(t1.b) from t1
+select t1.a, sum(t1.b), count(t1.b), avg(t1.b), count(*) from t1
join t2 on (t1.a = t2.a)
group by t1.a;
@@ -68,16 +68,12 @@ alter materialized view mat1 rebuild;
-- the view should be up to date and used
explain cbo
-select t1.a, sum(t1.b) from t1
+select t1.a, sum(t1.b), count(t1.b), avg(t1.b), count(*) from t1
join t2 on (t1.a = t2.a)
group by t1.a;
-select t1.a, sum(t1.b) from t1
+select t1.a, sum(t1.b), count(t1.b), avg(t1.b), count(*) from t1
join t2 on (t1.a = t2.a)
group by t1.a;
drop materialized view mat1;
-
-select t1.a, sum(t1.b) from t1
-join t2 on (t1.a = t2.a)
-group by t1.a;
diff --git
a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6.q.out
b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6.q.out
index 19f810b..f299bbd 100644
---
a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_6.q.out
+++
[hive] branch master updated: HIVE-25822: Unexpected result rows in case of outer join contains conditions only affecting one side (#2891) (Zoltan Haindrich reviewed by Stamatis Zampetakis, Aman Sinha
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 621bfd1 HIVE-25822: Unexpected result rows in case of outer join
contains conditions only affecting one side (#2891) (Zoltan Haindrich reviewed
by Stamatis Zampetakis, Aman Sinha)
621bfd1 is described below
commit 621bfd164f018063fe5e03d9f7a7d990ce22691a
Author: Zoltan Haindrich
AuthorDate: Tue Dec 21 11:50:45 2021 +0100
HIVE-25822: Unexpected result rows in case of outer join contains
conditions only affecting one side (#2891) (Zoltan Haindrich reviewed by
Stamatis Zampetakis, Aman Sinha)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 2 +
.../hive/ql/exec/CommonMergeJoinOperator.java | 80 --
.../clientpositive/outer_join_unexpected_rows.q| 17
.../test/results/clientpositive/llap/join46.q.out | 4 +-
.../results/clientpositive/llap/join_1to1.q.out| 64 +--
.../clientpositive/llap/join_emit_interval.q.out | 6 +-
.../results/clientpositive/llap/mapjoin2.q.out | 2 +-
.../llap/outer_join_unexpected_rows.q.out | 95 ++
.../llap/vector_full_outer_join2.q.out | 18 ++--
9 files changed, 208 insertions(+), 80 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index fd95070..b0b9b4f 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1836,6 +1836,8 @@ public class HiveConf extends Configuration {
HIVEALIAS("hive.alias", "", ""),
HIVEMAPSIDEAGGREGATE("hive.map.aggr", true, "Whether to use map-side
aggregation in Hive Group By queries"),
HIVEGROUPBYSKEW("hive.groupby.skewindata", false, "Whether there is skew
in data to optimize group by queries"),
+HIVE_JOIN_SHORTCUT_UNMATCHED_ROWS("hive.join.shortcut.unmatched.rows",
true,
+"Enables to shortcut processing of known filtered rows in merge joins.
internal use only. may affect correctness"),
HIVEJOINEMITINTERVAL("hive.join.emit.interval", 1000,
"How many rows in the right-most join operand Hive should buffer
before emitting the join result."),
HIVEJOINCACHESIZE("hive.join.cache.size", 25000,
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
index e574fb9..0044a04 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonMergeJoinOperator.java
@@ -42,12 +42,12 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.CommonMergeJoinDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
+import org.apache.hadoop.hive.ql.plan.JoinDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
/*
@@ -74,6 +74,7 @@ public class CommonMergeJoinOperator extends
AbstractMapJoinOperator[] nextKeyWritables;
transient RowContainer>[] nextGroupStorage;
transient RowContainer>[] candidateStorage;
+ transient RowContainer>[] unmatchedStorage;
transient String[] tagToAlias;
private transient boolean[] fetchDone;
@@ -94,6 +95,7 @@ public class CommonMergeJoinOperator extends
AbstractMapJoinOperator> unmatchedRC = JoinUtil.getRowContainer(hconf,
+ rowContainerStandardObjectInspectors[pos], pos, bucketSize,
spillTableDesc, conf, !hasFilter(pos), reporter);
+ unmatchedStorage[pos] = unmatchedRC;
}
for (byte pos = 0; pos < order.length; pos++) {
@@ -240,6 +248,18 @@ public class CommonMergeJoinOperator extends
AbstractMapJoinOperator value = getFilteredValue(alias, row);
+
+if (isOuterJoinUnmatchedRow(tag, value)) {
+ int type = condn[0].getType();
+ if (tag == 0 && (type == JoinDesc.LEFT_OUTER_JOIN || type ==
JoinDesc.FULL_OUTER_JOIN)) {
+unmatchedStorage[tag].addRow(value);
+ }
+ if (tag == 1 && (type == JoinDesc.RIGHT_OUTER_JOIN || type ==
JoinDesc.FULL_OUTER_JOIN)) {
+unmatchedStorage[tag].addRow(value);
+ }
+ emitUnmatchedRows(tag, false);
+ return;
+}
// compute keys and values as StandardObjects
List key = mergeJoinComputeKeys(row, alias);
// Fetch the first group for all
[hive] branch master updated (3a12055 -> bb26fac)
This is an automated email from the ASF dual-hosted git repository. pvary pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 3a12055 HIVE-25786: Auto-close browser window/tab after successful auth with SSO (Saihemanth Gantasala via Naveen Gangam) add bb26fac HIVE-25792: Recompile the query if CBO has failed (Peter Vary reviewed by Stamatis Zampetakis and Zoltan Haindrich)(#2865) No new revisions were added by this update. Summary of changes: .../java/org/apache/hadoop/hive/conf/HiveConf.java | 7 +- .../TestReExecuteKilledTezAMQueryPlugin.java | 2 +- .../java/org/apache/hadoop/hive/ql/Compiler.java | 19 +- ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 9 +- .../org/apache/hadoop/hive/ql/DriverFactory.java | 9 +- .../java/org/apache/hadoop/hive/ql/Executor.java | 6 +- .../java/org/apache/hadoop/hive/ql/HookRunner.java | 27 ++- .../apache/hadoop/hive/ql/hooks/HookContext.java | 6 +- .../hadoop/hive/ql/hooks/PrivateHookContext.java | 12 ++ .../hadoop/hive/ql/parse/CBOFallbackStrategy.java | 24 ++- .../hadoop/hive/ql/parse/CalcitePlanner.java | 26 +-- .../hive/ql/parse/ExplainSemanticAnalyzer.java | 7 +- .../hadoop/hive/ql/reexec/IReExecutionPlugin.java | 62 ++- .../hadoop/hive/ql/reexec/ReCompileException.java | 18 +- .../hive/ql/reexec/ReCompileWithoutCBOPlugin.java | 85 + .../apache/hadoop/hive/ql/reexec/ReExecDriver.java | 196 - .../hive/ql/reexec/ReExecuteLostAMQueryPlugin.java | 14 +- .../hive/ql/reexec/ReExecutionDagSubmitPlugin.java | 14 +- .../hive/ql/reexec/ReExecutionOverlayPlugin.java | 11 +- .../hadoop/hive/ql/reexec/ReOptimizePlugin.java| 2 +- .../cbo_fallback_wrong_configuration_exception.q | 6 + ql/src/test/queries/clientpositive/retry_failure.q | 2 +- .../queries/clientpositive/retry_failure_oom.q | 2 +- .../queries/clientpositive/retry_failure_reorder.q | 2 +- .../clientpositive/retry_failure_stat_changes.q| 2 +- .../queries/clientpositive/runtime_stats_hs2.q | 2 +- .../queries/clientpositive/vector_retry_failure.q | 2 +- ...bo_fallback_wrong_configuration_exception.q.out | 1 + 28 files changed, 374 insertions(+), 201 deletions(-) copy itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/security/TestZookeeperTokenStorePlain.java => ql/src/java/org/apache/hadoop/hive/ql/reexec/ReCompileException.java (64%) create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/reexec/ReCompileWithoutCBOPlugin.java create mode 100644 ql/src/test/queries/clientnegative/cbo_fallback_wrong_configuration_exception.q create mode 100644 ql/src/test/results/clientnegative/cbo_fallback_wrong_configuration_exception.q.out
