[hive] branch master updated (11cbf17 -> 6fbd54c)
This is an automated email from the ASF dual-hosted git repository. krisztiankasa pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 11cbf17 HIVE-23806: Avoid clearing column stat states in all partition in case schema is extended (#1215) (Zoltan Haindrich reviewed by Peter Vary) add 6fbd54c HIVE-23817. Pushing TopN Key operator PKFK inner joins (Attila Magyar, reviewed by Jesus Camacho Rodriguez, Krisztian Kasa) No new revisions were added by this update. Summary of changes: .../org/apache/hadoop/hive/ql/parse/HintParser.g |2 + .../apache/hadoop/hive/ql/parse/HiveLexerParent.g |1 + .../optimizer/calcite/translator/ASTConverter.java | 43 + .../topnkey/TopNKeyPushdownProcessor.java | 122 +- .../apache/hadoop/hive/ql/parse/QBJoinTree.java| 20 + .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 20 +- .../org/apache/hadoop/hive/ql/plan/JoinDesc.java | 34 + .../apache/hadoop/hive/ql/plan/TopNKeyDesc.java| 11 + .../queries/clientpositive/topnkey_inner_join.q| 63 + .../queries/clientpositive/topnkey_inner_join2.q | 56 + .../clientpositive/llap/topnkey_inner_join.q.out | 1394 .../clientpositive/llap/topnkey_inner_join2.q.out | 388 ++ .../perf/tez/constraints/mv_query44.q.out | 118 +- .../perf/tez/constraints/query44.q.out | 118 +- .../perf/tez/constraints/query65.q.out | 168 +-- .../perf/tez/constraints/query99.q.out | 132 +- 16 files changed, 2413 insertions(+), 277 deletions(-) create mode 100644 ql/src/test/queries/clientpositive/topnkey_inner_join.q create mode 100644 ql/src/test/queries/clientpositive/topnkey_inner_join2.q create mode 100644 ql/src/test/results/clientpositive/llap/topnkey_inner_join.q.out create mode 100644 ql/src/test/results/clientpositive/llap/topnkey_inner_join2.q.out
[hive] branch master updated: HIVE-23806: Avoid clearing column stat states in all partition in case schema is extended (#1215) (Zoltan Haindrich reviewed by Peter Vary)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 11cbf17 HIVE-23806: Avoid clearing column stat states in all
partition in case schema is extended (#1215) (Zoltan Haindrich reviewed by
Peter Vary)
11cbf17 is described below
commit 11cbf17e57dc13b800aa9ef4f987582b9219e3dd
Author: Zoltan Haindrich
AuthorDate: Wed Jul 29 10:02:26 2020 +0200
HIVE-23806: Avoid clearing column stat states in all partition in case
schema is extended (#1215) (Zoltan Haindrich reviewed by Peter Vary)
---
.../org/apache/hadoop/hive/metastore/HiveAlterHandler.java | 8 ++--
.../hadoop/hive/metastore/utils/MetaStoreServerUtils.java | 14 ++
2 files changed, 20 insertions(+), 2 deletions(-)
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 2beeab4..0cfb329 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -59,7 +59,6 @@ import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -362,7 +361,12 @@ public class HiveAlterHandler implements AlterHandler {
if (isPartitionedTable) {
//Currently only column related changes can be cascaded in alter
table
- if(!MetaStoreServerUtils.areSameColumns(oldt.getSd().getCols(),
newt.getSd().getCols())) {
+ boolean runPartitionMetadataUpdate =
+ (cascade &&
!MetaStoreServerUtils.areSameColumns(oldt.getSd().getCols(),
newt.getSd().getCols()));
+ // we may skip the update entirely if there are only new columns
added
+ runPartitionMetadataUpdate |=
+ !cascade &&
!MetaStoreServerUtils.arePrefixColumns(oldt.getSd().getCols(),
newt.getSd().getCols());
+ if (runPartitionMetadataUpdate) {
parts = msdb.getPartitions(catName, dbname, name, -1);
for (Partition part : parts) {
Partition oldPart = new Partition(part);
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
index d6cf43c..1969ddd 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
@@ -32,6 +32,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -501,6 +502,19 @@ public class MetaStoreServerUtils {
return ListUtils.isEqualList(oldCols, newCols);
}
+ /**
+ * Returns true if p is a prefix of s.
+ */
+ public static boolean arePrefixColumns(List p,
List s) {
+if (p == s) {
+ return true;
+}
+if (p.size() > s.size()) {
+ return false;
+}
+return ListUtils.isEqualList(p, s.subList(0, p.size()));
+ }
+
public static void updateBasicState(EnvironmentContext environmentContext,
Map
params) {
if (params == null) {
[hive] branch master updated: disable TestStatsReplicationScenariosMMNoAutogather, TestCodahaleReportersConf
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5684401 disable TestStatsReplicationScenariosMMNoAutogather,
TestCodahaleReportersConf
5684401 is described below
commit 56844015ffac23c10bfb898f27960eb5acb923d5
Author: Zoltan Haindrich
AuthorDate: Wed Jul 29 07:24:17 2020 +
disable TestStatsReplicationScenariosMMNoAutogather,
TestCodahaleReportersConf
---
.../hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java | 1 +
.../hive/ql/parse/TestStatsReplicationScenariosMMNoAutogather.java | 1 +
2 files changed, 2 insertions(+)
diff --git
a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java
b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java
index e89a605..80db730 100644
---
a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java
+++
b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java
@@ -33,6 +33,7 @@ import java.io.File;
/**
* Unit tests for Codahale reporter config backward compatibility
*/
+@org.junit.Ignore("HIVE-23945")
public class TestCodahaleReportersConf {
private static File workDir = new File(System.getProperty("test.tmp.dir"));
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenariosMMNoAutogather.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenariosMMNoAutogather.java
index 1d77796..9df81b3 100644
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenariosMMNoAutogather.java
+++
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestStatsReplicationScenariosMMNoAutogather.java
@@ -30,6 +30,7 @@ import java.util.Map;
/**
* Tests statistics replication for ACID tables.
*/
+@org.junit.Ignore("HIVE-23944")
public class TestStatsReplicationScenariosMMNoAutogather extends
TestStatsReplicationScenarios {
@Rule
public final TestName testName = new TestName();
