[hive] 01/02: HIVE-25684: Many (~16K) skipped tests in TestGenericUDFInitializeOnCompareUDF (Stamatis Zampetakis, reviewed by Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit db6288dc0f14267c2b22d0c98a33152eadf2c155
Author: Stamatis Zampetakis
AuthorDate: Tue Nov 9 14:59:28 2021 +0100
HIVE-25684: Many (~16K) skipped tests in
TestGenericUDFInitializeOnCompareUDF (Stamatis Zampetakis, reviewed by Zoltan
Haindrich)
Remove parameterized combinations leading to skipped tests and pass
exactly the arguments we need in each test method.
Closes #2775
---
.../TestGenericUDFInitializeOnCompareUDF.java | 76 +++---
1 file changed, 39 insertions(+), 37 deletions(-)
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFInitializeOnCompareUDF.java
b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFInitializeOnCompareUDF.java
index c3c0f01..ff8d66e 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFInitializeOnCompareUDF.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFInitializeOnCompareUDF.java
@@ -23,17 +23,18 @@ import
org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.junit.Assert;
-import org.junit.Assume;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
+import java.util.stream.Stream;
+import static
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE;
+import static
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.UNION;
import static
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardListObjectInspector;
import static
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardMapObjectInspector;
import static
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.getStandardStructObjectInspector;
@@ -57,17 +58,9 @@ import static
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFacto
*
*
*/
-@RunWith(Parameterized.class)
public class TestGenericUDFInitializeOnCompareUDF {
- private final UDFArguments args;
-
- public TestGenericUDFInitializeOnCompareUDF(UDFArguments arguments) {
-this.args = arguments;
- }
-
- @Parameterized.Parameters(name = "{0}")
- public static Collection generateArguments() {
+ private static Collection generateArguments() {
List arguments = new ArrayList<>();
List primitives = new ArrayList<>();
@@ -96,9 +89,29 @@ public class TestGenericUDFInitializeOnCompareUDF {
return arguments;
}
- @Test
- public void testArgsWithDifferentTypeCategoriesThrowsException() {
-
Assume.assumeFalse(args.left.getCategory().equals(args.right.getCategory()));
+ private static Stream generateArgsWithDifferentCategories() {
+return generateArguments().stream().filter(args ->
!args.left.getCategory().equals(args.right.getCategory()));
+ }
+
+ private static Stream
generateArgsWithSameCategoryNoBothPrimitive() {
+return generateArguments().stream().filter(args -> {
+ ObjectInspector.Category left = args.left.getCategory();
+ ObjectInspector.Category right = args.right.getCategory();
+ return left.equals(right) && !(PRIMITIVE.equals(left) &&
PRIMITIVE.equals(right));
+});
+ }
+
+ private static Stream
generateArgsWithSameTypesNoCategoryUnion() {
+return generateArguments().stream().filter(args -> {
+ TypeInfo left = TypeInfoUtils.getTypeInfoFromObjectInspector(args.left);
+ TypeInfo right =
TypeInfoUtils.getTypeInfoFromObjectInspector(args.right);
+ return left.equals(right) && !UNION.equals(left.getCategory()) &&
!UNION.equals(right.getCategory());
+});
+ }
+
+ @ParameterizedTest
+ @MethodSource("generateArgsWithDifferentCategories")
+ public void testArgsWithDifferentTypeCategoriesThrowsException(UDFArguments
args) {
List udfs = Arrays.asList(
new GenericUDFOPEqual(),
new GenericUDFOPEqualNS(),
@@ -113,20 +126,14 @@ public class TestGenericUDFInitializeOnCompareUDF {
try {
u.initialize(new ObjectInspector[] { args.left, args.right });
}catch (UDFArgumentException e){
-Assert.assertTrue("Unexpected message for " + u.getUdfName(),
e.getMessage().contains("Type mismatch"));
+Assertions.assertTrue(e.getMessage().contains("Type mismatch"),
"Unexpected message for " + u.getUdfName());
}
}
}
-
[hive] 02/02: HIVE-25676: Uncaught exception in QTestDatabaseHandler causes unrelated test failures (Stamatis Zampetakis, reviewed by Alessandro Solimando, Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit d78608c4cec174817f5526065adc43fee78ed7c0
Author: Stamatis Zampetakis
AuthorDate: Fri Nov 5 14:35:50 2021 +0100
HIVE-25676: Uncaught exception in QTestDatabaseHandler causes unrelated
test failures (Stamatis Zampetakis, reviewed by Alessandro Solimando, Zoltan
Haindrich)
Catch the exception, log the problem and continue cleaning up the
environment to avoid unrelated failures in other tests and invalid
internal state.
Minor refactoring removing redundant isEmpty() check.
Closes #2766
---
.../apache/hadoop/hive/ql/qoption/QTestDatabaseHandler.java | 12 +---
1 file changed, 5 insertions(+), 7 deletions(-)
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestDatabaseHandler.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestDatabaseHandler.java
index 1c1e7db..85a09b5 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestDatabaseHandler.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestDatabaseHandler.java
@@ -104,9 +104,6 @@ public class QTestDatabaseHandler implements
QTestOptionHandler {
@Override
public void beforeTest(QTestUtil qt) throws Exception {
-if (databaseToScript.isEmpty()) {
- return;
-}
for (Map.Entry dbEntry :
databaseToScript.entrySet()) {
String scriptsDir = QTestUtil.getScriptsDir(qt.getConf());
Path dbScript = Paths.get(scriptsDir, dbEntry.getValue());
@@ -122,12 +119,13 @@ public class QTestDatabaseHandler implements
QTestOptionHandler {
@Override
public void afterTest(QTestUtil qt) throws Exception {
-if (databaseToScript.isEmpty()) {
- return;
-}
for (Map.Entry dbEntry :
databaseToScript.entrySet()) {
AbstractExternalDB db = dbEntry.getKey().create();
- db.cleanupDockerContainer();
+ try {
+db.cleanupDockerContainer();
+ } catch (Exception e) {
+LOG.error("Failed to cleanup database {}", dbEntry.getKey(), e);
+ }
}
databaseToScript.clear();
}
[hive] branch master updated (49a4397 -> d78608c)
This is an automated email from the ASF dual-hosted git repository. zabetak pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 49a4397 HIVE-25697: Upgrade commons-compress to 1.21 (#2788) (Ramesh Kumar Thangarajan reviewed by Zoltan Haindrich) new db6288d HIVE-25684: Many (~16K) skipped tests in TestGenericUDFInitializeOnCompareUDF (Stamatis Zampetakis, reviewed by Zoltan Haindrich) new d78608c HIVE-25676: Uncaught exception in QTestDatabaseHandler causes unrelated test failures (Stamatis Zampetakis, reviewed by Alessandro Solimando, Zoltan Haindrich) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../hive/ql/qoption/QTestDatabaseHandler.java | 12 ++-- .../TestGenericUDFInitializeOnCompareUDF.java | 76 +++--- 2 files changed, 44 insertions(+), 44 deletions(-)
[hive] branch master updated: HIVE-25697: Upgrade commons-compress to 1.21 (#2788) (Ramesh Kumar Thangarajan reviewed by Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository. kgyrtkirk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new 49a4397 HIVE-25697: Upgrade commons-compress to 1.21 (#2788) (Ramesh Kumar Thangarajan reviewed by Zoltan Haindrich) 49a4397 is described below commit 49a43979ec0ad43bb3ab38da96f8bae97fe50e23 Author: Ramesh Kumar AuthorDate: Mon Nov 15 08:03:15 2021 -0800 HIVE-25697: Upgrade commons-compress to 1.21 (#2788) (Ramesh Kumar Thangarajan reviewed by Zoltan Haindrich) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7a25fc1..cd654d0 100644 --- a/pom.xml +++ b/pom.xml @@ -124,7 +124,7 @@ 1.15 3.2.2 4.1 -1.19 +1.21 1.10 1.1 2.6
[hive] branch master updated: HIVE-25365: Insufficient privileges to show partitions when partition columns are authorized (#2515) (Zhihua Deng reviewed by Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 209b5e3 HIVE-25365: Insufficient privileges to show partitions when
partition columns are authorized (#2515) (Zhihua Deng reviewed by Zoltan
Haindrich)
209b5e3 is described below
commit 209b5e37b43ddbe6ffcd37815d9600d24fa77882
Author: dengzh
AuthorDate: Mon Nov 15 17:00:48 2021 +0800
HIVE-25365: Insufficient privileges to show partitions when partition
columns are authorized (#2515) (Zhihua Deng reviewed by Zoltan Haindrich)
---
.../partition/show/ShowPartitionAnalyzer.java | 3 +
.../hadoop/hive/ql/parse/TestColumnAccess.java | 65 ++
2 files changed, 44 insertions(+), 24 deletions(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java
index a4158d6..40500f1 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/show/ShowPartitionAnalyzer.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.hooks.ReadEntity;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer;
+import org.apache.hadoop.hive.ql.parse.ColumnAccessInfo;
import org.apache.hadoop.hive.ql.parse.HiveParser;
import org.apache.hadoop.hive.ql.parse.HiveTableName;
import org.apache.hadoop.hive.ql.parse.RowResolver;
@@ -76,6 +77,8 @@ public class ShowPartitionAnalyzer extends
BaseSemanticAnalyzer {
Table table = getTable(HiveTableName.of(tableName));
inputs.add(new ReadEntity(table));
+setColumnAccessInfo(new ColumnAccessInfo());
+table.getPartColNames().forEach(col ->
getColumnAccessInfo().add(table.getCompleteName(), col));
ExprNodeDesc filter = getShowPartitionsFilter(table, ast);
String orderBy = getShowPartitionsOrder(table, ast);
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestColumnAccess.java
b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestColumnAccess.java
index c4e336a..305c170 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestColumnAccess.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestColumnAccess.java
@@ -41,6 +41,7 @@ public class TestColumnAccess {
Driver driver = createDriver();
driver.run("create table t1(id1 int, name1 string)");
driver.run("create table t2(id2 int, id1 int, name2 string)");
+driver.run("create table t3(id1 int) partitioned by (`date` int, p0
string)");
driver.run("create view v1 as select * from t1");
}
@@ -49,6 +50,7 @@ public class TestColumnAccess {
Driver driver = createDriver();
driver.run("drop table t1");
driver.run("drop table t2");
+driver.run("drop table t3");
driver.run("drop view v1");
}
@@ -64,16 +66,16 @@ public class TestColumnAccess {
List cols =
columnAccessInfo.getTableToColumnAccessMap().get("default@t1");
Assert.assertNotNull(cols);
Assert.assertEquals(2, cols.size());
-Assert.assertNotNull(cols.contains("id1"));
-Assert.assertNotNull(cols.contains("name1"));
+Assert.assertTrue(cols.contains("id1"));
+Assert.assertTrue(cols.contains("name1"));
// check access columns from readEntity
Map> tableColsMap =
getColsFromReadEntity(plan.getInputs());
cols = tableColsMap.get("default@t1");
Assert.assertNotNull(cols);
Assert.assertEquals(2, cols.size());
-Assert.assertNotNull(cols.contains("id1"));
-Assert.assertNotNull(cols.contains("name1"));
+Assert.assertTrue(cols.contains("id1"));
+Assert.assertTrue(cols.contains("name1"));
}
@Test
@@ -88,14 +90,14 @@ public class TestColumnAccess {
List cols =
columnAccessInfo.getTableToColumnAccessMap().get("default@t1");
Assert.assertNotNull(cols);
Assert.assertEquals(2, cols.size());
-Assert.assertNotNull(cols.contains("id1"));
-Assert.assertNotNull(cols.contains("name1"));
+Assert.assertTrue(cols.contains("id1"));
+Assert.assertTrue(cols.contains("name1"));
cols = columnAccessInfo.getTableToColumnAccessMap().get("default@t2");
Assert.assertNotNull(cols);
Assert.assertEquals(3, cols.size());
-Assert.assertNotNull(cols.contains("id2"));
-Assert.assertNotNull(cols.contains("id1"));
-Assert.assertNotNull(cols.contains("name1"));
+Assert.assertTrue(cols.contains("id2"));
+Assert.assertTrue(cols.contains("id1"));
+Assert.assertTrue(cols.contains("name2"));
// check access columns from readEntity
@@ -103,14 +105,14 @@ public class TestColumnAccess {
cols = tableColsMap.get("default@t1");
[hive] branch master updated: HIVE-25636: Bump Xerce2 to 2.12.1. (#2740) (Takanobu Asanuma reviewed by Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository. kgyrtkirk pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git The following commit(s) were added to refs/heads/master by this push: new 24b84ea HIVE-25636: Bump Xerce2 to 2.12.1. (#2740) (Takanobu Asanuma reviewed by Zoltan Haindrich) 24b84ea is described below commit 24b84ea1ef7ce36bfabdaae2a2f6d03e71cb9c9d Author: Takanobu Asanuma AuthorDate: Mon Nov 15 17:56:05 2021 +0900 HIVE-25636: Bump Xerce2 to 2.12.1. (#2740) (Takanobu Asanuma reviewed by Zoltan Haindrich) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b4e6ca4..7a25fc1 100644 --- a/pom.xml +++ b/pom.xml @@ -211,7 +211,7 @@ 1.1.4 1.4 1.5 -2.9.1 +2.12.1 2.2.1 3.5.5 1.1
[hive] branch master updated: HIVE-25681: Drop support for multi-threaded qtest execution via QTestRunnerUtils (#2769) (Stamatis Zampetakis reviewed by Zoltan Haindrich)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 03a2aa8 HIVE-25681: Drop support for multi-threaded qtest execution
via QTestRunnerUtils (#2769) (Stamatis Zampetakis reviewed by Zoltan Haindrich)
03a2aa8 is described below
commit 03a2aa85c0cc6dc47ae90db0694a8442b376d90c
Author: Stamatis Zampetakis
AuthorDate: Mon Nov 15 09:54:49 2021 +0100
HIVE-25681: Drop support for multi-threaded qtest execution via
QTestRunnerUtils (#2769) (Stamatis Zampetakis reviewed by Zoltan Haindrich)
1. Remove QTestRunnerUtils#queryListRunnerMultiThreaded and related code
2. Remove (disabled) unit tests for this API (TestMTQueries).
The code is not being used by the current test infrastructure and this
is unlikely to change in the near future (especially after HIVE-22942).
Remove the code to facilitate code evolution and maintenance.
---
.../org/apache/hadoop/hive/ql/TestMTQueries.java | 63 ---
.../apache/hadoop/hive/ql/QTestRunnerUtils.java| 115 +
2 files changed, 2 insertions(+), 176 deletions(-)
diff --git
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
deleted file mode 100644
index f2c81fd..000
---
a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestMTQueries.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql;
-
-import java.io.File;
-
-import org.junit.Ignore;
-import org.junit.Test;
-import static org.junit.Assert.fail;
-
-/**
- * Suite for testing running of queries in multi-threaded mode.
- */
-@Ignore("Ignore until HIVE-23138 is finished")
-public class TestMTQueries extends BaseTestQueries {
-
- public TestMTQueries() {
-File logDirFile = new File(logDir);
-if (!(logDirFile.exists() || logDirFile.mkdirs())) {
- fail("Could not create " + logDir);
-}
- }
-
- @Test
- public void testMTQueries1() throws Exception {
-String[] testNames = new String[] {"join2.q", "groupby1.q", "input1.q",
"input19.q"};
-
-File[] qfiles = setupQFiles(testNames);
-QTestUtil[] qts = QTestRunnerUtils.queryListRunnerSetup(qfiles, resDir,
logDir, "q_test_init_src_with_stats.sql",
- "q_test_cleanup_src_with_stats.sql");
-for (QTestUtil util : qts) {
- util.postInit();
- // derby fails creating multiple stats aggregator concurrently
- util.getConf().setBoolean("hive.exec.submitviachild", true);
- util.getConf().setBoolean("hive.exec.submit.local.task.via.child", true);
- util.getConf().setBoolean("hive.vectorized.execution.enabled", true);
- util.getConf().set("hive.stats.dbclass", "fs");
- util.getConf().set("hive.mapred.mode", "nonstrict");
- util.getConf().set("hive.stats.column.autogather", "false");
- util.newSession();
-}
-boolean success = QTestRunnerUtils.queryListRunnerMultiThreaded(qfiles,
qts);
-if (!success) {
- fail("One or more queries failed");
-}
- }
-}
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestRunnerUtils.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestRunnerUtils.java
index ffd9952..6996ff7 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestRunnerUtils.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestRunnerUtils.java
@@ -21,66 +21,8 @@ package org.apache.hadoop.hive.ql;
import java.io.File;
import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.hive.ql.QTestMiniClusters.MiniClusterType;
public class QTestRunnerUtils {
- public static final String DEFAULT_INIT_SCRIPT = "q_test_init.sql";
- public static final String DEFAULT_CLEANUP_SCRIPT = "q_test_cleanup.sql";
-
- /**
- * QTRunner: Runnable class for running a single query file.
- **/
- public static class QTRunner implements Runnable {
-private final QTestUtil qt;
-private final File file;
-
-public
