[hive] branch master updated (247d8aa -> 1aa6ce8)
This is an automated email from the ASF dual-hosted git repository.
szita pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git.
from 247d8aa HIVE-25874: Slow filter evaluation of nest struct fields in
vectorized executions (#2952) (Zoltan Haindrich reviewed by Krisztian Kasa)
add 1aa6ce8 HIVE-25967: Prevent residual expressions from getting
serialized in Iceberg splits (#3041) (Adam Szita, reviewed by Peter Vary)
No new revisions were added by this update.
Summary of changes:
.../apache/iceberg/mr/hive/HiveIcebergSplit.java | 42 +++
.../iceberg/mr/TestInputFormatReaderDeletes.java | 1 +
.../mr/{ => hive}/TestIcebergInputFormats.java | 47 +-
3 files changed, 88 insertions(+), 2 deletions(-)
rename iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/{ =>
hive}/TestIcebergInputFormats.java (92%)
[hive] branch master updated (756a8fc -> 247d8aa)
This is an automated email from the ASF dual-hosted git repository. kgyrtkirk pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hive.git. from 756a8fc HIVE-25960: Fix S3a recursive listing logic. (#3031). (Ayush Saxena reviewed by Laszlo Bodor) add 247d8aa HIVE-25874: Slow filter evaluation of nest struct fields in vectorized executions (#2952) (Zoltan Haindrich reviewed by Krisztian Kasa) No new revisions were added by this update. Summary of changes: .../vector/expressions/VectorUDFStructField.java | 3 +- .../clientpositive/vectorization_nested_struct.q | 21 ++ .../llap/vectorization_nested_struct.q.out | 81 ++ 3 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 ql/src/test/queries/clientpositive/vectorization_nested_struct.q create mode 100644 ql/src/test/results/clientpositive/llap/vectorization_nested_struct.q.out
[hive] branch branch-3 updated: HIVE-25844: Exception deserialization error-s may cause beeline to terminate immediately (backport HIVE-24772) (#2918) (Zoltan Haindrich reviewed by Krisztian Kasa)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new 1087c4e HIVE-25844: Exception deserialization error-s may cause
beeline to terminate immediately (backport HIVE-24772) (#2918) (Zoltan
Haindrich reviewed by Krisztian Kasa)
1087c4e is described below
commit 1087c4e14a2668eab914939f1f92dae2f049cf7e
Author: Zoltan Haindrich
AuthorDate: Tue Feb 22 15:29:52 2022 +0100
HIVE-25844: Exception deserialization error-s may cause beeline to
terminate immediately (backport HIVE-24772) (#2918) (Zoltan Haindrich reviewed
by Krisztian Kasa)
---
.../apache/hive/minikdc/TestJdbcWithMiniKdc.java | 28 +---
.../apache/hive/service/cli/HiveSQLException.java | 167 +
.../hive/service/cli/thrift/ThriftCLIService.java | 67 +
.../hive/service/cli/TestHiveSQLException.java | 112 +-
4 files changed, 79 insertions(+), 295 deletions(-)
diff --git
a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java
b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java
index e526b48..353d157 100644
---
a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java
+++
b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/TestJdbcWithMiniKdc.java
@@ -221,22 +221,15 @@ public class TestJdbcWithMiniKdc {
* it's not allowed to impersonate
* @throws Exception
*/
- @Test
+ @Test(expected = HiveSQLException.class)
public void testNegativeTokenAuth() throws Exception {
miniHiveKdc.loginUser(MiniHiveKdc.HIVE_TEST_SUPER_USER);
hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL());
try {
// retrieve token and store in the cache
- String token = ((HiveConnection)hs2Conn).getDelegationToken(
+ ((HiveConnection)hs2Conn).getDelegationToken(
MiniHiveKdc.HIVE_TEST_USER_2, MiniHiveKdc.HIVE_SERVICE_PRINCIPAL);
-
- fail(MiniHiveKdc.HIVE_TEST_SUPER_USER + " shouldn't be allowed to
retrieve token for " +
- MiniHiveKdc.HIVE_TEST_USER_2);
-} catch (SQLException e) {
- // Expected error
- assertEquals("Unexpected type of exception class thrown",
HiveSQLException.class, e.getClass());
- assertTrue(e.getCause().getCause().getMessage().contains("is not allowed
to impersonate"));
} finally {
hs2Conn.close();
}
@@ -260,22 +253,11 @@ public class TestJdbcWithMiniKdc {
* impersonate the given user
* @throws Exception
*/
- @Test
+ @Test(expected = SQLException.class)
public void testNegativeProxyAuth() throws Exception {
miniHiveKdc.loginUser(MiniHiveKdc.HIVE_TEST_SUPER_USER);
-try {
- hs2Conn = DriverManager.getConnection(miniHS2.getJdbcURL("default",
- ";hive.server2.proxy.user=" + MiniHiveKdc.HIVE_TEST_USER_2));
- verifyProperty(SESSION_USER_NAME, MiniHiveKdc.HIVE_TEST_USER_2);
- fail(MiniHiveKdc.HIVE_TEST_SUPER_USER + " shouldn't be allowed proxy
connection for "
- + MiniHiveKdc.HIVE_TEST_USER_2);
-} catch (SQLException e) {
- // Expected error
- e.printStackTrace();
- assertTrue(e.getMessage().contains("Failed to validate proxy
privilege"));
- assertTrue(e.getCause().getCause().getCause().getMessage()
- .contains("is not allowed to impersonate"));
-}
+hs2Conn = DriverManager
+.getConnection(miniHS2.getJdbcURL("default",
";hive.server2.proxy.user=" + MiniHiveKdc.HIVE_TEST_USER_2));
}
/**
diff --git a/service/src/java/org/apache/hive/service/cli/HiveSQLException.java
b/service/src/java/org/apache/hive/service/cli/HiveSQLException.java
index 5f9ff43..f180830 100644
--- a/service/src/java/org/apache/hive/service/cli/HiveSQLException.java
+++ b/service/src/java/org/apache/hive/service/cli/HiveSQLException.java
@@ -19,39 +19,52 @@
package org.apache.hive.service.cli;
import java.sql.SQLException;
-import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import org.apache.hive.service.rpc.thrift.TStatus;
import org.apache.hive.service.rpc.thrift.TStatusCode;
+import com.google.common.annotations.VisibleForTesting;
+
/**
- * HiveSQLException.
- *
+ * An exception that provides information on a Hive access error or other
+ * errors.
*/
public class HiveSQLException extends SQLException {
- /**
- *
- */
- private static final long serialVersionUID = -6095254671958748094L;
+ private static final long serialVersionUID = -6095254671958748095L;
+
+ @VisibleForTesting
+ public static final List DEFAULT_INFO =
+ Collections.singletonList("Server-side error; please check HS2 logs.");
/**
- *
+ * Constructor.
*/
public HiveSQLException() {
super();
}
/**
- * @param reason
+ * Constructs a SQLException object
[hive] branch master updated: HIVE-25960: Fix S3a recursive listing logic. (#3031). (Ayush Saxena reviewed by Laszlo Bodor)
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 756a8fc HIVE-25960: Fix S3a recursive listing logic. (#3031). (Ayush
Saxena reviewed by Laszlo Bodor)
756a8fc is described below
commit 756a8fce9f75e139fd7c5cdc45ff0eb629ad9504
Author: Ayush Saxena
AuthorDate: Tue Feb 22 19:44:02 2022 +0530
HIVE-25960: Fix S3a recursive listing logic. (#3031). (Ayush Saxena
reviewed by Laszlo Bodor)
---
.../java/org/apache/hadoop/hive/common/FileUtils.java | 16 +++-
.../org/apache/hadoop/hive/common/TestFileUtils.java| 17 +
.../apache/hadoop/hive/metastore/utils/FileUtils.java | 16 +++-
3 files changed, 47 insertions(+), 2 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
index b617623..e92b700 100644
--- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
@@ -361,13 +361,27 @@ public final class FileUtils {
RemoteIterator remoteIterator =
fs.listFiles(base.getPath(), true);
while (remoteIterator.hasNext()) {
LocatedFileStatus each = remoteIterator.next();
- Path relativePath = new
Path(each.getPath().toString().replace(base.toString(), ""));
+ Path relativePath = makeRelative(base.getPath(), each.getPath());
if
(org.apache.hadoop.hive.metastore.utils.FileUtils.RemoteIteratorWithFilter.HIDDEN_FILES_FULL_PATH_FILTER.accept(relativePath))
{
results.add(each);
}
}
}
+ /**
+ * Returns a relative path wrt the parent path.
+ * @param parentPath the parent path.
+ * @param childPath the child path.
+ * @return childPath relative to parent path.
+ */
+ public static Path makeRelative(Path parentPath, Path childPath) {
+String parentString =
+parentPath.toString().endsWith(Path.SEPARATOR) ? parentPath.toString()
: parentPath.toString() + Path.SEPARATOR;
+String childString =
+childPath.toString().endsWith(Path.SEPARATOR) ? childPath.toString() :
childPath.toString() + Path.SEPARATOR;
+return new Path(childString.replaceFirst(parentString, ""));
+ }
+
public static boolean isS3a(FileSystem fs) {
try {
return "s3a".equalsIgnoreCase(fs.getScheme());
diff --git a/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
b/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
index 15e74db..4f4d604 100644
--- a/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
+++ b/common/src/test/org/apache/hadoop/hive/common/TestFileUtils.java
@@ -264,4 +264,21 @@ public class TestFileUtils {
equalsIgnoreCase("Distcp is called with doAsUser and delete
source set as true"));
}
}
+
+ @Test
+ public void testMakeRelative() {
+Path parentPath = new Path("/user/hive/database");
+Path childPath = new Path(parentPath, "table/dir/subdir");
+Path relativePath = FileUtils.makeRelative(parentPath, childPath);
+assertEquals("table/dir/subdir", relativePath.toString());
+
+// try with parent as Root.
+relativePath = FileUtils.makeRelative(new Path(Path.SEPARATOR), childPath);
+assertEquals("user/hive/database/table/dir/subdir",
relativePath.toString());
+
+// try with non child path, it should return the child path as is.
+childPath = new Path("/user/hive/database1/table/dir/subdir");
+relativePath = FileUtils.makeRelative(parentPath, childPath);
+assertEquals(childPath.toString(), relativePath.toString());
+ }
}
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
index dad71b7..b8ab28b 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/FileUtils.java
@@ -404,13 +404,27 @@ public class FileUtils {
RemoteIterator remoteIterator = fs.listFiles(base,
true);
while (remoteIterator.hasNext()) {
LocatedFileStatus each = remoteIterator.next();
- Path relativePath = new
Path(each.getPath().toString().replace(base.toString(), ""));
+ Path relativePath = makeRelative(base, each.getPath());
if
(RemoteIteratorWithFilter.HIDDEN_FILES_FULL_PATH_FILTER.accept(relativePath)) {
results.add(each);
}
}
}
+ /**
+ * Returns a relative path wrt the parent path.
+ * @param parentPath the parent path.
+ * @param childPath the child path.
+ * @return childPath relative to parent path.
+ */
+ public static Path
[hive] branch master updated: HIVE-25938: Print excluded rules from CBO (Alessandro Solimando, reviewed by Stamatis Zampetakis, John Sherman)
This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 3c532c2 HIVE-25938: Print excluded rules from CBO (Alessandro
Solimando, reviewed by Stamatis Zampetakis, John Sherman)
3c532c2 is described below
commit 3c532c2cd2603edc60c721282d45390e910a0358
Author: Alessandro Solimando
AuthorDate: Tue Feb 8 17:09:19 2022 +0100
HIVE-25938: Print excluded rules from CBO (Alessandro Solimando, reviewed
by Stamatis Zampetakis, John Sherman)
Closes #3011
---
.../apache/hadoop/hive/ql/exec/ExplainTask.java| 39 +--
.../hadoop/hive/ql/parse/CalcitePlanner.java | 24 -
.../queries/clientpositive/excluded_rule_explain.q | 11 ++
.../llap/excluded_rule_explain.q.out | 112 +
.../llap/rule_exclusion_config.q.out | 8 ++
5 files changed, 184 insertions(+), 10 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
index c59f44f..59f9044 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExplainTask.java
@@ -101,8 +101,12 @@ public class ExplainTask extends Task
implements Serializable {
private static final Logger LOG =
LoggerFactory.getLogger(ExplainTask.class.getName());
public static final String STAGE_DEPENDENCIES = "STAGE DEPENDENCIES";
+ private static final String EXCLUDED_RULES_PREFIX = "Excluded rules: ";
private static final long serialVersionUID = 1L;
public static final String EXPL_COLUMN_NAME = "Explain";
+ private static final String CBO_INFO_JSON_LABEL = "cboInfo";
+ private static final String CBO_PLAN_JSON_LABEL = "CBOPlan";
+ private static final String CBO_PLAN_TEXT_LABEL = "CBO PLAN:";
private final Set> visitedOps = new HashSet>();
private boolean isLogical = false;
@@ -152,15 +156,22 @@ public class ExplainTask extends Task
implements Serializable {
return outJSONObject;
}
- public JSONObject getJSONCBOPlan(PrintStream out, ExplainWork work) throws
Exception {
+ public JSONObject getJSONCBOPlan(PrintStream out, ExplainWork work) {
JSONObject outJSONObject = new JSONObject(new LinkedHashMap<>());
boolean jsonOutput = work.isFormatted();
String cboPlan = work.getCboPlan();
if (cboPlan != null) {
+ String ruleExclusionRegex = getRuleExcludedRegex();
if (jsonOutput) {
-outJSONObject.put("CBOPlan", cboPlan);
+outJSONObject.put(CBO_PLAN_JSON_LABEL, cboPlan);
+if (!ruleExclusionRegex.isEmpty()) {
+ outJSONObject.put(CBO_INFO_JSON_LABEL, EXCLUDED_RULES_PREFIX +
ruleExclusionRegex);
+}
} else {
-out.println("CBO PLAN:");
+if (!ruleExclusionRegex.isEmpty()) {
+ out.println(EXCLUDED_RULES_PREFIX + ruleExclusionRegex + "\n");
+}
+out.println(CBO_PLAN_TEXT_LABEL);
out.println(cboPlan);
}
}
@@ -272,6 +283,8 @@ public class ExplainTask extends Task
implements Serializable {
boolean jsonOutput, boolean isExtended, boolean appendTaskType, String
cboInfo,
String cboPlan, String optimizedSQL, String stageIdRearrange) throws
Exception {
+String ruleExclusionRegex = getRuleExcludedRegex();
+
// If the user asked for a formatted output, dump the json output
// in the output stream
JSONObject outJSONObject = new JSONObject(new LinkedHashMap<>());
@@ -282,9 +295,15 @@ public class ExplainTask extends Task
implements Serializable {
if (cboPlan != null) {
if (jsonOutput) {
-outJSONObject.put("CBOPlan", cboPlan);
+outJSONObject.put(CBO_PLAN_JSON_LABEL, cboPlan);
+if (!ruleExclusionRegex.isEmpty()) {
+ outJSONObject.put(CBO_INFO_JSON_LABEL, EXCLUDED_RULES_PREFIX +
ruleExclusionRegex);
+}
} else {
-out.print("CBO PLAN:");
+if (!ruleExclusionRegex.isEmpty()) {
+ out.println(EXCLUDED_RULES_PREFIX + ruleExclusionRegex);
+}
+out.print(CBO_PLAN_TEXT_LABEL);
out.println(cboPlan);
}
}
@@ -327,6 +346,10 @@ public class ExplainTask extends Task
implements Serializable {
}
if (!suppressOthersForVectorization) {
+ if (!jsonOutput && !ruleExclusionRegex.isEmpty()) {
+out.println(EXCLUDED_RULES_PREFIX + ruleExclusionRegex + "\n");
+ }
+
JSONObject jsonDependencies = outputDependencies(out, jsonOutput,
appendTaskType, ordered);
if (out != null) {
@@ -335,7 +358,7 @@ public class ExplainTask extends Task
implements Serializable {
if (jsonOutput) {
if (cboInfo != null) {
- outJSONObject.put("cboInfo", cboInfo);
+ outJSONObject.put(CBO_INFO_JSON_LABEL, cboInfo);
}
[hive] branch master updated (f15de94 -> bf09d78)
This is an automated email from the ASF dual-hosted git repository.
klcopp pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git.
from f15de94 HIVE-25918: Invalid stats after multi inserting into the same
partition (Krisztian Kasa, reviewed by Zoltan Haindrich)
add bf09d78 HIVE-25959: Expose Compaction Observability delta metrics
using the JsonReporter (Viktor Csomor, reviewed by Karen Coppage)
No new revisions were added by this update.
Summary of changes:
.../hive/ql/txn/compactor/CompactorTest.java | 12 +-
.../txn/compactor/TestDeltaFilesMetricFlags.java | 30 ++-
.../ql/txn/compactor/TestDeltaFilesMetrics.java| 229 +
.../common/metrics/metrics2/CodahaleMetrics.java | 16 +-
.../metrics/metrics2/JsonFileMetricsReporter.java | 8 +-
.../hive/metastore/metrics/AcidMetricService.java | 18 +-
.../hive/metastore/metrics/JsonReporter.java | 16 +-
.../metrics/MapCapableJsonMetricsModule.java | 150 ++
.../hadoop/hive/metastore/metrics/MapMetrics.java | 33 +--
.../hadoop/hive/metastore/metrics/Metrics.java | 51 -
.../TestMetricRegistry.java} | 44 ++--
.../hadoop/hive/metastore/metrics/TestMetrics.java | 26 ++-
12 files changed, 463 insertions(+), 170 deletions(-)
create mode 100644
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/MapCapableJsonMetricsModule.java
copy ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedSupport.java
=>
standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/metrics/MapMetrics.java
(61%)
copy
standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/{TestEmbeddedHiveMetaStore.java
=> metrics/TestMetricRegistry.java} (54%)
