Repository: hive Updated Branches: refs/heads/master a0fdf00fe -> 8398fbf3d
HIVE-11250: Change in spark.executor.instances (and others) doesn't take effect after RSC is launched for HS2 (Jimmy, reviewed by Xuefu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8398fbf3 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8398fbf3 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8398fbf3 Branch: refs/heads/master Commit: 8398fbf3dd0937a0a4a3d540977a95f97425f566 Parents: a0fdf00 Author: Jimmy Xiang <jxi...@cloudera.com> Authored: Tue Aug 4 11:23:34 2015 -0700 Committer: Jimmy Xiang <jxi...@cloudera.com> Committed: Thu Aug 6 09:00:09 2015 -0700 ---------------------------------------------------------------------- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 1 + .../ql/exec/spark/status/impl/LocalSparkJobStatus.java | 2 +- .../ql/exec/spark/status/impl/RemoteSparkJobStatus.java | 2 +- .../apache/hadoop/hive/ql/processors/SetProcessor.java | 4 ++++ .../org/apache/hadoop/hive/ql/session/SessionState.java | 10 +++++++--- 5 files changed, 14 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/8398fbf3/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index fe4eb9f..36bb394 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2723,6 +2723,7 @@ public class HiveConf extends Configuration { super(other); hiveJar = other.hiveJar; auxJars = other.auxJars; + isSparkConfigUpdated = other.isSparkConfigUpdated; origProp = (Properties)other.origProp.clone(); restrictList.addAll(other.restrictList); modWhiteListPattern = other.modWhiteListPattern; http://git-wip-us.apache.org/repos/asf/hive/blob/8398fbf3/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java index 5d62596..c6f1b8d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java @@ -203,7 +203,7 @@ public class LocalSparkJobStatus implements SparkJobStatus { } } - results.put("EexcutorDeserializeTime", executorDeserializeTime); + results.put("ExecutorDeserializeTime", executorDeserializeTime); results.put("ExecutorRunTime", executorRunTime); results.put("ResultSize", resultSize); results.put("JvmGCTime", jvmGCTime); http://git-wip-us.apache.org/repos/asf/hive/blob/8398fbf3/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java index 9b1ef2c..072bac9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java @@ -220,7 +220,7 @@ public class RemoteSparkJobStatus implements SparkJobStatus { Map<String, Long> results = new LinkedHashMap<String, Long>(); Metrics allMetrics = metricsCollection.getAllMetrics(); - results.put("EexcutorDeserializeTime", allMetrics.executorDeserializeTime); + results.put("ExecutorDeserializeTime", allMetrics.executorDeserializeTime); results.put("ExecutorRunTime", allMetrics.executorRunTime); results.put("ResultSize", allMetrics.resultSize); results.put("JvmGCTime", allMetrics.jvmGCTime); http://git-wip-us.apache.org/repos/asf/hive/blob/8398fbf3/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java index d271d6d..2414e12 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java @@ -168,6 +168,10 @@ public class SetProcessor implements CommandProcessor { } } conf.verifyAndSet(key, value); + if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key) + && !"spark".equals(value)) { + SessionState.get().closeSparkSession(); + } if (register) { SessionState.get().getOverriddenConfigurations().put(key, value); } http://git-wip-us.apache.org/repos/asf/hive/blob/8398fbf3/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 510d8a7..7ed8e5f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -762,7 +762,7 @@ public class SessionState { getAuthorizer() : getAuthorizerV2(); } - public Class getAuthorizerInterface() { + public Class<?> getAuthorizerInterface() { return getAuthorizationMode() == AuthorizationMode.V1 ? HiveAuthorizationProvider.class : HiveAuthorizer.class; } @@ -1490,6 +1490,12 @@ public class SessionState { tezSessionState = null; } + closeSparkSession(); + registry.closeCUDFLoaders(); + dropSessionPaths(conf); + } + + public void closeSparkSession() { if (sparkSession != null) { try { SparkSessionManagerImpl.getInstance().closeSession(sparkSession); @@ -1499,8 +1505,6 @@ public class SessionState { sparkSession = null; } } - registry.closeCUDFLoaders(); - dropSessionPaths(conf); } public AuthorizationMode getAuthorizationMode(){