Repository: hive Updated Branches: refs/heads/branch-1 4eb889b05 -> a59e81c31
HIVE-11250: Change in spark.executor.instances (and others) doesn't take effect after RSC is launched for HS2 (Jimmy, reviewed by Xuefu) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a59e81c3 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a59e81c3 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a59e81c3 Branch: refs/heads/branch-1 Commit: a59e81c314aa29b58f14095f6caed0f51e3835e0 Parents: 4eb889b Author: Jimmy Xiang <jxi...@cloudera.com> Authored: Tue Aug 4 11:23:34 2015 -0700 Committer: Jimmy Xiang <jxi...@cloudera.com> Committed: Thu Aug 6 09:02:40 2015 -0700 ---------------------------------------------------------------------- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 1 + .../ql/exec/spark/status/impl/LocalSparkJobStatus.java | 2 +- .../ql/exec/spark/status/impl/RemoteSparkJobStatus.java | 2 +- .../apache/hadoop/hive/ql/processors/SetProcessor.java | 4 ++++ .../org/apache/hadoop/hive/ql/session/SessionState.java | 10 +++++++--- 5 files changed, 14 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/a59e81c3/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 6f66772..95aaf55 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2721,6 +2721,7 @@ public class HiveConf extends Configuration { super(other); hiveJar = other.hiveJar; auxJars = other.auxJars; + isSparkConfigUpdated = other.isSparkConfigUpdated; origProp = (Properties)other.origProp.clone(); restrictList.addAll(other.restrictList); modWhiteListPattern = other.modWhiteListPattern; http://git-wip-us.apache.org/repos/asf/hive/blob/a59e81c3/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java index 5d62596..c6f1b8d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/LocalSparkJobStatus.java @@ -203,7 +203,7 @@ public class LocalSparkJobStatus implements SparkJobStatus { } } - results.put("EexcutorDeserializeTime", executorDeserializeTime); + results.put("ExecutorDeserializeTime", executorDeserializeTime); results.put("ExecutorRunTime", executorRunTime); results.put("ResultSize", resultSize); results.put("JvmGCTime", jvmGCTime); http://git-wip-us.apache.org/repos/asf/hive/blob/a59e81c3/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java index 9b1ef2c..072bac9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/status/impl/RemoteSparkJobStatus.java @@ -220,7 +220,7 @@ public class RemoteSparkJobStatus implements SparkJobStatus { Map<String, Long> results = new LinkedHashMap<String, Long>(); Metrics allMetrics = metricsCollection.getAllMetrics(); - results.put("EexcutorDeserializeTime", allMetrics.executorDeserializeTime); + results.put("ExecutorDeserializeTime", allMetrics.executorDeserializeTime); results.put("ExecutorRunTime", allMetrics.executorRunTime); results.put("ResultSize", allMetrics.resultSize); results.put("JvmGCTime", allMetrics.jvmGCTime); http://git-wip-us.apache.org/repos/asf/hive/blob/a59e81c3/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java index d271d6d..2414e12 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java @@ -168,6 +168,10 @@ public class SetProcessor implements CommandProcessor { } } conf.verifyAndSet(key, value); + if (HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname.equals(key) + && !"spark".equals(value)) { + SessionState.get().closeSparkSession(); + } if (register) { SessionState.get().getOverriddenConfigurations().put(key, value); } http://git-wip-us.apache.org/repos/asf/hive/blob/a59e81c3/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 9caf27e..383ba6a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -754,7 +754,7 @@ public class SessionState { getAuthorizer() : getAuthorizerV2(); } - public Class getAuthorizerInterface() { + public Class<?> getAuthorizerInterface() { return getAuthorizationMode() == AuthorizationMode.V1 ? HiveAuthorizationProvider.class : HiveAuthorizer.class; } @@ -1482,6 +1482,12 @@ public class SessionState { tezSessionState = null; } + closeSparkSession(); + registry.closeCUDFLoaders(); + dropSessionPaths(conf); + } + + public void closeSparkSession() { if (sparkSession != null) { try { SparkSessionManagerImpl.getInstance().closeSession(sparkSession); @@ -1491,8 +1497,6 @@ public class SessionState { sparkSession = null; } } - registry.closeCUDFLoaders(); - dropSessionPaths(conf); } public AuthorizationMode getAuthorizationMode(){
