Repository: spark Updated Branches: refs/heads/master 34fc48fb5 -> 1f0de3c1c
[SPARK-19991][CORE][YARN] FileSegmentManagedBuffer performance improvement ## What changes were proposed in this pull request? Avoid `NoSuchElementException` every time `ConfigProvider.get(val, default)` falls back to default. This apparently causes non-trivial overhead in at least one path, and can easily be avoided. See https://github.com/apache/spark/pull/17329 ## How was this patch tested? Existing tests Author: Sean Owen <so...@cloudera.com> Closes #17567 from srowen/SPARK-19991. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1f0de3c1 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1f0de3c1 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1f0de3c1 Branch: refs/heads/master Commit: 1f0de3c1c85a41eadc7c4131bdc948405f340099 Parents: 34fc48f Author: Sean Owen <so...@cloudera.com> Authored: Sun Apr 9 08:44:02 2017 +0100 Committer: Sean Owen <so...@cloudera.com> Committed: Sun Apr 9 08:44:02 2017 +0100 ---------------------------------------------------------------------- .../java/org/apache/spark/network/util/MapConfigProvider.java | 6 ++++++ .../apache/spark/network/yarn/util/HadoopConfigProvider.java | 6 ++++++ .../org/apache/spark/network/netty/SparkTransportConf.scala | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/1f0de3c1/common/network-common/src/main/java/org/apache/spark/network/util/MapConfigProvider.java ---------------------------------------------------------------------- diff --git a/common/network-common/src/main/java/org/apache/spark/network/util/MapConfigProvider.java b/common/network-common/src/main/java/org/apache/spark/network/util/MapConfigProvider.java index 9cfee7f..a2cf87d 100644 --- a/common/network-common/src/main/java/org/apache/spark/network/util/MapConfigProvider.java +++ b/common/network-common/src/main/java/org/apache/spark/network/util/MapConfigProvider.java @@ -43,6 +43,12 @@ public class MapConfigProvider extends ConfigProvider { } @Override + public String get(String name, String defaultValue) { + String value = config.get(name); + return value == null ? defaultValue : value; + } + + @Override public Iterable<Map.Entry<String, String>> getAll() { return config.entrySet(); } http://git-wip-us.apache.org/repos/asf/spark/blob/1f0de3c1/common/network-yarn/src/main/java/org/apache/spark/network/yarn/util/HadoopConfigProvider.java ---------------------------------------------------------------------- diff --git a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/util/HadoopConfigProvider.java b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/util/HadoopConfigProvider.java index 62a6cca..8beb033 100644 --- a/common/network-yarn/src/main/java/org/apache/spark/network/yarn/util/HadoopConfigProvider.java +++ b/common/network-yarn/src/main/java/org/apache/spark/network/yarn/util/HadoopConfigProvider.java @@ -42,6 +42,12 @@ public class HadoopConfigProvider extends ConfigProvider { } @Override + public String get(String name, String defaultValue) { + String value = conf.get(name); + return value == null ? defaultValue : value; + } + + @Override public Iterable<Map.Entry<String, String>> getAll() { return conf; } http://git-wip-us.apache.org/repos/asf/spark/blob/1f0de3c1/core/src/main/scala/org/apache/spark/network/netty/SparkTransportConf.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/network/netty/SparkTransportConf.scala b/core/src/main/scala/org/apache/spark/network/netty/SparkTransportConf.scala index df520f8..25f7bcb 100644 --- a/core/src/main/scala/org/apache/spark/network/netty/SparkTransportConf.scala +++ b/core/src/main/scala/org/apache/spark/network/netty/SparkTransportConf.scala @@ -60,7 +60,7 @@ object SparkTransportConf { new TransportConf(module, new ConfigProvider { override def get(name: String): String = conf.get(name) - + override def get(name: String, defaultValue: String): String = conf.get(name, defaultValue) override def getAll(): java.lang.Iterable[java.util.Map.Entry[String, String]] = { conf.getAll.toMap.asJava.entrySet() } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org