Repository: spark Updated Branches: refs/heads/master aee2bd2c7 -> 8fd178d21
[SPARK-19260] Spaces or "%20" in path parameter are not correctly handled with⦠JIRA Issue: https://issues.apache.org/jira/browse/SPARK-19260 ## What changes were proposed in this pull request? 1. âspark.history.fs.logDirectoryâ supports with space character and â%20â characters. 2. As usually, if the run classpath includes hdfs-site.xml and core-site.xml files, the supplied path eg."/test" which does not contain a scheme should be taken as a HDFS path rather than a local path since the path parameter is a Hadoop dir. ## How was this patch tested? Update Unit Test and take some manual tests local: .sbin/start-history-server.sh "file:/a b" .sbin/start-history-server.sh "/abc%20c" (without hdfs-site.xml,core-site.xml) .sbin/start-history-server.sh "/a b" (without hdfs-site.xml,core-site.xml) .sbin/start-history-server.sh "/a b/a bc%20c" (without hdfs-site.xml,core-site.xml) hdfs: .sbin/start-history-server.sh "hdfs:/namenode:9000/a b" .sbin/start-history-server.sh "/a b" (with hdfs-site.xml,core-site.xml) .sbin/start-history-server.sh "/a b/a bc%20c" (with hdfs-site.xml,core-site.xml) Author: zuotingbing <zuo.tingbi...@zte.com.cn> Closes #16614 from zuotingbing/SPARK-19260. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8fd178d2 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8fd178d2 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8fd178d2 Branch: refs/heads/master Commit: 8fd178d2151da53c0edc7ed3a92ebd01780d7702 Parents: aee2bd2 Author: zuotingbing <zuo.tingbi...@zte.com.cn> Authored: Tue Feb 7 12:21:36 2017 +0000 Committer: Sean Owen <so...@cloudera.com> Committed: Tue Feb 7 12:21:36 2017 +0000 ---------------------------------------------------------------------- .../scala/org/apache/spark/deploy/history/FsHistoryProvider.scala | 3 +-- .../org/apache/spark/deploy/history/FsHistoryProviderSuite.scala | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/8fd178d2/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala index cd241d6..9012736 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala @@ -94,7 +94,6 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock) Math.ceil(Runtime.getRuntime.availableProcessors() / 4f).toInt) private val logDir = conf.getOption("spark.history.fs.logDirectory") - .map { d => Utils.resolveURI(d).toString } .getOrElse(DEFAULT_LOG_DIR) private val HISTORY_UI_ACLS_ENABLE = conf.getBoolean("spark.history.ui.acls.enable", false) @@ -105,7 +104,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock) "; groups with admin permissions" + HISTORY_UI_ADMIN_ACLS_GROUPS.toString) private val hadoopConf = SparkHadoopUtil.get.newConfiguration(conf) - private val fs = Utils.getHadoopFileSystem(logDir, hadoopConf) + private val fs = new Path(logDir).getFileSystem(hadoopConf) // Used by check event thread and clean log thread. // Scheduled thread pool size must be one, otherwise it will have concurrent issues about fs http://git-wip-us.apache.org/repos/asf/spark/blob/8fd178d2/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala index 8cb359e..ec580a4 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala @@ -47,7 +47,7 @@ class FsHistoryProviderSuite extends SparkFunSuite with BeforeAndAfter with Matc private var testDir: File = null before { - testDir = Utils.createTempDir() + testDir = Utils.createTempDir(namePrefix = s"a b%20c+d") } after { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
