Repository: spark
Updated Branches:
  refs/heads/master aee2bd2c7 -> 8fd178d21


[SPARK-19260] Spaces or "%20" in path parameter are not correctly handled 
with…

JIRA Issue: https://issues.apache.org/jira/browse/SPARK-19260

## What changes were proposed in this pull request?

1. “spark.history.fs.logDirectory” supports with space character and 
“%20” characters.
2. As usually, if the run classpath includes hdfs-site.xml and core-site.xml 
files, the supplied path eg."/test" which does not contain a scheme should be 
taken as a HDFS path rather than a local path since the path parameter is a 
Hadoop dir.

## How was this patch tested?
Update Unit Test and take some manual tests

local:
.sbin/start-history-server.sh "file:/a b"
.sbin/start-history-server.sh "/abc%20c" (without hdfs-site.xml,core-site.xml)
.sbin/start-history-server.sh "/a b" (without hdfs-site.xml,core-site.xml)
.sbin/start-history-server.sh "/a b/a bc%20c" (without 
hdfs-site.xml,core-site.xml)

hdfs:
.sbin/start-history-server.sh "hdfs:/namenode:9000/a b"
.sbin/start-history-server.sh "/a b" (with hdfs-site.xml,core-site.xml)
.sbin/start-history-server.sh "/a b/a bc%20c" (with hdfs-site.xml,core-site.xml)

Author: zuotingbing <zuo.tingbi...@zte.com.cn>

Closes #16614 from zuotingbing/SPARK-19260.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8fd178d2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8fd178d2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8fd178d2

Branch: refs/heads/master
Commit: 8fd178d2151da53c0edc7ed3a92ebd01780d7702
Parents: aee2bd2
Author: zuotingbing <zuo.tingbi...@zte.com.cn>
Authored: Tue Feb 7 12:21:36 2017 +0000
Committer: Sean Owen <so...@cloudera.com>
Committed: Tue Feb 7 12:21:36 2017 +0000

----------------------------------------------------------------------
 .../scala/org/apache/spark/deploy/history/FsHistoryProvider.scala | 3 +--
 .../org/apache/spark/deploy/history/FsHistoryProviderSuite.scala  | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/8fd178d2/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
----------------------------------------------------------------------
diff --git 
a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala 
b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index cd241d6..9012736 100644
--- 
a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ 
b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -94,7 +94,6 @@ private[history] class FsHistoryProvider(conf: SparkConf, 
clock: Clock)
     Math.ceil(Runtime.getRuntime.availableProcessors() / 4f).toInt)
 
   private val logDir = conf.getOption("spark.history.fs.logDirectory")
-    .map { d => Utils.resolveURI(d).toString }
     .getOrElse(DEFAULT_LOG_DIR)
 
   private val HISTORY_UI_ACLS_ENABLE = 
conf.getBoolean("spark.history.ui.acls.enable", false)
@@ -105,7 +104,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, 
clock: Clock)
     "; groups with admin permissions" + HISTORY_UI_ADMIN_ACLS_GROUPS.toString)
 
   private val hadoopConf = SparkHadoopUtil.get.newConfiguration(conf)
-  private val fs = Utils.getHadoopFileSystem(logDir, hadoopConf)
+  private val fs = new Path(logDir).getFileSystem(hadoopConf)
 
   // Used by check event thread and clean log thread.
   // Scheduled thread pool size must be one, otherwise it will have concurrent 
issues about fs

http://git-wip-us.apache.org/repos/asf/spark/blob/8fd178d2/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
----------------------------------------------------------------------
diff --git 
a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
 
b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
index 8cb359e..ec580a4 100644
--- 
a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
+++ 
b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
@@ -47,7 +47,7 @@ class FsHistoryProviderSuite extends SparkFunSuite with 
BeforeAndAfter with Matc
   private var testDir: File = null
 
   before {
-    testDir = Utils.createTempDir()
+    testDir = Utils.createTempDir(namePrefix = s"a b%20c+d")
   }
 
   after {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org

Reply via email to