Repository: spark
Updated Branches:
refs/heads/branch-2.0 8e26b74fc -> 4f66bf5fb
[SPARK-8603][SPARKR] Use shell() instead of system2() for SparkR on Windows
## What changes were proposed in this pull request?
This PR corrects SparkR to use `shell()` instead of `system2()` on Windows.
Using `system2(...)` on Windows does not process windows file separator `\`.
`shell(tralsate = TRUE, ...)` can treat this problem. So, this was changed to
be chosen according to OS.
Existing tests were failed on Windows due to this problem. For example, those
were failed.
```
8. Failure: sparkJars tag in SparkContext (test_includeJAR.R#34)
9. Failure: sparkJars tag in SparkContext (test_includeJAR.R#36)
```
The cases above were due to using of `system2`.
In addition, this PR also fixes some tests failed on Windows.
```
5. Failure: sparkJars sparkPackages as comma-separated strings
(test_context.R#128)
6. Failure: sparkJars sparkPackages as comma-separated strings
(test_context.R#131)
7. Failure: sparkJars sparkPackages as comma-separated strings
(test_context.R#134)
```
The cases above were due to a weird behaviour of `normalizePath()`. On Linux,
if the path does not exist, it just prints out the input but it prints out
including the current path on Windows.
```r
# On Linus
path <- normalizePath("aa")
print(path)
[1] "aa"
# On Windows
path <- normalizePath("aa")
print(path)
[1] "C:\\Users\\aa"
```
## How was this patch tested?
Jenkins tests and manually tested in a Window machine as below:
Here is the
[stdout](https://gist.github.com/HyukjinKwon/4bf35184f3a30f3bce987a58ec2bbbab)
of testing.
Closes #7025
Author: hyukjinkwon <gurwls...@gmail.com>
Author: Hyukjin Kwon <gurwls...@gmail.com>
Author: Prakash PC <prakash.chi...@gmail.com>
Closes #13165 from HyukjinKwon/pr/7025.
(cherry picked from commit 1c403733b89258e57daf7b8b0a2011981ad7ed8a)
Signed-off-by: Shivaram Venkataraman <shiva...@cs.berkeley.edu>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4f66bf5f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4f66bf5f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4f66bf5f
Branch: refs/heads/branch-2.0
Commit: 4f66bf5fba6befdb49ef2f8e5e3037cc3e601508
Parents: 8e26b74
Author: hyukjinkwon <gurwls...@gmail.com>
Authored: Thu May 26 20:55:06 2016 -0700
Committer: Shivaram Venkataraman <shiva...@cs.berkeley.edu>
Committed: Thu May 26 20:55:13 2016 -0700
----------------------------------------------------------------------
R/WINDOWS.md | 2 +-
R/pkg/R/client.R | 4 ++--
R/pkg/R/utils.R | 9 ++++++++
R/pkg/inst/tests/testthat/test_Windows.R | 26 ++++++++++++++++++++++++
R/pkg/inst/tests/testthat/test_context.R | 6 +++---
R/pkg/inst/tests/testthat/test_includeJAR.R | 7 +++----
6 files changed, 44 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/4f66bf5f/R/WINDOWS.md
----------------------------------------------------------------------
diff --git a/R/WINDOWS.md b/R/WINDOWS.md
index f948ed3..f67a1c5 100644
--- a/R/WINDOWS.md
+++ b/R/WINDOWS.md
@@ -28,6 +28,6 @@ To run the SparkR unit tests on Windows, the following steps
are required âass
```
R -e "install.packages('testthat', repos='http://cran.us.r-project.org')"
- .\bin\spark-submit2.cmd --conf spark.hadoop.fs.defualt.name="file:///"
R\pkg\tests\run-all.R
+ .\bin\spark-submit2.cmd --conf spark.hadoop.fs.default.name="file:///"
R\pkg\tests\run-all.R
```
http://git-wip-us.apache.org/repos/asf/spark/blob/4f66bf5f/R/pkg/R/client.R
----------------------------------------------------------------------
diff --git a/R/pkg/R/client.R b/R/pkg/R/client.R
index 25e9939..2d341d8 100644
--- a/R/pkg/R/client.R
+++ b/R/pkg/R/client.R
@@ -38,7 +38,7 @@ determineSparkSubmitBin <- function() {
if (.Platform$OS.type == "unix") {
sparkSubmitBinName <- "spark-submit"
} else {
- sparkSubmitBinName <- "spark-submit.cmd"
+ sparkSubmitBinName <- "spark-submit2.cmd"
}
sparkSubmitBinName
}
@@ -69,5 +69,5 @@ launchBackend <- function(args, sparkHome, jars,
sparkSubmitOpts, packages) {
}
combinedArgs <- generateSparkSubmitArgs(args, sparkHome, jars,
sparkSubmitOpts, packages)
cat("Launching java with spark-submit command", sparkSubmitBin,
combinedArgs, "\n")
- invisible(system2(sparkSubmitBin, combinedArgs, wait = F))
+ invisible(launchScript(sparkSubmitBin, combinedArgs))
}
http://git-wip-us.apache.org/repos/asf/spark/blob/4f66bf5f/R/pkg/R/utils.R
----------------------------------------------------------------------
diff --git a/R/pkg/R/utils.R b/R/pkg/R/utils.R
index 784f737..e734366 100644
--- a/R/pkg/R/utils.R
+++ b/R/pkg/R/utils.R
@@ -664,3 +664,12 @@ varargsToJProperties <- function(...) {
}
props
}
+
+launchScript <- function(script, combinedArgs, capture = FALSE) {
+ if (.Platform$OS.type == "windows") {
+ scriptWithArgs <- paste(script, combinedArgs, sep = " ")
+ shell(scriptWithArgs, translate = TRUE, wait = capture, intern = capture)
# nolint
+ } else {
+ system2(script, combinedArgs, wait = capture, stdout = capture)
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/4f66bf5f/R/pkg/inst/tests/testthat/test_Windows.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_Windows.R
b/R/pkg/inst/tests/testthat/test_Windows.R
new file mode 100644
index 0000000..8813e18
--- /dev/null
+++ b/R/pkg/inst/tests/testthat/test_Windows.R
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+context("Windows-specific tests")
+
+test_that("sparkJars tag in SparkContext", {
+ if (.Platform$OS.type != "windows") {
+ skip("This test is only for Windows, skipped")
+ }
+ testOutput <- launchScript("ECHO", "a/b/c", capture = TRUE)
+ abcPath <- testOutput[1]
+ expect_equal(abcPath, "a\\b\\c")
+})
http://git-wip-us.apache.org/repos/asf/spark/blob/4f66bf5f/R/pkg/inst/tests/testthat/test_context.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_context.R
b/R/pkg/inst/tests/testthat/test_context.R
index c1f1a89..15915e2 100644
--- a/R/pkg/inst/tests/testthat/test_context.R
+++ b/R/pkg/inst/tests/testthat/test_context.R
@@ -129,13 +129,13 @@ test_that("getClientModeSparkSubmitOpts() returns
spark-submit args from whiteli
test_that("sparkJars sparkPackages as comma-separated strings", {
expect_warning(processSparkJars(" a, b "))
jars <- suppressWarnings(processSparkJars(" a, b "))
- expect_equal(jars, c("a", "b"))
+ expect_equal(lapply(jars, basename), list("a", "b"))
jars <- suppressWarnings(processSparkJars(" abc ,, def "))
- expect_equal(jars, c("abc", "def"))
+ expect_equal(lapply(jars, basename), list("abc", "def"))
jars <- suppressWarnings(processSparkJars(c(" abc ,, def ", "", "xyz", " ",
"a,b")))
- expect_equal(jars, c("abc", "def", "xyz", "a", "b"))
+ expect_equal(lapply(jars, basename), list("abc", "def", "xyz", "a", "b"))
p <- processSparkPackages(c("ghi", "lmn"))
expect_equal(p, c("ghi", "lmn"))
http://git-wip-us.apache.org/repos/asf/spark/blob/4f66bf5f/R/pkg/inst/tests/testthat/test_includeJAR.R
----------------------------------------------------------------------
diff --git a/R/pkg/inst/tests/testthat/test_includeJAR.R
b/R/pkg/inst/tests/testthat/test_includeJAR.R
index f89aa8e..512dd39 100644
--- a/R/pkg/inst/tests/testthat/test_includeJAR.R
+++ b/R/pkg/inst/tests/testthat/test_includeJAR.R
@@ -21,10 +21,9 @@ runScript <- function() {
sparkTestJarPath <- "R/lib/SparkR/test_support/sparktestjar_2.10-1.0.jar"
jarPath <- paste("--jars", shQuote(file.path(sparkHome, sparkTestJarPath)))
scriptPath <- file.path(sparkHome, "R/lib/SparkR/tests/testthat/jarTest.R")
- submitPath <- file.path(sparkHome, "bin/spark-submit")
- res <- system2(command = submitPath,
- args = c(jarPath, scriptPath),
- stdout = TRUE)
+ submitPath <- file.path(sparkHome, paste("bin/", determineSparkSubmitBin(),
sep = ""))
+ combinedArgs <- paste(jarPath, scriptPath, sep = " ")
+ res <- launchScript(submitPath, combinedArgs, capture = TRUE)
tail(res, 2)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
