Repository: spark
Updated Branches:
refs/heads/master 76e8a1d7e -> 1eebfbe19
[SPARK-21208][R] Adds setLocalProperty and getLocalProperty in R
## What changes were proposed in this pull request?
This PR adds `setLocalProperty` and `getLocalProperty`in R.
```R
> df <- createDataFrame(iris)
> setLocalProperty("spark.job.description", "Hello world!")
> count(df)
> setLocalProperty("spark.job.description", "Hi !!")
> count(df)
```
<img width="775" alt="2017-12-25 4 18 07"
src="https://user-images.githubusercontent.com/6477701/34335213-60655a7c-e990-11e7-88aa-12debe311627.png";>
```R
> print(getLocalProperty("spark.job.description"))
NULL
> setLocalProperty("spark.job.description", "Hello world!")
> print(getLocalProperty("spark.job.description"))
[1] "Hello world!"
> setLocalProperty("spark.job.description", "Hi !!")
> print(getLocalProperty("spark.job.description"))
[1] "Hi !!"
```
## How was this patch tested?
Manually tested and a test in `R/pkg/tests/fulltests/test_context.R`.
Author: hyukjinkwon <gurwls...@gmail.com>
Closes #20075 from HyukjinKwon/SPARK-21208.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1eebfbe1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1eebfbe1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1eebfbe1
Branch: refs/heads/master
Commit: 1eebfbe192060af3c81cd086bc5d5a7e80d09e77
Parents: 76e8a1d
Author: hyukjinkwon <gurwls...@gmail.com>
Authored: Thu Dec 28 20:18:47 2017 +0900
Committer: hyukjinkwon <gurwls...@gmail.com>
Committed: Thu Dec 28 20:18:47 2017 +0900
----------------------------------------------------------------------
R/pkg/NAMESPACE | 4 ++-
R/pkg/R/sparkR.R | 45 +++++++++++++++++++++++++++++++
R/pkg/tests/fulltests/test_context.R | 33 ++++++++++++++++++++++-
3 files changed, 80 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/1eebfbe1/R/pkg/NAMESPACE
----------------------------------------------------------------------
diff --git a/R/pkg/NAMESPACE b/R/pkg/NAMESPACE
index 4b699de..ce3eec0 100644
--- a/R/pkg/NAMESPACE
+++ b/R/pkg/NAMESPACE
@@ -76,7 +76,9 @@ exportMethods("glm",
export("setJobGroup",
"clearJobGroup",
"cancelJobGroup",
- "setJobDescription")
+ "setJobDescription",
+ "setLocalProperty",
+ "getLocalProperty")
# Export Utility methods
export("setLogLevel")
http://git-wip-us.apache.org/repos/asf/spark/blob/1eebfbe1/R/pkg/R/sparkR.R
----------------------------------------------------------------------
diff --git a/R/pkg/R/sparkR.R b/R/pkg/R/sparkR.R
index fb5f1d2..965471f 100644
--- a/R/pkg/R/sparkR.R
+++ b/R/pkg/R/sparkR.R
@@ -560,10 +560,55 @@ cancelJobGroup <- function(sc, groupId) {
#'}
#' @note setJobDescription since 2.3.0
setJobDescription <- function(value) {
+ if (!is.null(value)) {
+ value <- as.character(value)
+ }
sc <- getSparkContext()
invisible(callJMethod(sc, "setJobDescription", value))
}
+#' Set a local property that affects jobs submitted from this thread, such as
the
+#' Spark fair scheduler pool.
+#'
+#' @param key The key for a local property.
+#' @param value The value for a local property.
+#' @rdname setLocalProperty
+#' @name setLocalProperty
+#' @examples
+#'\dontrun{
+#' setLocalProperty("spark.scheduler.pool", "poolA")
+#'}
+#' @note setLocalProperty since 2.3.0
+setLocalProperty <- function(key, value) {
+ if (is.null(key) || is.na(key)) {
+ stop("key should not be NULL or NA.")
+ }
+ if (!is.null(value)) {
+ value <- as.character(value)
+ }
+ sc <- getSparkContext()
+ invisible(callJMethod(sc, "setLocalProperty", as.character(key), value))
+}
+
+#' Get a local property set in this thread, or \code{NULL} if it is missing.
See
+#' \code{setLocalProperty}.
+#'
+#' @param key The key for a local property.
+#' @rdname getLocalProperty
+#' @name getLocalProperty
+#' @examples
+#'\dontrun{
+#' getLocalProperty("spark.scheduler.pool")
+#'}
+#' @note getLocalProperty since 2.3.0
+getLocalProperty <- function(key) {
+ if (is.null(key) || is.na(key)) {
+ stop("key should not be NULL or NA.")
+ }
+ sc <- getSparkContext()
+ callJMethod(sc, "getLocalProperty", as.character(key))
+}
+
sparkConfToSubmitOps <- new.env()
sparkConfToSubmitOps[["spark.driver.memory"]] <- "--driver-memory"
sparkConfToSubmitOps[["spark.driver.extraClassPath"]] <-
"--driver-class-path"
http://git-wip-us.apache.org/repos/asf/spark/blob/1eebfbe1/R/pkg/tests/fulltests/test_context.R
----------------------------------------------------------------------
diff --git a/R/pkg/tests/fulltests/test_context.R
b/R/pkg/tests/fulltests/test_context.R
index 77635c5..f0d0a51 100644
--- a/R/pkg/tests/fulltests/test_context.R
+++ b/R/pkg/tests/fulltests/test_context.R
@@ -100,7 +100,6 @@ test_that("job group functions can be called", {
setJobGroup("groupId", "job description", TRUE)
cancelJobGroup("groupId")
clearJobGroup()
- setJobDescription("job description")
suppressWarnings(setJobGroup(sc, "groupId", "job description", TRUE))
suppressWarnings(cancelJobGroup(sc, "groupId"))
@@ -108,6 +107,38 @@ test_that("job group functions can be called", {
sparkR.session.stop()
})
+test_that("job description and local properties can be set and got", {
+ sc <- sparkR.sparkContext(master = sparkRTestMaster)
+ setJobDescription("job description")
+ expect_equal(getLocalProperty("spark.job.description"), "job description")
+ setJobDescription(1234)
+ expect_equal(getLocalProperty("spark.job.description"), "1234")
+ setJobDescription(NULL)
+ expect_equal(getLocalProperty("spark.job.description"), NULL)
+ setJobDescription(NA)
+ expect_equal(getLocalProperty("spark.job.description"), NULL)
+
+ setLocalProperty("spark.scheduler.pool", "poolA")
+ expect_equal(getLocalProperty("spark.scheduler.pool"), "poolA")
+ setLocalProperty("spark.scheduler.pool", NULL)
+ expect_equal(getLocalProperty("spark.scheduler.pool"), NULL)
+ setLocalProperty("spark.scheduler.pool", NA)
+ expect_equal(getLocalProperty("spark.scheduler.pool"), NULL)
+
+ setLocalProperty(4321, 1234)
+ expect_equal(getLocalProperty(4321), "1234")
+ setLocalProperty(4321, NULL)
+ expect_equal(getLocalProperty(4321), NULL)
+ setLocalProperty(4321, NA)
+ expect_equal(getLocalProperty(4321), NULL)
+
+ expect_error(setLocalProperty(NULL, "should fail"), "key should not be NULL
or NA")
+ expect_error(getLocalProperty(NULL), "key should not be NULL or NA")
+ expect_error(setLocalProperty(NA, "should fail"), "key should not be NULL or
NA")
+ expect_error(getLocalProperty(NA), "key should not be NULL or NA")
+ sparkR.session.stop()
+})
+
test_that("utility function can be called", {
sparkR.sparkContext(master = sparkRTestMaster)
setLogLevel("ERROR")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
![https://user-images.githubusercontent.com/6477701/34335213-60655a7c-e990-11e7-88aa-12debe311627.png"](https://user-images.githubusercontent.com/6477701/34335213-60655a7c-e990-11e7-88aa-12debe311627.png"){kind=link}