spark git commit: [SPARK-9855] [SPARKR] Add expression functions into SparkR whose params are simple
Repository: spark
Updated Branches:
refs/heads/master 0d1d146c2 -> f4bc01f1f
[SPARK-9855] [SPARKR] Add expression functions into SparkR whose params are
simple
I added lots of expression functions for SparkR. This PR includes only
functions whose params are only `(Column)` or `(Column, Column)`. And I think
we need to improve how to test those functions. However, it would be better to
work on another issue.
## Diff Summary
- Add lots of functions in `functions.R` and their generic in `generic.R`
- Add aliases for `ceiling` and `sign`
- Move expression functions from `column.R` to `functions.R`
- Modify `rdname` from `column` to `functions`
I haven't supported `not` function, because the name has a collesion with
`testthat` package. I didn't think of the way to define it.
## New Supported Functions
```
approxCountDistinct
ascii
base64
bin
bitwiseNOT
ceil (alias: ceiling)
crc32
dayofmonth
dayofyear
explode
factorial
hex
hour
initcap
isNaN
last_day
length
log2
ltrim
md5
minute
month
negate
quarter
reverse
round
rtrim
second
sha1
signum (alias: sign)
size
soundex
to_date
trim
unbase64
unhex
weekofyear
year
datediff
levenshtein
months_between
nanvl
pmod
```
## JIRA
[[SPARK-9855] Add expression functions into SparkR whose params are simple -
ASF JIRA](https://issues.apache.org/jira/browse/SPARK-9855)
Author: Yu ISHIKAWA
Closes #8123 from yu-iskw/SPARK-9855.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f4bc01f1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f4bc01f1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f4bc01f1
Branch: refs/heads/master
Commit: f4bc01f1f33a93e6affe5c8a3e33ffbd92d03f38
Parents: 0d1d146
Author: Yu ISHIKAWA
Authored: Wed Aug 12 18:33:27 2015 -0700
Committer: Shivaram Venkataraman
Committed: Wed Aug 12 18:33:27 2015 -0700
--
R/pkg/DESCRIPTION| 1 +
R/pkg/R/column.R | 81 ---
R/pkg/R/functions.R | 123 ++
R/pkg/R/generics.R | 185 +++---
R/pkg/inst/tests/test_sparkSQL.R | 21 ++--
5 files changed, 309 insertions(+), 102 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/f4bc01f1/R/pkg/DESCRIPTION
--
diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index 4949d86..83e6489 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -29,6 +29,7 @@ Collate:
'client.R'
'context.R'
'deserialize.R'
+'functions.R'
'mllib.R'
'serialize.R'
'sparkR.R'
http://git-wip-us.apache.org/repos/asf/spark/blob/f4bc01f1/R/pkg/R/column.R
--
diff --git a/R/pkg/R/column.R b/R/pkg/R/column.R
index eeaf9f1..328f595 100644
--- a/R/pkg/R/column.R
+++ b/R/pkg/R/column.R
@@ -60,12 +60,6 @@ operators <- list(
)
column_functions1 <- c("asc", "desc", "isNull", "isNotNull")
column_functions2 <- c("like", "rlike", "startsWith", "endsWith", "getField",
"getItem", "contains")
-functions <- c("min", "max", "sum", "avg", "mean", "count", "abs", "sqrt",
- "first", "last", "lower", "upper", "sumDistinct",
- "acos", "asin", "atan", "cbrt", "ceiling", "cos", "cosh", "exp",
- "expm1", "floor", "log", "log10", "log1p", "rint", "sign",
- "sin", "sinh", "tan", "tanh", "toDegrees", "toRadians")
-binary_mathfunctions <- c("atan2", "hypot")
createOperator <- function(op) {
setMethod(op,
@@ -111,33 +105,6 @@ createColumnFunction2 <- function(name) {
})
}
-createStaticFunction <- function(name) {
- setMethod(name,
-signature(x = "Column"),
-function(x) {
- if (name == "ceiling") {
- name <- "ceil"
- }
- if (name == "sign") {
- name <- "signum"
- }
- jc <- callJStatic("org.apache.spark.sql.functions", name, x@jc)
- column(jc)
-})
-}
-
-createBinaryMathfunctions <- function(name) {
- setMethod(name,
-signature(y = "Column"),
-function(y, x) {
- if (class(x) == "Column") {
-x <- x@jc
- }
- jc <- callJStatic("org.apache.spark.sql.functions", name, y@jc,
x)
- column(jc)
-})
-}
-
createMethods <- function() {
for (op in names(operators)) {
createOperator(op)
@@ -148,12 +115,6 @@ createMethods <- function() {
for (name in column_functions2) {
createColumnFunction2(name)
}
- for (x in functions) {
-createStaticFunction(x)
- }
- for (name in binary_mathfunctions) {
-createBinaryMathfunctions(name)
- }
}
createMethods()
@@ -242,4
spark git commit: [SPARK-9855] [SPARKR] Add expression functions into SparkR whose params are simple
Repository: spark
Updated Branches:
refs/heads/branch-1.5 62ab2a4c6 -> ca39c9e91
[SPARK-9855] [SPARKR] Add expression functions into SparkR whose params are
simple
I added lots of expression functions for SparkR. This PR includes only
functions whose params are only `(Column)` or `(Column, Column)`. And I think
we need to improve how to test those functions. However, it would be better to
work on another issue.
## Diff Summary
- Add lots of functions in `functions.R` and their generic in `generic.R`
- Add aliases for `ceiling` and `sign`
- Move expression functions from `column.R` to `functions.R`
- Modify `rdname` from `column` to `functions`
I haven't supported `not` function, because the name has a collesion with
`testthat` package. I didn't think of the way to define it.
## New Supported Functions
```
approxCountDistinct
ascii
base64
bin
bitwiseNOT
ceil (alias: ceiling)
crc32
dayofmonth
dayofyear
explode
factorial
hex
hour
initcap
isNaN
last_day
length
log2
ltrim
md5
minute
month
negate
quarter
reverse
round
rtrim
second
sha1
signum (alias: sign)
size
soundex
to_date
trim
unbase64
unhex
weekofyear
year
datediff
levenshtein
months_between
nanvl
pmod
```
## JIRA
[[SPARK-9855] Add expression functions into SparkR whose params are simple -
ASF JIRA](https://issues.apache.org/jira/browse/SPARK-9855)
Author: Yu ISHIKAWA
Closes #8123 from yu-iskw/SPARK-9855.
(cherry picked from commit f4bc01f1f33a93e6affe5c8a3e33ffbd92d03f38)
Signed-off-by: Shivaram Venkataraman
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ca39c9e9
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ca39c9e9
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ca39c9e9
Branch: refs/heads/branch-1.5
Commit: ca39c9e91602223f5665ab6942b917c4900bd996
Parents: 62ab2a4
Author: Yu ISHIKAWA
Authored: Wed Aug 12 18:33:27 2015 -0700
Committer: Shivaram Venkataraman
Committed: Wed Aug 12 18:33:35 2015 -0700
--
R/pkg/DESCRIPTION| 1 +
R/pkg/R/column.R | 81 ---
R/pkg/R/functions.R | 123 ++
R/pkg/R/generics.R | 185 +++---
R/pkg/inst/tests/test_sparkSQL.R | 21 ++--
5 files changed, 309 insertions(+), 102 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/ca39c9e9/R/pkg/DESCRIPTION
--
diff --git a/R/pkg/DESCRIPTION b/R/pkg/DESCRIPTION
index 4949d86..83e6489 100644
--- a/R/pkg/DESCRIPTION
+++ b/R/pkg/DESCRIPTION
@@ -29,6 +29,7 @@ Collate:
'client.R'
'context.R'
'deserialize.R'
+'functions.R'
'mllib.R'
'serialize.R'
'sparkR.R'
http://git-wip-us.apache.org/repos/asf/spark/blob/ca39c9e9/R/pkg/R/column.R
--
diff --git a/R/pkg/R/column.R b/R/pkg/R/column.R
index eeaf9f1..328f595 100644
--- a/R/pkg/R/column.R
+++ b/R/pkg/R/column.R
@@ -60,12 +60,6 @@ operators <- list(
)
column_functions1 <- c("asc", "desc", "isNull", "isNotNull")
column_functions2 <- c("like", "rlike", "startsWith", "endsWith", "getField",
"getItem", "contains")
-functions <- c("min", "max", "sum", "avg", "mean", "count", "abs", "sqrt",
- "first", "last", "lower", "upper", "sumDistinct",
- "acos", "asin", "atan", "cbrt", "ceiling", "cos", "cosh", "exp",
- "expm1", "floor", "log", "log10", "log1p", "rint", "sign",
- "sin", "sinh", "tan", "tanh", "toDegrees", "toRadians")
-binary_mathfunctions <- c("atan2", "hypot")
createOperator <- function(op) {
setMethod(op,
@@ -111,33 +105,6 @@ createColumnFunction2 <- function(name) {
})
}
-createStaticFunction <- function(name) {
- setMethod(name,
-signature(x = "Column"),
-function(x) {
- if (name == "ceiling") {
- name <- "ceil"
- }
- if (name == "sign") {
- name <- "signum"
- }
- jc <- callJStatic("org.apache.spark.sql.functions", name, x@jc)
- column(jc)
-})
-}
-
-createBinaryMathfunctions <- function(name) {
- setMethod(name,
-signature(y = "Column"),
-function(y, x) {
- if (class(x) == "Column") {
-x <- x@jc
- }
- jc <- callJStatic("org.apache.spark.sql.functions", name, y@jc,
x)
- column(jc)
-})
-}
-
createMethods <- function() {
for (op in names(operators)) {
createOperator(op)
@@ -148,12 +115,6 @@ createMethods <- function() {
for (name in column_functions2) {
createColumnFunction2(name)
}
- for (x in functions) {
-createStaticFunction(x)
-
