This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sedona.git
The following commit(s) were added to refs/heads/master by this push:
new 536f498 [SEDONA-67] apply workaround for SPARK-37202; re-enable R
builds for Spark 3.2 (#561)
536f498 is described below
commit 536f4980a8590542fb2b19928d6890ac8612c2dd
Author: Yitao Li <[email protected]>
AuthorDate: Wed Nov 10 08:19:30 2021 +0000
[SEDONA-67] apply workaround for SPARK-37202; re-enable R builds for Spark
3.2 (#561)
---
.github/workflows/r.yml | 4 ++--
R/tests/testthat/test-crs-transform.R | 3 +++
R/tests/testthat/test-dbplyr-integration.R | 37 +++++++++++++++++++++++++-----
3 files changed, 36 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
index ec2628f..b4396a0 100644
--- a/.github/workflows/r.yml
+++ b/.github/workflows/r.yml
@@ -15,7 +15,7 @@ jobs:
strategy:
fail-fast: true
matrix:
- spark: [2.4.8, 3.0.3, 3.1.2]
+ spark: [2.4.8, 3.0.3, 3.1.2, 3.2.0]
scala: [2.11.8, 2.12.8]
r: [oldrel, release]
exclude:
@@ -24,7 +24,7 @@ jobs:
r: oldrel
- spark: 3.2.0
scala: 2.11.8
- r: release
+ r: release
- spark: 3.1.2
scala: 2.11.8
r: oldrel
diff --git a/R/tests/testthat/test-crs-transform.R
b/R/tests/testthat/test-crs-transform.R
index 3056486..1c65e12 100644
--- a/R/tests/testthat/test-crs-transform.R
+++ b/R/tests/testthat/test-crs-transform.R
@@ -30,6 +30,9 @@ test_that("crs_transform() works as expected", {
sdf_register() %>%
head(5) %>%
dplyr::transmute(x = ST_X(geometry), y = ST_Y(geometry)) %>%
+ # NOTE: the extra `sdf_register()` call is a workaround until
SPARK-37202 is
+ # fixed
+ sdf_register(name = random_string()) %>%
collect(),
tibble::tribble(
~x, ~y,
diff --git a/R/tests/testthat/test-dbplyr-integration.R
b/R/tests/testthat/test-dbplyr-integration.R
index c5223dd..41aaa5f 100644
--- a/R/tests/testthat/test-dbplyr-integration.R
+++ b/R/tests/testthat/test-dbplyr-integration.R
@@ -21,7 +21,10 @@ sc <- testthat_spark_connection()
test_that("ST_Point() works as expected", {
sdf <- sdf_len(sc, 1) %>%
- dplyr::mutate(pt = ST_Point(-40, 40))
+ dplyr::mutate(pt = ST_Point(-40, 40)) %>%
+ # NOTE: the extra `sdf_register()` call is a workaround until SPARK-37202
is
+ # fixed
+ sdf_register(name = random_string())
df <- sdf %>% collect()
expect_equal(nrow(df), 1)
@@ -57,6 +60,9 @@ test_that("ST_PolygonFromEnvelope() works as expected", {
test_that("ST_Buffer() works as expected", {
sdf <- sdf_len(sc, 1) %>%
dplyr::mutate(pt = ST_Point(-40, 40)) %>%
+ # NOTE: the extra `sdf_register()` call is a workaround until SPARK-37202
is
+ # fixed
+ sdf_register(name = random_string()) %>%
dplyr::compute()
expect_equal(
@@ -74,7 +80,11 @@ test_that("ST_Buffer() works as expected", {
test_that("ST_PrecisionReduce() works as expected", {
sdf <- sdf_len(sc, 1) %>%
dplyr::mutate(rectangle = ST_PolygonFromEnvelope(-40.12345678,
-30.12345678, 40.11111111, 30.11111111)) %>%
- dplyr::mutate(rectangle = ST_PrecisionReduce(rectangle, 2))
+ dplyr::mutate(rectangle = ST_PrecisionReduce(rectangle, 2)) %>%
+ # NOTE: the extra `sdf_register()` call is a workaround until SPARK-37202
is
+ # fixed
+ sdf_register(name = random_string())
+
df <- sdf %>% collect()
expect_equal(nrow(df), 1)
@@ -94,6 +104,9 @@ test_that("ST_PrecisionReduce() works as expected", {
test_that("ST_SimplifyPreserveTopology() works as expected", {
sdf <- sdf_len(sc, 1) %>%
dplyr::mutate(pt = ST_Point(-40, 40)) %>%
+ # NOTE: the extra `sdf_register()` call is a workaround until SPARK-37202
is
+ # fixed
+ sdf_register(name = random_string()) %>%
dplyr::compute()
expect_equal(
@@ -111,7 +124,10 @@ test_that("ST_SimplifyPreserveTopology() works as
expected", {
test_that("ST_GeometryN() works as expected", {
sdf <- sdf_len(sc, 1) %>%
dplyr::mutate(pts = ST_GeomFromText("MULTIPOINT((1 2), (3 4), (5 6), (8
9))")) %>%
- dplyr::transmute(pt = ST_GeometryN(pts, 2))
+ dplyr::transmute(pt = ST_GeometryN(pts, 2)) %>%
+ # NOTE: the extra `sdf_register()` call is a workaround until SPARK-37202
is
+ # fixed
+ sdf_register(name = random_string())
df <- sdf %>% collect()
expect_equal(nrow(df), 1)
@@ -128,7 +144,10 @@ test_that("ST_GeometryN() works as expected", {
test_that("ST_InteriorRingN() works as expected", {
sdf <- sdf_len(sc, 1) %>%
dplyr::mutate(polygon = ST_GeomFromText("POLYGON((0 0, 0 5, 5 5, 5 0, 0
0), (1 1, 2 1, 2 2, 1 2, 1 1), (1 3, 2 3, 2 4, 1 4, 1 3), (3 3, 4 3, 4 4, 3 4,
3 3))")) %>%
- dplyr::transmute(interior_ring = ST_InteriorRingN(polygon, 0))
+ dplyr::transmute(interior_ring = ST_InteriorRingN(polygon, 0)) %>%
+ # NOTE: the extra `sdf_register()` call is a workaround until SPARK-37202
is
+ # fixed
+ sdf_register(name = random_string())
df <- sdf %>% collect()
expect_equal(nrow(df), 1)
@@ -147,7 +166,10 @@ test_that("ST_InteriorRingN() works as expected", {
test_that("ST_AddPoint() works as expected", {
sdf <- sdf_len(sc, 1) %>%
dplyr::mutate(linestring = ST_GeomFromText("LINESTRING(0 0, 1 1, 1 0)"))
%>%
- dplyr::transmute(linestring = ST_AddPoint(linestring,
ST_GeomFromText("Point(21 52)"), 1))
+ dplyr::transmute(linestring = ST_AddPoint(linestring,
ST_GeomFromText("Point(21 52)"), 1)) %>%
+ # NOTE: the extra `sdf_register()` call is a workaround until SPARK-37202
is
+ # fixed
+ sdf_register(name = random_string())
df <- sdf %>% collect()
expect_equal(nrow(df), 1)
@@ -183,7 +205,10 @@ test_that("ST_AddPoint() works as expected", {
test_that("ST_RemovePoint() works as expected", {
sdf <- sdf_len(sc, 1) %>%
dplyr::mutate(linestring = ST_GeomFromText("LINESTRING(0 0, 21 52, 1 1, 1
0)")) %>%
- dplyr::transmute(linestring = ST_RemovePoint(linestring, 1))
+ dplyr::transmute(linestring = ST_RemovePoint(linestring, 1)) %>%
+ # NOTE: the extra `sdf_register()` call is a workaround until SPARK-37202
is
+ # fixed
+ sdf_register(name = random_string())
df <- sdf %>% collect()
expect_equal(nrow(df), 1)
