This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 837843b [SPARK-32810][SQL][TESTS][FOLLOWUP][3.0] Check path globbing
in JSON/CSV datasources v1 and v2
837843b is described below
commit 837843bea40eec842c782e3c719f8d81024d8a06
Author: Max Gekk <max.g...@gmail.com>
AuthorDate: Wed Sep 9 21:16:16 2020 +0900
[SPARK-32810][SQL][TESTS][FOLLOWUP][3.0] Check path globbing in JSON/CSV
datasources v1 and v2
### What changes were proposed in this pull request?
In the PR, I propose to move the test `SPARK-32810: CSV and JSON data
sources should be able to read files with escaped glob metacharacter in the
paths` from `DataFrameReaderWriterSuite` to `CSVSuite` and to `JsonSuite`. This
will allow to run the same test in `CSVv1Suite`/`CSVv2Suite` and in
`JsonV1Suite`/`JsonV2Suite`.
### Why are the changes needed?
To improve test coverage by checking JSON/CSV datasources v1 and v2.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
By running affected test suites:
```
$ build/sbt "sql/test:testOnly
org.apache.spark.sql.execution.datasources.csv.*"
$ build/sbt "sql/test:testOnly
org.apache.spark.sql.execution.datasources.json.*"
```
Closes #29690 from MaxGekk/globbing-paths-when-inferring-schema-dsv2-3.0.
Authored-by: Max Gekk <max.g...@gmail.com>
Signed-off-by: HyukjinKwon <gurwls...@apache.org>
---
.../sql/execution/datasources/csv/CSVSuite.scala | 13 ++++++++++++
.../sql/execution/datasources/json/JsonSuite.scala | 13 ++++++++++++
.../sql/test/DataFrameReaderWriterSuite.scala | 23 ----------------------
3 files changed, 26 insertions(+), 23 deletions(-)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
index 9ba2cab..4e93ea3 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/csv/CSVSuite.scala
@@ -2365,6 +2365,19 @@ abstract class CSVSuite extends QueryTest with
SharedSparkSession with TestCsvDa
assert(df.count() == 3)
}
}
+
+ test("SPARK-32810: CSV data source should be able to read files with " +
+ "escaped glob metacharacter in the paths") {
+ withTempDir { dir =>
+ val basePath = dir.getCanonicalPath
+ // test CSV writer / reader without specifying schema
+ val csvTableName = "[abc]"
+ spark.range(3).coalesce(1).write.csv(s"$basePath/$csvTableName")
+ val readback = spark.read
+ .csv(s"$basePath/${"""(\[|\]|\{|\})""".r.replaceAllIn(csvTableName,
"""\\$1""")}")
+ assert(readback.collect sameElements Array(Row("0"), Row("1"), Row("2")))
+ }
+ }
}
class CSVv1Suite extends CSVSuite {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index 38b5e77..8eb5432 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -2706,6 +2706,19 @@ abstract class JsonSuite extends QueryTest with
SharedSparkSession with TestJson
checkAnswer(json, Row(null))
}
}
+
+ test("SPARK-32810: JSON data source should be able to read files with " +
+ "escaped glob metacharacter in the paths") {
+ withTempDir { dir =>
+ val basePath = dir.getCanonicalPath
+ // test JSON writer / reader without specifying schema
+ val jsonTableName = "{def}"
+ spark.range(3).coalesce(1).write.json(s"$basePath/$jsonTableName")
+ val readback = spark.read
+ .json(s"$basePath/${"""(\[|\]|\{|\})""".r.replaceAllIn(jsonTableName,
"""\\$1""")}")
+ assert(readback.collect sameElements Array(Row(0), Row(1), Row(2)))
+ }
+ }
}
class JsonV1Suite extends JsonSuite {
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
index f48f445..c7ca012 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/test/DataFrameReaderWriterSuite.scala
@@ -1105,27 +1105,4 @@ class DataFrameReaderWriterSuite extends QueryTest with
SharedSparkSession with
}
}
}
-
- test("SPARK-32810: CSV and JSON data sources should be able to read files
with " +
- "escaped glob metacharacter in the paths") {
- def escape(str: String): String = {
- """(\[|\]|\{|\})""".r.replaceAllIn(str, """\\$1""")
- }
-
- withTempDir { dir =>
- val basePath = dir.getCanonicalPath
-
- // test CSV writer / reader without specifying schema
- val csvTableName = "[abc]"
- spark.range(3).coalesce(1).write.csv(s"$basePath/$csvTableName")
- val csvDf = spark.read.csv(s"$basePath/${escape(csvTableName)}")
- assert(csvDf.collect sameElements Array(Row("0"), Row("1"), Row("2")))
-
- // test JSON writer / reader without specifying schema
- val jsonTableName = "{def}"
- spark.range(3).coalesce(1).write.json(s"$basePath/$jsonTableName")
- val jsonDf = spark.read.json(s"$basePath/${escape(jsonTableName)}")
- assert(jsonDf.collect sameElements Array(Row(0), Row(1), Row(2)))
- }
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
