beliefer commented on a change in pull request #28060: [SPARK-31291][SQL][TEST]
Avoid load test data if test case not uses them
URL: https://github.com/apache/spark/pull/28060#discussion_r405285989
##########
File path: sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
##########
@@ -570,82 +579,94 @@ class SQLQueryTestSuite extends QueryTest with
SharedSparkSession {
}
/** Load built-in test tables into the SparkSession. */
- private def loadTestData(session: SparkSession): Unit = {
+ private def loadTestData(session: SparkSession, testTables: Seq[String]):
Unit = {
import session.implicits._
- (1 to 100).map(i => (i, i.toString)).toDF("key",
"value").createOrReplaceTempView("testdata")
-
- ((Seq(1, 2, 3), Seq(Seq(1, 2, 3))) :: (Seq(2, 3, 4), Seq(Seq(2, 3, 4))) ::
Nil)
- .toDF("arraycol", "nestedarraycol")
- .createOrReplaceTempView("arraydata")
-
- (Tuple1(Map(1 -> "a1", 2 -> "b1", 3 -> "c1", 4 -> "d1", 5 -> "e1")) ::
- Tuple1(Map(1 -> "a2", 2 -> "b2", 3 -> "c2", 4 -> "d2")) ::
- Tuple1(Map(1 -> "a3", 2 -> "b3", 3 -> "c3")) ::
- Tuple1(Map(1 -> "a4", 2 -> "b4")) ::
- Tuple1(Map(1 -> "a5")) :: Nil)
- .toDF("mapcol")
- .createOrReplaceTempView("mapdata")
-
- session
- .read
- .format("csv")
- .options(Map("delimiter" -> "\t", "header" -> "false"))
- .schema("a int, b float")
- .load(testFile("test-data/postgresql/agg.data"))
- .createOrReplaceTempView("aggtest")
-
- session
- .read
- .format("csv")
- .options(Map("delimiter" -> "\t", "header" -> "false"))
- .schema(
- """
- |unique1 int,
- |unique2 int,
- |two int,
- |four int,
- |ten int,
- |twenty int,
- |hundred int,
- |thousand int,
- |twothousand int,
- |fivethous int,
- |tenthous int,
- |odd int,
- |even int,
- |stringu1 string,
- |stringu2 string,
- |string4 string
- """.stripMargin)
- .load(testFile("test-data/postgresql/onek.data"))
- .createOrReplaceTempView("onek")
-
- session
- .read
- .format("csv")
- .options(Map("delimiter" -> "\t", "header" -> "false"))
- .schema(
- """
- |unique1 int,
- |unique2 int,
- |two int,
- |four int,
- |ten int,
- |twenty int,
- |hundred int,
- |thousand int,
- |twothousand int,
- |fivethous int,
- |tenthous int,
- |odd int,
- |even int,
- |stringu1 string,
- |stringu2 string,
- |string4 string
- """.stripMargin)
- .load(testFile("test-data/postgresql/tenk.data"))
- .createOrReplaceTempView("tenk1")
+ if (testTables.contains("testdata")) {
+ (1 to 100).map(i => (i, i.toString)).toDF("key",
"value").createOrReplaceTempView("testdata")
+ }
+
+ if (testTables.contains("arraydata")) {
+ ((Seq(1, 2, 3), Seq(Seq(1, 2, 3))) :: (Seq(2, 3, 4), Seq(Seq(2, 3, 4)))
:: Nil)
+ .toDF("arraycol", "nestedarraycol")
+ .createOrReplaceTempView("arraydata")
Review comment:
@cloud-fan
When use `sparkSession.newSession()`, will failed. such as:
```
23:25:05.078 ERROR org.apache.spark.sql.SQLQueryTestSuite: Error using
configs:
[info] - operators.sql *** FAILED *** (5 seconds, 722 milliseconds)
[info] operators.sql
[info] Expected "struct<[(- key):int,(+ key):int]>", but got "struct<[]>"
Schema did not match for query #4
[info] select -key, +key from testdata where key = 2: -- !query
[info] select -key, +key from testdata where key = 2
[info] -- !query schema
[info] struct<>
[info] -- !query output
[info] org.apache.spark.sql.AnalysisException
[info] Table or view not found: testdata; line 1 pos 23
(SQLQueryTestSuite.scala:464)
[info] org.scalatest.exceptions.TestFailedException:
[info] at
org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:530)
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
