This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e2eb6d8 [SPARK-38203][SQL] Fix SQLInsertTestSuite and
SchemaPruningSuite under ANSI mode
e2eb6d8 is described below
commit e2eb6d8e0437e2305a99c897e2e7fe4b544a3573
Author: Gengliang Wang <[email protected]>
AuthorDate: Tue Feb 15 13:37:23 2022 +0800
[SPARK-38203][SQL] Fix SQLInsertTestSuite and SchemaPruningSuite under ANSI
mode
### What changes were proposed in this pull request?
Fix test failures of following tests under ANSI mode:
- HiveSQLInsertTestSuite
- FileSourceSQLInsertTestSuite
- ParquetV1SchemaPruningSuite
- ParquetV2SchemaPruningSuite
### Why are the changes needed?
To set up a new GA job to run tests with ANSI mode before 3.3.0 release.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Manual enable ANSI mode and test.
Closes #35511 from gengliangwang/fixParquetPruneAnsi.
Authored-by: Gengliang Wang <[email protected]>
Signed-off-by: Gengliang Wang <[email protected]>
---
.../org/apache/spark/sql/SQLInsertTestSuite.scala | 26 ++++++++++++++--------
.../execution/datasources/SchemaPruningSuite.scala | 4 ++++
2 files changed, 21 insertions(+), 9 deletions(-)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/SQLInsertTestSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SQLInsertTestSuite.scala
index 2f56fba..a3f6023 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLInsertTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLInsertTestSuite.scala
@@ -286,20 +286,28 @@ trait SQLInsertTestSuite extends QueryTest with
SQLTestUtils {
} else {
SQLConf.StoreAssignmentPolicy.values
}
+
+ def shouldThrowException(policy: SQLConf.StoreAssignmentPolicy.Value):
Boolean = policy match {
+ case SQLConf.StoreAssignmentPolicy.ANSI |
SQLConf.StoreAssignmentPolicy.STRICT =>
+ true
+ case SQLConf.StoreAssignmentPolicy.LEGACY =>
+ SQLConf.get.ansiEnabled
+ }
+
testingPolicies.foreach { policy =>
withSQLConf(
- SQLConf.STORE_ASSIGNMENT_POLICY.key -> policy.toString) {
+ SQLConf.STORE_ASSIGNMENT_POLICY.key -> policy.toString,
+ SQLConf.ANSI_ENABLED.key -> "false") {
withTable("t") {
sql("create table t(a int, b string) using parquet partitioned by
(a)")
- policy match {
- case SQLConf.StoreAssignmentPolicy.ANSI |
SQLConf.StoreAssignmentPolicy.STRICT =>
- val errorMsg = intercept[NumberFormatException] {
- sql("insert into t partition(a='ansi') values('ansi')")
- }.getMessage
- assert(errorMsg.contains("invalid input syntax for type numeric:
ansi"))
- case SQLConf.StoreAssignmentPolicy.LEGACY =>
+ if (shouldThrowException(policy)) {
+ val errorMsg = intercept[NumberFormatException] {
sql("insert into t partition(a='ansi') values('ansi')")
- checkAnswer(sql("select * from t"), Row("ansi", null) :: Nil)
+ }.getMessage
+ assert(errorMsg.contains("invalid input syntax for type numeric:
ansi"))
+ } else {
+ sql("insert into t partition(a='ansi') values('ansi')")
+ checkAnswer(sql("select * from t"), Row("ansi", null) :: Nil)
}
}
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
index 6fd966c..fe50e4e 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
@@ -21,6 +21,7 @@ import java.io.File
import org.scalactic.Equality
+import org.apache.spark.SparkConf
import org.apache.spark.sql.{DataFrame, QueryTest, Row}
import org.apache.spark.sql.catalyst.SchemaPruningTest
import org.apache.spark.sql.catalyst.expressions.Concat
@@ -57,6 +58,9 @@ abstract class SchemaPruningSuite
contactId: Int,
employer: Employer)
+ override protected def sparkConf: SparkConf =
+ super.sparkConf.set(SQLConf.ANSI_STRICT_INDEX_OPERATOR.key, "false")
+
val janeDoe = FullName("Jane", "X.", "Doe")
val johnDoe = FullName("John", "Y.", "Doe")
val susanSmith = FullName("Susan", "Z.", "Smith")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
