This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1016f274a95e [SPARK-51906][SQL][FOLLOW-UP] Enforce ANSI mode in
previous tests
1016f274a95e is described below
commit 1016f274a95efa88bcb17fba8611ffe4316d56c5
Author: Szehon Ho <[email protected]>
AuthorDate: Mon May 12 09:09:02 2025 +0900
[SPARK-51906][SQL][FOLLOW-UP] Enforce ANSI mode in previous tests
### What changes were proposed in this pull request?
Limit the tests added in https://github.com/apache/spark/pull/50701 and
https://github.com/apache/spark/pull/50593 only for ANSI_ENABLED mode.
### Why are the changes needed?
These tests fail in non-ANSI mode. The reason is that
https://github.com/apache/spark/blob/master/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/V2ExpressionUtils.scala
only converts majority of catalyst => V2 expressions in ANSI mode. So , we do
not get any V2Expression in non-ANSI case.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing unit test run with SPARK_ANSI_SQL_MODE=false
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #50851 from szehon-ho/add_column_default_val_follow.
Authored-by: Szehon Ho <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../sql/connector/DataSourceV2DataFrameSuite.scala | 240 +++++++++++----------
1 file changed, 122 insertions(+), 118 deletions(-)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala
index ba5d3cd940ab..5574814db625 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2DataFrameSuite.scala
@@ -346,137 +346,141 @@ class DataSourceV2DataFrameSuite
test("create/replace table with complex foldable default values") {
val tableName = "testcat.ns1.ns2.tbl"
withTable(tableName) {
- val createExec = executeAndKeepPhysicalPlan[CreateTableExec] {
- sql(
- s"""
- |CREATE TABLE $tableName (
- | id INT,
- | salary INT DEFAULT (100 + 23),
- | dep STRING DEFAULT ('h' || 'r'),
- | active BOOLEAN DEFAULT CAST(1 AS BOOLEAN)
- |) USING foo
- |""".stripMargin)
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+ val createExec = executeAndKeepPhysicalPlan[CreateTableExec] {
+ sql(
+ s"""
+ |CREATE TABLE $tableName (
+ | id INT,
+ | salary INT DEFAULT (100 + 23),
+ | dep STRING DEFAULT ('h' || 'r'),
+ | active BOOLEAN DEFAULT CAST(1 AS BOOLEAN)
+ |) USING foo
+ |""".stripMargin)
+ }
+
+ checkDefaultValues(
+ createExec.columns,
+ Array(
+ null,
+ new ColumnDefaultValue(
+ "(100 + 23)",
+ new GeneralScalarExpression(
+ "+",
+ Array(LiteralValue(100, IntegerType), LiteralValue(23,
IntegerType))),
+ LiteralValue(123, IntegerType)),
+ new ColumnDefaultValue(
+ "('h' || 'r')",
+ new GeneralScalarExpression(
+ "CONCAT",
+ Array(
+ LiteralValue(UTF8String.fromString("h"), StringType),
+ LiteralValue(UTF8String.fromString("r"), StringType))),
+ LiteralValue(UTF8String.fromString("hr"), StringType)),
+ new ColumnDefaultValue(
+ "CAST(1 AS BOOLEAN)",
+ new V2Cast(LiteralValue(1, IntegerType), IntegerType,
BooleanType),
+ LiteralValue(true, BooleanType))))
+
+ val df1 = Seq(1).toDF("id")
+ df1.writeTo(tableName).append()
+
+ sql(s"ALTER TABLE $tableName ALTER COLUMN dep SET DEFAULT ('i' ||
't')")
+
+ val df2 = Seq(2).toDF("id")
+ df2.writeTo(tableName).append()
+
+ checkAnswer(
+ sql(s"SELECT * FROM $tableName"),
+ Seq(
+ Row(1, 123, "hr", true),
+ Row(2, 123, "it", true)))
+
+ val replaceExec = executeAndKeepPhysicalPlan[ReplaceTableExec] {
+ sql(
+ s"""
+ |REPLACE TABLE $tableName (
+ | id INT,
+ | salary INT DEFAULT (50 * 2),
+ | dep STRING DEFAULT ('un' || 'known'),
+ | active BOOLEAN DEFAULT CAST(0 AS BOOLEAN)
+ |) USING foo
+ |""".stripMargin)
+ }
+
+ checkDefaultValues(
+ replaceExec.columns,
+ Array(
+ null,
+ new ColumnDefaultValue(
+ "(50 * 2)",
+ new GeneralScalarExpression(
+ "*",
+ Array(LiteralValue(50, IntegerType), LiteralValue(2,
IntegerType))),
+ LiteralValue(100, IntegerType)),
+ new ColumnDefaultValue(
+ "('un' || 'known')",
+ new GeneralScalarExpression(
+ "CONCAT",
+ Array(
+ LiteralValue(UTF8String.fromString("un"), StringType),
+ LiteralValue(UTF8String.fromString("known"), StringType))),
+ LiteralValue(UTF8String.fromString("unknown"), StringType)),
+ new ColumnDefaultValue(
+ "CAST(0 AS BOOLEAN)",
+ new V2Cast(LiteralValue(0, IntegerType), IntegerType,
BooleanType),
+ LiteralValue(false, BooleanType))))
+
+ val df3 = Seq(1).toDF("id")
+ df3.writeTo(tableName).append()
+
+ checkAnswer(
+ sql(s"SELECT * FROM $tableName"),
+ Seq(Row(1, 100, "unknown", false)))
}
-
- checkDefaultValues(
- createExec.columns,
- Array(
- null,
- new ColumnDefaultValue(
- "(100 + 23)",
- new GeneralScalarExpression(
- "+",
- Array(LiteralValue(100, IntegerType), LiteralValue(23,
IntegerType))),
- LiteralValue(123, IntegerType)),
- new ColumnDefaultValue(
- "('h' || 'r')",
- new GeneralScalarExpression(
- "CONCAT",
- Array(
- LiteralValue(UTF8String.fromString("h"), StringType),
- LiteralValue(UTF8String.fromString("r"), StringType))),
- LiteralValue(UTF8String.fromString("hr"), StringType)),
- new ColumnDefaultValue(
- "CAST(1 AS BOOLEAN)",
- new V2Cast(LiteralValue(1, IntegerType), IntegerType, BooleanType),
- LiteralValue(true, BooleanType))))
-
- val df1 = Seq(1).toDF("id")
- df1.writeTo(tableName).append()
-
- sql(s"ALTER TABLE $tableName ALTER COLUMN dep SET DEFAULT ('i' || 't')")
-
- val df2 = Seq(2).toDF("id")
- df2.writeTo(tableName).append()
-
- checkAnswer(
- sql(s"SELECT * FROM $tableName"),
- Seq(
- Row(1, 123, "hr", true),
- Row(2, 123, "it", true)))
-
- val replaceExec = executeAndKeepPhysicalPlan[ReplaceTableExec] {
- sql(
- s"""
- |REPLACE TABLE $tableName (
- | id INT,
- | salary INT DEFAULT (50 * 2),
- | dep STRING DEFAULT ('un' || 'known'),
- | active BOOLEAN DEFAULT CAST(0 AS BOOLEAN)
- |) USING foo
- |""".stripMargin)
- }
-
- checkDefaultValues(
- replaceExec.columns,
- Array(
- null,
- new ColumnDefaultValue(
- "(50 * 2)",
- new GeneralScalarExpression(
- "*",
- Array(LiteralValue(50, IntegerType), LiteralValue(2,
IntegerType))),
- LiteralValue(100, IntegerType)),
- new ColumnDefaultValue(
- "('un' || 'known')",
- new GeneralScalarExpression(
- "CONCAT",
- Array(
- LiteralValue(UTF8String.fromString("un"), StringType),
- LiteralValue(UTF8String.fromString("known"), StringType))),
- LiteralValue(UTF8String.fromString("unknown"), StringType)),
- new ColumnDefaultValue(
- "CAST(0 AS BOOLEAN)",
- new V2Cast(LiteralValue(0, IntegerType), IntegerType, BooleanType),
- LiteralValue(false, BooleanType))))
-
- val df3 = Seq(1).toDF("id")
- df3.writeTo(tableName).append()
-
- checkAnswer(
- sql(s"SELECT * FROM $tableName"),
- Seq(Row(1, 100, "unknown", false)))
}
}
test("alter table with complex foldable default values") {
val tableName = "testcat.ns1.ns2.tbl"
- withTable(tableName) {
- sql(
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> "true") {
+ withTable(tableName) {
+ sql(
s"""
|CREATE TABLE $tableName (
| dummy INT
|) USING foo
|""".stripMargin)
- val alterExec = executeAndKeepPhysicalPlan[AlterTableExec] {
- sql(s"ALTER TABLE $tableName ADD COLUMNS (" +
- s"salary INT DEFAULT (100 + 23), " +
- s"dep STRING DEFAULT ('h' || 'r'), " +
- s"active BOOLEAN DEFAULT CAST(1 AS BOOLEAN))")
+ val alterExec = executeAndKeepPhysicalPlan[AlterTableExec] {
+ sql(s"ALTER TABLE $tableName ADD COLUMNS (" +
+ s"salary INT DEFAULT (100 + 23), " +
+ s"dep STRING DEFAULT ('h' || 'r'), " +
+ s"active BOOLEAN DEFAULT CAST(1 AS BOOLEAN))")
+ }
+
+ checkDefaultValues(
+ alterExec.changes.map(_.asInstanceOf[AddColumn]).toArray,
+ Array(
+ new ColumnDefaultValue(
+ "(100 + 23)",
+ new GeneralScalarExpression(
+ "+",
+ Array(LiteralValue(100, IntegerType), LiteralValue(23,
IntegerType))),
+ LiteralValue(123, IntegerType)),
+ new ColumnDefaultValue(
+ "('h' || 'r')",
+ new GeneralScalarExpression(
+ "CONCAT",
+ Array(
+ LiteralValue(UTF8String.fromString("h"), StringType),
+ LiteralValue(UTF8String.fromString("r"), StringType))),
+ LiteralValue(UTF8String.fromString("hr"), StringType)),
+ new ColumnDefaultValue(
+ "CAST(1 AS BOOLEAN)",
+ new V2Cast(LiteralValue(1, IntegerType), IntegerType,
BooleanType),
+ LiteralValue(true, BooleanType))))
}
-
- checkDefaultValues(
- alterExec.changes.map(_.asInstanceOf[AddColumn]).toArray,
- Array(
- new ColumnDefaultValue(
- "(100 + 23)",
- new GeneralScalarExpression(
- "+",
- Array(LiteralValue(100, IntegerType), LiteralValue(23,
IntegerType))),
- LiteralValue(123, IntegerType)),
- new ColumnDefaultValue(
- "('h' || 'r')",
- new GeneralScalarExpression(
- "CONCAT",
- Array(
- LiteralValue(UTF8String.fromString("h"), StringType),
- LiteralValue(UTF8String.fromString("r"), StringType))),
- LiteralValue(UTF8String.fromString("hr"), StringType)),
- new ColumnDefaultValue(
- "CAST(1 AS BOOLEAN)",
- new V2Cast(LiteralValue(1, IntegerType), IntegerType, BooleanType),
- LiteralValue(true, BooleanType))))
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
