This is an automated email from the ASF dual-hosted git repository.
zml1206 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 6f179f5814 [MINOR] Remove stale complex type scan fallback config from
tests (#12476)
6f179f5814 is described below
commit 6f179f5814fee88d13f712ebcecdc7b470d856fb
Author: Mingliang Zhu <[email protected]>
AuthorDate: Thu Jul 9 10:10:28 2026 +0800
[MINOR] Remove stale complex type scan fallback config from tests (#12476)
---
.../VeloxOrcDataTypeValidationSuite.scala | 172 +++++------
.../VeloxParquetDataTypeValidationSuite.scala | 172 +++++------
.../apache/gluten/execution/VeloxScanSuite.scala | 6 +-
.../execution/GlutenQueryComparisonTest.scala | 3 -
.../hive/execution/GlutenHiveSQLQueryCHSuite.scala | 41 ++-
.../hive/execution/GlutenHiveSQLQuerySuite.scala | 3 +-
.../hive/execution/GlutenHiveSQLQueryCHSuite.scala | 41 ++-
.../spark/sql/sources/GlutenInsertSuite.scala | 334 ++++++++++-----------
.../hive/execution/GlutenHiveSQLQueryCHSuite.scala | 41 ++-
.../spark/sql/sources/GlutenInsertSuite.scala | 334 ++++++++++-----------
.../hive/execution/GlutenHiveSQLQueryCHSuite.scala | 41 ++-
.../spark/sql/sources/GlutenInsertSuite.scala | 334 ++++++++++-----------
.../hive/execution/GlutenHiveSQLQueryCHSuite.scala | 41 ++-
.../spark/sql/sources/GlutenInsertSuite.scala | 334 ++++++++++-----------
14 files changed, 915 insertions(+), 982 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxOrcDataTypeValidationSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxOrcDataTypeValidationSuite.scala
index be8ad6cd4b..87c641a2dc 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxOrcDataTypeValidationSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxOrcDataTypeValidationSuite.scala
@@ -331,108 +331,102 @@ class VeloxOrcDataTypeValidationSuite extends
VeloxWholeStageTransformerSuite {
}
test("Array type") {
- withSQLConf(("spark.gluten.sql.complexType.scan.fallback.enabled",
"false")) {
- // Validation: BatchScan.
- runQueryAndCompare("select array from type1") {
- checkGlutenPlan[BatchScanExecTransformer]
- }
+ // Validation: BatchScan.
+ runQueryAndCompare("select array from type1") {
+ checkGlutenPlan[BatchScanExecTransformer]
+ }
- // Validation: BatchScan Project Aggregate Expand Sort Limit
- runQueryAndCompare(
- "select int, array from type1 " +
- " group by grouping sets(int, array) sort by array, int limit 1") {
- df =>
- {
- val executedPlan = getExecutedPlan(df)
- assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
- }
- }
+ // Validation: BatchScan Project Aggregate Expand Sort Limit
+ runQueryAndCompare(
+ "select int, array from type1 " +
+ " group by grouping sets(int, array) sort by array, int limit 1") {
+ df =>
+ {
+ val executedPlan = getExecutedPlan(df)
+ assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
+ }
+ }
- // Validation: BroadHashJoin, Filter, Project
- super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
- runQueryAndCompare(
- "select type1.array from type1," +
- " type2 where type1.array = type2.array") { _ => }
+ // Validation: BroadHashJoin, Filter, Project
+ super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
+ runQueryAndCompare(
+ "select type1.array from type1," +
+ " type2 where type1.array = type2.array") { _ => }
- // Validation: ShuffledHashJoin, Filter, Project
- super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
- runQueryAndCompare(
- "select type1.array from type1," +
- " type2 where type1.array = type2.array") { _ => }
- }
+ // Validation: ShuffledHashJoin, Filter, Project
+ super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
+ runQueryAndCompare(
+ "select type1.array from type1," +
+ " type2 where type1.array = type2.array") { _ => }
}
test("Map type") {
- withSQLConf(("spark.gluten.sql.complexType.scan.fallback.enabled",
"false")) {
- // Validation: BatchScan Project Limit
- runQueryAndCompare("select map from type1 limit 1") {
- df =>
- {
- val executedPlan = getExecutedPlan(df)
- assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
- }
- }
- // Validation: BatchScan Project Aggregate Sort Limit
- // TODO validate Expand operator support map type ?
- runQueryAndCompare(
- "select map['key'] from type1 group by map['key']" +
- " sort by map['key'] limit 1") {
- df =>
- {
- val executedPlan = getExecutedPlan(df)
- assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
- }
- }
+ // Validation: BatchScan Project Limit
+ runQueryAndCompare("select map from type1 limit 1") {
+ df =>
+ {
+ val executedPlan = getExecutedPlan(df)
+ assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
+ }
+ }
+ // Validation: BatchScan Project Aggregate Sort Limit
+ // TODO validate Expand operator support map type ?
+ runQueryAndCompare(
+ "select map['key'] from type1 group by map['key']" +
+ " sort by map['key'] limit 1") {
+ df =>
+ {
+ val executedPlan = getExecutedPlan(df)
+ assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
+ }
+ }
- // Validation: BroadHashJoin, Filter, Project
- super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
- runQueryAndCompare(
- "select type1.map['key'] from type1," +
- " type2 where type1.map['key'] = type2.map['key']") { _ => }
+ // Validation: BroadHashJoin, Filter, Project
+ super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
+ runQueryAndCompare(
+ "select type1.map['key'] from type1," +
+ " type2 where type1.map['key'] = type2.map['key']") { _ => }
- // Validation: ShuffledHashJoin, Filter, Project
- super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
- runQueryAndCompare(
- "select type1.map['key'] from type1," +
- " type2 where type1.map['key'] = type2.map['key']") { _ => }
- }
+ // Validation: ShuffledHashJoin, Filter, Project
+ super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
+ runQueryAndCompare(
+ "select type1.map['key'] from type1," +
+ " type2 where type1.map['key'] = type2.map['key']") { _ => }
}
test("Struct type") {
- withSQLConf(("spark.gluten.sql.complexType.scan.fallback.enabled",
"false")) {
- // Validation: BatchScan Project Limit
- runQueryAndCompare("select struct from type1") {
- df =>
- {
- val executedPlan = getExecutedPlan(df)
- assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
- }
- }
- // Validation: BatchScan Project Aggregate Sort Limit
- // TODO validate Expand operator support Struct type ?
- runQueryAndCompare(
- "select int, struct.struct_1 from type1 " +
- "sort by struct.struct_1 limit 1") {
- df =>
- {
- val executedPlan = getExecutedPlan(df)
- assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
- assert(executedPlan.exists(plan =>
plan.isInstanceOf[ProjectExecTransformer]))
- }
- }
+ // Validation: BatchScan Project Limit
+ runQueryAndCompare("select struct from type1") {
+ df =>
+ {
+ val executedPlan = getExecutedPlan(df)
+ assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
+ }
+ }
+ // Validation: BatchScan Project Aggregate Sort Limit
+ // TODO validate Expand operator support Struct type ?
+ runQueryAndCompare(
+ "select int, struct.struct_1 from type1 " +
+ "sort by struct.struct_1 limit 1") {
+ df =>
+ {
+ val executedPlan = getExecutedPlan(df)
+ assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
+ assert(executedPlan.exists(plan =>
plan.isInstanceOf[ProjectExecTransformer]))
+ }
+ }
- // Validation: BroadHashJoin, Filter, Project
- super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
- runQueryAndCompare(
- "select type1.struct.struct_1 from type1," +
- " type2 where type1.struct.struct_1 = type2.struct.struct_1") { _ =>
}
+ // Validation: BroadHashJoin, Filter, Project
+ super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
+ runQueryAndCompare(
+ "select type1.struct.struct_1 from type1," +
+ " type2 where type1.struct.struct_1 = type2.struct.struct_1") { _ => }
- // Validation: ShuffledHashJoin, Filter, Project
- super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
- runQueryAndCompare(
- "select type1.struct.struct_1 from type1," +
- " type2 where type1.struct.struct_1 = type2.struct.struct_1") { _ =>
}
- }
+ // Validation: ShuffledHashJoin, Filter, Project
+ super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
+ runQueryAndCompare(
+ "select type1.struct.struct_1 from type1," +
+ " type2 where type1.struct.struct_1 = type2.struct.struct_1") { _ => }
}
test("Decimal type") {
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxParquetDataTypeValidationSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxParquetDataTypeValidationSuite.scala
index 945b1244ae..1408b4c651 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxParquetDataTypeValidationSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxParquetDataTypeValidationSuite.scala
@@ -332,108 +332,102 @@ class VeloxParquetDataTypeValidationSuite extends
VeloxWholeStageTransformerSuit
}
test("Array type") {
- withSQLConf(("spark.gluten.sql.complexType.scan.fallback.enabled",
"false")) {
- // Validation: BatchScan.
- runQueryAndCompare("select array from type1") {
- checkGlutenPlan[BatchScanExecTransformer]
- }
+ // Validation: BatchScan.
+ runQueryAndCompare("select array from type1") {
+ checkGlutenPlan[BatchScanExecTransformer]
+ }
- // Validation: BatchScan Project Aggregate Expand Sort Limit
- runQueryAndCompare(
- "select int, array from type1 " +
- " group by grouping sets(int, array) sort by array, int limit 1") {
- df =>
- {
- val executedPlan = getExecutedPlan(df)
- assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
- }
- }
+ // Validation: BatchScan Project Aggregate Expand Sort Limit
+ runQueryAndCompare(
+ "select int, array from type1 " +
+ " group by grouping sets(int, array) sort by array, int limit 1") {
+ df =>
+ {
+ val executedPlan = getExecutedPlan(df)
+ assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
+ }
+ }
- // Validation: BroadHashJoin, Filter, Project
- super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
- runQueryAndCompare(
- "select type1.array from type1," +
- " type2 where type1.array = type2.array") { _ => }
+ // Validation: BroadHashJoin, Filter, Project
+ super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
+ runQueryAndCompare(
+ "select type1.array from type1," +
+ " type2 where type1.array = type2.array") { _ => }
- // Validation: ShuffledHashJoin, Filter, Project
- super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
- runQueryAndCompare(
- "select type1.array from type1," +
- " type2 where type1.array = type2.array") { _ => }
- }
+ // Validation: ShuffledHashJoin, Filter, Project
+ super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
+ runQueryAndCompare(
+ "select type1.array from type1," +
+ " type2 where type1.array = type2.array") { _ => }
}
test("Map type") {
- withSQLConf(("spark.gluten.sql.complexType.scan.fallback.enabled",
"false")) {
- // Validation: BatchScan Project Limit
- runQueryAndCompare("select map from type1 limit 1") {
- df =>
- {
- val executedPlan = getExecutedPlan(df)
- assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
- }
- }
- // Validation: BatchScan Project Aggregate Sort Limit
- // TODO validate Expand operator support map type ?
- runQueryAndCompare(
- "select map['key'] from type1 group by map['key']" +
- " sort by map['key'] limit 1") {
- df =>
- {
- val executedPlan = getExecutedPlan(df)
- assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
- }
- }
+ // Validation: BatchScan Project Limit
+ runQueryAndCompare("select map from type1 limit 1") {
+ df =>
+ {
+ val executedPlan = getExecutedPlan(df)
+ assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
+ }
+ }
+ // Validation: BatchScan Project Aggregate Sort Limit
+ // TODO validate Expand operator support map type ?
+ runQueryAndCompare(
+ "select map['key'] from type1 group by map['key']" +
+ " sort by map['key'] limit 1") {
+ df =>
+ {
+ val executedPlan = getExecutedPlan(df)
+ assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
+ }
+ }
- // Validation: BroadHashJoin, Filter, Project
- super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
- runQueryAndCompare(
- "select type1.map['key'] from type1," +
- " type2 where type1.map['key'] = type2.map['key']") { _ => }
+ // Validation: BroadHashJoin, Filter, Project
+ super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
+ runQueryAndCompare(
+ "select type1.map['key'] from type1," +
+ " type2 where type1.map['key'] = type2.map['key']") { _ => }
- // Validation: ShuffledHashJoin, Filter, Project
- super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
- runQueryAndCompare(
- "select type1.map['key'] from type1," +
- " type2 where type1.map['key'] = type2.map['key']") { _ => }
- }
+ // Validation: ShuffledHashJoin, Filter, Project
+ super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
+ runQueryAndCompare(
+ "select type1.map['key'] from type1," +
+ " type2 where type1.map['key'] = type2.map['key']") { _ => }
}
test("Struct type") {
- withSQLConf(("spark.gluten.sql.complexType.scan.fallback.enabled",
"false")) {
- // Validation: BatchScan Project Limit
- runQueryAndCompare("select struct from type1") {
- df =>
- {
- val executedPlan = getExecutedPlan(df)
- assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
- }
- }
- // Validation: BatchScan Project Aggregate Sort Limit
- // TODO validate Expand operator support Struct type ?
- runQueryAndCompare(
- "select int, struct.struct_1 from type1 " +
- "sort by struct.struct_1 limit 1") {
- df =>
- {
- val executedPlan = getExecutedPlan(df)
- assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
- assert(executedPlan.exists(plan =>
plan.isInstanceOf[ProjectExecTransformer]))
- }
- }
+ // Validation: BatchScan Project Limit
+ runQueryAndCompare("select struct from type1") {
+ df =>
+ {
+ val executedPlan = getExecutedPlan(df)
+ assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
+ }
+ }
+ // Validation: BatchScan Project Aggregate Sort Limit
+ // TODO validate Expand operator support Struct type ?
+ runQueryAndCompare(
+ "select int, struct.struct_1 from type1 " +
+ "sort by struct.struct_1 limit 1") {
+ df =>
+ {
+ val executedPlan = getExecutedPlan(df)
+ assert(executedPlan.exists(plan =>
plan.isInstanceOf[BatchScanExecTransformer]))
+ assert(executedPlan.exists(plan =>
plan.isInstanceOf[ProjectExecTransformer]))
+ }
+ }
- // Validation: BroadHashJoin, Filter, Project
- super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
- runQueryAndCompare(
- "select type1.struct.struct_1 from type1," +
- " type2 where type1.struct.struct_1 = type2.struct.struct_1") { _ =>
}
+ // Validation: BroadHashJoin, Filter, Project
+ super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "10M")
+ runQueryAndCompare(
+ "select type1.struct.struct_1 from type1," +
+ " type2 where type1.struct.struct_1 = type2.struct.struct_1") { _ => }
- // Validation: ShuffledHashJoin, Filter, Project
- super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
- runQueryAndCompare(
- "select type1.struct.struct_1 from type1," +
- " type2 where type1.struct.struct_1 = type2.struct.struct_1") { _ =>
}
- }
+ // Validation: ShuffledHashJoin, Filter, Project
+ super.sparkConf.set("spark.sql.autoBroadcastJoinThreshold", "-1")
+ runQueryAndCompare(
+ "select type1.struct.struct_1 from type1," +
+ " type2 where type1.struct.struct_1 = type2.struct.struct_1") { _ => }
}
test("Decimal type") {
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxScanSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxScanSuite.scala
index 48e26c9a1f..0ba22bf4df 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxScanSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxScanSuite.scala
@@ -219,8 +219,7 @@ class VeloxScanSuite extends
VeloxWholeStageTransformerSuite {
test("parquet index based schema evolution") {
withSQLConf(
- VeloxConfig.PARQUET_USE_COLUMN_NAMES.key -> "false",
- "spark.gluten.sql.complexType.scan.fallback.enabled" -> "false") {
+ VeloxConfig.PARQUET_USE_COLUMN_NAMES.key -> "false") {
withTempDir {
dir =>
val path = dir.getCanonicalPath
@@ -263,8 +262,7 @@ class VeloxScanSuite extends
VeloxWholeStageTransformerSuite {
test("ORC index based schema evolution") {
withSQLConf(
- VeloxConfig.ORC_USE_COLUMN_NAMES.key -> "false",
- "spark.gluten.sql.complexType.scan.fallback.enabled" -> "false") {
+ VeloxConfig.ORC_USE_COLUMN_NAMES.key -> "false") {
withTempDir {
dir =>
val path = dir.getCanonicalPath
diff --git
a/gluten-substrait/src/test/scala/org/apache/gluten/execution/GlutenQueryComparisonTest.scala
b/gluten-substrait/src/test/scala/org/apache/gluten/execution/GlutenQueryComparisonTest.scala
index af529edd2e..e175b314bd 100644
---
a/gluten-substrait/src/test/scala/org/apache/gluten/execution/GlutenQueryComparisonTest.scala
+++
b/gluten-substrait/src/test/scala/org/apache/gluten/execution/GlutenQueryComparisonTest.scala
@@ -101,9 +101,6 @@ abstract class GlutenQueryComparisonTest extends
GlutenQueryTest {
val df = dataframe()
expected = df.collect()
}
- // By default, we will fallback complex type scan but here we should allow
- // to test support of complex type
- spark.conf.set("spark.gluten.sql.complexType.scan.fallback.enabled",
"false")
val df = dataframe()
if (cache) {
df.cache()
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
index 264a7e7836..1020294d88 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
@@ -38,8 +38,7 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
testGluten("5182: Fix failed to parse post join filters") {
withSQLConf(
- "spark.sql.hive.convertMetastoreParquet" -> "false",
- "spark.gluten.sql.complexType.scan.fallback.enabled" -> "false") {
+ "spark.sql.hive.convertMetastoreParquet" -> "false") {
sql("DROP TABLE IF EXISTS test_5182_0;")
sql("DROP TABLE IF EXISTS test_5182_1;")
sql(
@@ -76,27 +75,23 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
}
testGluten("5249: Reading csv may throw Unexpected empty column") {
- withSQLConf(
- "spark.gluten.sql.complexType.scan.fallback.enabled" -> "false"
- ) {
- sql("DROP TABLE IF EXISTS test_5249;")
- sql(
- "CREATE TABLE test_5249 (name STRING, uid STRING) " +
- "ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' " +
- "STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' " +
- "OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';")
- sql("INSERT INTO test_5249 VALUES('name_1', 'id_1');")
- val df = spark.sql(
- "SELECT name, uid, count(distinct uid) total_uid_num from test_5249 " +
- "group by name, uid with cube;")
- checkAnswer(
- df,
- Seq(
- Row("name_1", "id_1", 1),
- Row("name_1", null, 1),
- Row(null, "id_1", 1),
- Row(null, null, 1)))
- }
+ sql("DROP TABLE IF EXISTS test_5249;")
+ sql(
+ "CREATE TABLE test_5249 (name STRING, uid STRING) " +
+ "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
" +
+ "STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' " +
+ "OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';")
+ sql("INSERT INTO test_5249 VALUES('name_1', 'id_1');")
+ val df = spark.sql(
+ "SELECT name, uid, count(distinct uid) total_uid_num from test_5249 " +
+ "group by name, uid with cube;")
+ checkAnswer(
+ df,
+ Seq(
+ Row("name_1", "id_1", 1),
+ Row("name_1", null, 1),
+ Row(null, "id_1", 1),
+ Row(null, null, 1)))
spark.sessionState.catalog.dropTable(
TableIdentifier("test_5249"),
ignoreIfNotExists = true,
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
index 97ec1cfba8..e0ca4ae441 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQuerySuite.scala
@@ -118,8 +118,7 @@ class GlutenHiveSQLQuerySuite extends
GlutenHiveSQLQuerySuiteBase {
testGluten("avoid unnecessary filter binding for subfield during scan") {
withSQLConf(
- "spark.sql.hive.convertMetastoreParquet" -> "false",
- "spark.gluten.sql.complexType.scan.fallback.enabled" -> "false") {
+ "spark.sql.hive.convertMetastoreParquet" -> "false") {
sql("DROP TABLE IF EXISTS test_subfield")
sql(
"CREATE TABLE test_subfield (name STRING, favorite_color STRING," +
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
index 264a7e7836..1020294d88 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
@@ -38,8 +38,7 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
testGluten("5182: Fix failed to parse post join filters") {
withSQLConf(
- "spark.sql.hive.convertMetastoreParquet" -> "false",
- "spark.gluten.sql.complexType.scan.fallback.enabled" -> "false") {
+ "spark.sql.hive.convertMetastoreParquet" -> "false") {
sql("DROP TABLE IF EXISTS test_5182_0;")
sql("DROP TABLE IF EXISTS test_5182_1;")
sql(
@@ -76,27 +75,23 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
}
testGluten("5249: Reading csv may throw Unexpected empty column") {
- withSQLConf(
- "spark.gluten.sql.complexType.scan.fallback.enabled" -> "false"
- ) {
- sql("DROP TABLE IF EXISTS test_5249;")
- sql(
- "CREATE TABLE test_5249 (name STRING, uid STRING) " +
- "ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' " +
- "STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' " +
- "OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';")
- sql("INSERT INTO test_5249 VALUES('name_1', 'id_1');")
- val df = spark.sql(
- "SELECT name, uid, count(distinct uid) total_uid_num from test_5249 " +
- "group by name, uid with cube;")
- checkAnswer(
- df,
- Seq(
- Row("name_1", "id_1", 1),
- Row("name_1", null, 1),
- Row(null, "id_1", 1),
- Row(null, null, 1)))
- }
+ sql("DROP TABLE IF EXISTS test_5249;")
+ sql(
+ "CREATE TABLE test_5249 (name STRING, uid STRING) " +
+ "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
" +
+ "STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' " +
+ "OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';")
+ sql("INSERT INTO test_5249 VALUES('name_1', 'id_1');")
+ val df = spark.sql(
+ "SELECT name, uid, count(distinct uid) total_uid_num from test_5249 " +
+ "group by name, uid with cube;")
+ checkAnswer(
+ df,
+ Seq(
+ Row("name_1", "id_1", 1),
+ Row("name_1", null, 1),
+ Row(null, "id_1", 1),
+ Row(null, null, 1)))
spark.sessionState.catalog.dropTable(
TableIdentifier("test_5249"),
ignoreIfNotExists = true,
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
index e6cc2937a4..7a5c6a53bb 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
@@ -433,194 +433,188 @@ class GlutenInsertSuite
}
testGluten("SPARK-39557 INSERT INTO statements with tables with array
defaults") {
- withSQLConf("spark.gluten.sql.complexType.scan.fallback.enabled" ->
"false") {
- import testImplicits._
- // Positive tests: array types are supported as default values.
- case class Config(dataSource: String, useDataFrames: Boolean = false)
- Seq(
- Config("parquet"),
- Config("parquet", useDataFrames = true),
- Config("orc"),
- Config("orc", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- sql("alter table t add column s array<int> default array(1, 2)")
- checkAnswer(spark.table("t"), Row(false, null))
- sql("insert into t(i) values (true)")
- checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Seq(1, 2))))
+ import testImplicits._
+ // Positive tests: array types are supported as default values.
+ case class Config(dataSource: String, useDataFrames: Boolean = false)
+ Seq(
+ Config("parquet"),
+ Config("parquet", useDataFrames = true),
+ Config("orc"),
+ Config("orc", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
- // Negative tests: provided array element types must match their
corresponding DEFAULT
- // declarations, if applicable.
- val incompatibleDefault =
- "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
- "table column s has a DEFAULT value with type"
- Seq(Config("parquet"), Config("parquet", true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- assert(intercept[AnalysisException] {
- sql("alter table t add column s array<int> default array('abc',
'def')")
- }.getMessage.contains(incompatibleDefault))
+ sql("alter table t add column s array<int> default array(1, 2)")
+ checkAnswer(spark.table("t"), Row(false, null))
+ sql("insert into t(i) values (true)")
+ checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true, Seq(1,
2))))
+ }
+ }
+ // Negative tests: provided array element types must match their
corresponding DEFAULT
+ // declarations, if applicable.
+ val incompatibleDefault =
+ "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
+ "table column s has a DEFAULT value with type"
+ Seq(Config("parquet"), Config("parquet", true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
+ assert(intercept[AnalysisException] {
+ sql("alter table t add column s array<int> default array('abc',
'def')")
+ }.getMessage.contains(incompatibleDefault))
+ }
}
}
testGluten("SPARK-39557 INSERT INTO statements with tables with struct
defaults") {
- withSQLConf("spark.gluten.sql.complexType.scan.fallback.enabled" ->
"false") {
-
- import testImplicits._
- // Positive tests: struct types are supported as default values.
- case class Config(dataSource: String, useDataFrames: Boolean = false)
- Seq(
- Config("parquet"),
- Config("parquet", useDataFrames = true),
- Config("orc"),
- Config("orc", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- sql(
- "alter table t add column s struct<x boolean, y string> default
struct(true, 'abc')")
- checkAnswer(spark.table("t"), Row(false, null))
- sql("insert into t(i) values (true)")
- checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Row(true, "abc"))))
+
+ import testImplicits._
+ // Positive tests: struct types are supported as default values.
+ case class Config(dataSource: String, useDataFrames: Boolean = false)
+ Seq(
+ Config("parquet"),
+ Config("parquet", useDataFrames = true),
+ Config("orc"),
+ Config("orc", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
+ sql(
+ "alter table t add column s struct<x boolean, y string> default
struct(true, 'abc')")
+ checkAnswer(spark.table("t"), Row(false, null))
+ sql("insert into t(i) values (true)")
+ checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Row(true, "abc"))))
+ }
+ }
- // Negative tests: provided map element types must match their
corresponding DEFAULT
- // declarations, if applicable.
- val incompatibleDefault =
- "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
- "table column s has a DEFAULT value with type"
- Seq(Config("parquet"), Config("parquet", true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- assert(intercept[AnalysisException] {
- sql("alter table t add column s struct<x boolean, y string>
default struct(42, 56)")
- }.getMessage.contains(incompatibleDefault))
+ // Negative tests: provided map element types must match their
corresponding DEFAULT
+ // declarations, if applicable.
+ val incompatibleDefault =
+ "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
+ "table column s has a DEFAULT value with type"
+ Seq(Config("parquet"), Config("parquet", true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
+ assert(intercept[AnalysisException] {
+ sql("alter table t add column s struct<x boolean, y string>
default struct(42, 56)")
+ }.getMessage.contains(incompatibleDefault))
+ }
}
}
ignoreGluten("SPARK-39557 INSERT INTO statements with tables with map
defaults") {
- withSQLConf("spark.gluten.sql.complexType.scan.fallback.enabled" ->
"false") {
-
- import testImplicits._
- // Positive tests: map types are supported as default values.
- case class Config(dataSource: String, useDataFrames: Boolean = false)
- Seq(
- Config("parquet"),
- Config("parquet", useDataFrames = true),
- Config("orc"),
- Config("orc", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- sql("alter table t add column s map<boolean, string> default
map(true, 'abc')")
- checkAnswer(spark.table("t"), Row(false, null))
- sql("insert into t(i) select true")
- checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Map(true -> "abc"))))
+
+ import testImplicits._
+ // Positive tests: map types are supported as default values.
+ case class Config(dataSource: String, useDataFrames: Boolean = false)
+ Seq(
+ Config("parquet"),
+ Config("parquet", useDataFrames = true),
+ Config("orc"),
+ Config("orc", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- withTable("t") {
- sql(s"""
- create table t(
- i int,
- s struct<
- x array<
- struct<a int, b int>>,
- y array<
- map<boolean, string>>>
- default struct(
- array(
- struct(1, 2)),
- array(
- map(false, 'def', true, 'jkl'))))
- using ${config.dataSource}""")
- sql("insert into t select 1, default")
- sql("alter table t alter column s drop default")
- if (config.useDataFrames) {
- Seq((2, null)).toDF.write.insertInto("t")
- } else {
- sql("insert into t select 2, default")
- }
- sql("""
- alter table t alter column s
- set default struct(
+ sql("alter table t add column s map<boolean, string> default
map(true, 'abc')")
+ checkAnswer(spark.table("t"), Row(false, null))
+ sql("insert into t(i) select true")
+ checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Map(true -> "abc"))))
+ }
+ withTable("t") {
+ sql(s"""
+ create table t(
+ i int,
+ s struct<
+ x array<
+ struct<a int, b int>>,
+ y array<
+ map<boolean, string>>>
+ default struct(
array(
- struct(3, 4)),
+ struct(1, 2)),
array(
- map(false, 'mno', true, 'pqr')))""")
- sql("insert into t select 3, default")
- sql("""
- alter table t
- add column t array<
- map<boolean, string>>
- default array(
- map(true, 'xyz'))""")
- sql("insert into t(i, s) select 4, default")
- checkAnswer(
- spark.table("t"),
- Seq(
- Row(1, Row(Seq(Row(1, 2)), Seq(Map(false -> "def", true ->
"jkl"))), null),
- Row(2, null, null),
- Row(3, Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true ->
"pqr"))), null),
- Row(
- 4,
- Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true -> "pqr"))),
- Seq(Map(true -> "xyz")))
- )
- )
+ map(false, 'def', true, 'jkl'))))
+ using ${config.dataSource}""")
+ sql("insert into t select 1, default")
+ sql("alter table t alter column s drop default")
+ if (config.useDataFrames) {
+ Seq((2, null)).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select 2, default")
}
- }
- // Negative tests: provided map element types must match their
corresponding DEFAULT
- // declarations, if applicable.
- val incompatibleDefault =
- "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
- "table column s has a DEFAULT value with type"
- Seq(Config("parquet"), Config("parquet", true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- assert(intercept[AnalysisException] {
- sql("alter table t add column s map<boolean, string> default
map(42, 56)")
- }.getMessage.contains(incompatibleDefault))
+ sql("""
+ alter table t alter column s
+ set default struct(
+ array(
+ struct(3, 4)),
+ array(
+ map(false, 'mno', true, 'pqr')))""")
+ sql("insert into t select 3, default")
+ sql("""
+ alter table t
+ add column t array<
+ map<boolean, string>>
+ default array(
+ map(true, 'xyz'))""")
+ sql("insert into t(i, s) select 4, default")
+ checkAnswer(
+ spark.table("t"),
+ Seq(
+ Row(1, Row(Seq(Row(1, 2)), Seq(Map(false -> "def", true ->
"jkl"))), null),
+ Row(2, null, null),
+ Row(3, Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true ->
"pqr"))), null),
+ Row(
+ 4,
+ Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true -> "pqr"))),
+ Seq(Map(true -> "xyz")))
+ )
+ )
+ }
+ }
+ // Negative tests: provided map element types must match their
corresponding DEFAULT
+ // declarations, if applicable.
+ val incompatibleDefault =
+ "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
+ "table column s has a DEFAULT value with type"
+ Seq(Config("parquet"), Config("parquet", true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
+ assert(intercept[AnalysisException] {
+ sql("alter table t add column s map<boolean, string> default
map(42, 56)")
+ }.getMessage.contains(incompatibleDefault))
+ }
}
}
}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
index 264a7e7836..1020294d88 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
@@ -38,8 +38,7 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
testGluten("5182: Fix failed to parse post join filters") {
withSQLConf(
- "spark.sql.hive.convertMetastoreParquet" -> "false",
- "spark.gluten.sql.complexType.scan.fallback.enabled" -> "false") {
+ "spark.sql.hive.convertMetastoreParquet" -> "false") {
sql("DROP TABLE IF EXISTS test_5182_0;")
sql("DROP TABLE IF EXISTS test_5182_1;")
sql(
@@ -76,27 +75,23 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
}
testGluten("5249: Reading csv may throw Unexpected empty column") {
- withSQLConf(
- "spark.gluten.sql.complexType.scan.fallback.enabled" -> "false"
- ) {
- sql("DROP TABLE IF EXISTS test_5249;")
- sql(
- "CREATE TABLE test_5249 (name STRING, uid STRING) " +
- "ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' " +
- "STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' " +
- "OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';")
- sql("INSERT INTO test_5249 VALUES('name_1', 'id_1');")
- val df = spark.sql(
- "SELECT name, uid, count(distinct uid) total_uid_num from test_5249 " +
- "group by name, uid with cube;")
- checkAnswer(
- df,
- Seq(
- Row("name_1", "id_1", 1),
- Row("name_1", null, 1),
- Row(null, "id_1", 1),
- Row(null, null, 1)))
- }
+ sql("DROP TABLE IF EXISTS test_5249;")
+ sql(
+ "CREATE TABLE test_5249 (name STRING, uid STRING) " +
+ "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
" +
+ "STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' " +
+ "OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';")
+ sql("INSERT INTO test_5249 VALUES('name_1', 'id_1');")
+ val df = spark.sql(
+ "SELECT name, uid, count(distinct uid) total_uid_num from test_5249 " +
+ "group by name, uid with cube;")
+ checkAnswer(
+ df,
+ Seq(
+ Row("name_1", "id_1", 1),
+ Row("name_1", null, 1),
+ Row(null, "id_1", 1),
+ Row(null, null, 1)))
spark.sessionState.catalog.dropTable(
TableIdentifier("test_5249"),
ignoreIfNotExists = true,
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
index 38e032aec3..75e2c39f79 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
@@ -405,194 +405,188 @@ class GlutenInsertSuite
}
testGluten("SPARK-39557 INSERT INTO statements with tables with array
defaults") {
- withSQLConf("spark.gluten.sql.complexType.scan.fallback.enabled" ->
"false") {
- import testImplicits._
- // Positive tests: array types are supported as default values.
- case class Config(dataSource: String, useDataFrames: Boolean = false)
- Seq(
- Config("parquet"),
- Config("parquet", useDataFrames = true),
- Config("orc"),
- Config("orc", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- sql("alter table t add column s array<int> default array(1, 2)")
- checkAnswer(spark.table("t"), Row(false, null))
- sql("insert into t(i) values (true)")
- checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Seq(1, 2))))
+ import testImplicits._
+ // Positive tests: array types are supported as default values.
+ case class Config(dataSource: String, useDataFrames: Boolean = false)
+ Seq(
+ Config("parquet"),
+ Config("parquet", useDataFrames = true),
+ Config("orc"),
+ Config("orc", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
- // Negative tests: provided array element types must match their
corresponding DEFAULT
- // declarations, if applicable.
- val incompatibleDefault =
- "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
- "table column `s` has a DEFAULT value"
- Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- assert(intercept[AnalysisException] {
- sql("alter table t add column s array<int> default array('abc',
'def')")
- }.getMessage.contains(incompatibleDefault))
+ sql("alter table t add column s array<int> default array(1, 2)")
+ checkAnswer(spark.table("t"), Row(false, null))
+ sql("insert into t(i) values (true)")
+ checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true, Seq(1,
2))))
+ }
+ }
+ // Negative tests: provided array element types must match their
corresponding DEFAULT
+ // declarations, if applicable.
+ val incompatibleDefault =
+ "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
+ "table column `s` has a DEFAULT value"
+ Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
+ assert(intercept[AnalysisException] {
+ sql("alter table t add column s array<int> default array('abc',
'def')")
+ }.getMessage.contains(incompatibleDefault))
+ }
}
}
testGluten("SPARK-39557 INSERT INTO statements with tables with struct
defaults") {
- withSQLConf("spark.gluten.sql.complexType.scan.fallback.enabled" ->
"false") {
-
- import testImplicits._
- // Positive tests: struct types are supported as default values.
- case class Config(dataSource: String, useDataFrames: Boolean = false)
- Seq(
- Config("parquet"),
- Config("parquet", useDataFrames = true),
- Config("orc"),
- Config("orc", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- sql(
- "alter table t add column s struct<x boolean, y string> default
struct(true, 'abc')")
- checkAnswer(spark.table("t"), Row(false, null))
- sql("insert into t(i) values (true)")
- checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Row(true, "abc"))))
+
+ import testImplicits._
+ // Positive tests: struct types are supported as default values.
+ case class Config(dataSource: String, useDataFrames: Boolean = false)
+ Seq(
+ Config("parquet"),
+ Config("parquet", useDataFrames = true),
+ Config("orc"),
+ Config("orc", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
+ sql(
+ "alter table t add column s struct<x boolean, y string> default
struct(true, 'abc')")
+ checkAnswer(spark.table("t"), Row(false, null))
+ sql("insert into t(i) values (true)")
+ checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Row(true, "abc"))))
+ }
+ }
- // Negative tests: provided map element types must match their
corresponding DEFAULT
- // declarations, if applicable.
- val incompatibleDefault =
- "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
- "table column `s` has a DEFAULT value"
- Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- assert(intercept[AnalysisException] {
- sql("alter table t add column s struct<x boolean, y string>
default struct(42, 56)")
- }.getMessage.contains(incompatibleDefault))
+ // Negative tests: provided map element types must match their
corresponding DEFAULT
+ // declarations, if applicable.
+ val incompatibleDefault =
+ "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
+ "table column `s` has a DEFAULT value"
+ Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
+ assert(intercept[AnalysisException] {
+ sql("alter table t add column s struct<x boolean, y string>
default struct(42, 56)")
+ }.getMessage.contains(incompatibleDefault))
+ }
}
}
ignoreGluten("SPARK-39557 INSERT INTO statements with tables with map
defaults") {
- withSQLConf("spark.gluten.sql.complexType.scan.fallback.enabled" ->
"false") {
-
- import testImplicits._
- // Positive tests: map types are supported as default values.
- case class Config(dataSource: String, useDataFrames: Boolean = false)
- Seq(
- Config("parquet"),
- Config("parquet", useDataFrames = true),
- Config("orc"),
- Config("orc", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- sql("alter table t add column s map<boolean, string> default
map(true, 'abc')")
- checkAnswer(spark.table("t"), Row(false, null))
- sql("insert into t(i) select true")
- checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Map(true -> "abc"))))
+
+ import testImplicits._
+ // Positive tests: map types are supported as default values.
+ case class Config(dataSource: String, useDataFrames: Boolean = false)
+ Seq(
+ Config("parquet"),
+ Config("parquet", useDataFrames = true),
+ Config("orc"),
+ Config("orc", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- withTable("t") {
- sql(s"""
- create table t(
- i int,
- s struct<
- x array<
- struct<a int, b int>>,
- y array<
- map<boolean, string>>>
- default struct(
- array(
- struct(1, 2)),
- array(
- map(false, 'def', true, 'jkl'))))
- using ${config.dataSource}""")
- sql("insert into t select 1, default")
- sql("alter table t alter column s drop default")
- if (config.useDataFrames) {
- Seq((2, null)).toDF.write.insertInto("t")
- } else {
- sql("insert into t select 2, default")
- }
- sql("""
- alter table t alter column s
- set default struct(
+ sql("alter table t add column s map<boolean, string> default
map(true, 'abc')")
+ checkAnswer(spark.table("t"), Row(false, null))
+ sql("insert into t(i) select true")
+ checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Map(true -> "abc"))))
+ }
+ withTable("t") {
+ sql(s"""
+ create table t(
+ i int,
+ s struct<
+ x array<
+ struct<a int, b int>>,
+ y array<
+ map<boolean, string>>>
+ default struct(
array(
- struct(3, 4)),
+ struct(1, 2)),
array(
- map(false, 'mno', true, 'pqr')))""")
- sql("insert into t select 3, default")
- sql("""
- alter table t
- add column t array<
- map<boolean, string>>
- default array(
- map(true, 'xyz'))""")
- sql("insert into t(i, s) select 4, default")
- checkAnswer(
- spark.table("t"),
- Seq(
- Row(1, Row(Seq(Row(1, 2)), Seq(Map(false -> "def", true ->
"jkl"))), null),
- Row(2, null, null),
- Row(3, Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true ->
"pqr"))), null),
- Row(
- 4,
- Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true -> "pqr"))),
- Seq(Map(true -> "xyz")))
- )
- )
+ map(false, 'def', true, 'jkl'))))
+ using ${config.dataSource}""")
+ sql("insert into t select 1, default")
+ sql("alter table t alter column s drop default")
+ if (config.useDataFrames) {
+ Seq((2, null)).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select 2, default")
}
- }
- // Negative tests: provided map element types must match their
corresponding DEFAULT
- // declarations, if applicable.
- val incompatibleDefault =
- "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
- "table column `s` has a DEFAULT value"
- Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- assert(intercept[AnalysisException] {
- sql("alter table t add column s map<boolean, string> default
map(42, 56)")
- }.getMessage.contains(incompatibleDefault))
+ sql("""
+ alter table t alter column s
+ set default struct(
+ array(
+ struct(3, 4)),
+ array(
+ map(false, 'mno', true, 'pqr')))""")
+ sql("insert into t select 3, default")
+ sql("""
+ alter table t
+ add column t array<
+ map<boolean, string>>
+ default array(
+ map(true, 'xyz'))""")
+ sql("insert into t(i, s) select 4, default")
+ checkAnswer(
+ spark.table("t"),
+ Seq(
+ Row(1, Row(Seq(Row(1, 2)), Seq(Map(false -> "def", true ->
"jkl"))), null),
+ Row(2, null, null),
+ Row(3, Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true ->
"pqr"))), null),
+ Row(
+ 4,
+ Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true -> "pqr"))),
+ Seq(Map(true -> "xyz")))
+ )
+ )
+ }
+ }
+ // Negative tests: provided map element types must match their
corresponding DEFAULT
+ // declarations, if applicable.
+ val incompatibleDefault =
+ "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
+ "table column `s` has a DEFAULT value"
+ Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
+ assert(intercept[AnalysisException] {
+ sql("alter table t add column s map<boolean, string> default
map(42, 56)")
+ }.getMessage.contains(incompatibleDefault))
+ }
}
}
}
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
index 264a7e7836..1020294d88 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
@@ -38,8 +38,7 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
testGluten("5182: Fix failed to parse post join filters") {
withSQLConf(
- "spark.sql.hive.convertMetastoreParquet" -> "false",
- "spark.gluten.sql.complexType.scan.fallback.enabled" -> "false") {
+ "spark.sql.hive.convertMetastoreParquet" -> "false") {
sql("DROP TABLE IF EXISTS test_5182_0;")
sql("DROP TABLE IF EXISTS test_5182_1;")
sql(
@@ -76,27 +75,23 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
}
testGluten("5249: Reading csv may throw Unexpected empty column") {
- withSQLConf(
- "spark.gluten.sql.complexType.scan.fallback.enabled" -> "false"
- ) {
- sql("DROP TABLE IF EXISTS test_5249;")
- sql(
- "CREATE TABLE test_5249 (name STRING, uid STRING) " +
- "ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' " +
- "STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' " +
- "OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';")
- sql("INSERT INTO test_5249 VALUES('name_1', 'id_1');")
- val df = spark.sql(
- "SELECT name, uid, count(distinct uid) total_uid_num from test_5249 " +
- "group by name, uid with cube;")
- checkAnswer(
- df,
- Seq(
- Row("name_1", "id_1", 1),
- Row("name_1", null, 1),
- Row(null, "id_1", 1),
- Row(null, null, 1)))
- }
+ sql("DROP TABLE IF EXISTS test_5249;")
+ sql(
+ "CREATE TABLE test_5249 (name STRING, uid STRING) " +
+ "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
" +
+ "STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' " +
+ "OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';")
+ sql("INSERT INTO test_5249 VALUES('name_1', 'id_1');")
+ val df = spark.sql(
+ "SELECT name, uid, count(distinct uid) total_uid_num from test_5249 " +
+ "group by name, uid with cube;")
+ checkAnswer(
+ df,
+ Seq(
+ Row("name_1", "id_1", 1),
+ Row("name_1", null, 1),
+ Row(null, "id_1", 1),
+ Row(null, null, 1)))
spark.sessionState.catalog.dropTable(
TableIdentifier("test_5249"),
ignoreIfNotExists = true,
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
index 0437e29a7d..c71e2cdd19 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
@@ -406,195 +406,189 @@ class GlutenInsertSuite
// TODO: fix in Spark-4.0
ignoreGluten("SPARK-39557 INSERT INTO statements with tables with array
defaults") {
- withSQLConf("spark.gluten.sql.complexType.scan.fallback.enabled" ->
"false") {
- import testImplicits._
- // Positive tests: array types are supported as default values.
- case class Config(dataSource: String, useDataFrames: Boolean = false)
- Seq(
- Config("parquet"),
- Config("parquet", useDataFrames = true),
- Config("orc"),
- Config("orc", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- sql("alter table t add column s array<int> default array(1, 2)")
- checkAnswer(spark.table("t"), Row(false, null))
- sql("insert into t(i) values (true)")
- checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Seq(1, 2))))
+ import testImplicits._
+ // Positive tests: array types are supported as default values.
+ case class Config(dataSource: String, useDataFrames: Boolean = false)
+ Seq(
+ Config("parquet"),
+ Config("parquet", useDataFrames = true),
+ Config("orc"),
+ Config("orc", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
- // Negative tests: provided array element types must match their
corresponding DEFAULT
- // declarations, if applicable.
- val incompatibleDefault =
- "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
- "table column `s` has a DEFAULT value"
- Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- assert(intercept[AnalysisException] {
- sql("alter table t add column s array<int> default array('abc',
'def')")
- }.getMessage.contains(incompatibleDefault))
+ sql("alter table t add column s array<int> default array(1, 2)")
+ checkAnswer(spark.table("t"), Row(false, null))
+ sql("insert into t(i) values (true)")
+ checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true, Seq(1,
2))))
+ }
+ }
+ // Negative tests: provided array element types must match their
corresponding DEFAULT
+ // declarations, if applicable.
+ val incompatibleDefault =
+ "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
+ "table column `s` has a DEFAULT value"
+ Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
+ assert(intercept[AnalysisException] {
+ sql("alter table t add column s array<int> default array('abc',
'def')")
+ }.getMessage.contains(incompatibleDefault))
+ }
}
}
// TODO: fix in Spark-4.0
ignoreGluten("SPARK-39557 INSERT INTO statements with tables with struct
defaults") {
- withSQLConf("spark.gluten.sql.complexType.scan.fallback.enabled" ->
"false") {
-
- import testImplicits._
- // Positive tests: struct types are supported as default values.
- case class Config(dataSource: String, useDataFrames: Boolean = false)
- Seq(
- Config("parquet"),
- Config("parquet", useDataFrames = true),
- Config("orc"),
- Config("orc", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- sql(
- "alter table t add column s struct<x boolean, y string> default
struct(true, 'abc')")
- checkAnswer(spark.table("t"), Row(false, null))
- sql("insert into t(i) values (true)")
- checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Row(true, "abc"))))
+
+ import testImplicits._
+ // Positive tests: struct types are supported as default values.
+ case class Config(dataSource: String, useDataFrames: Boolean = false)
+ Seq(
+ Config("parquet"),
+ Config("parquet", useDataFrames = true),
+ Config("orc"),
+ Config("orc", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
+ sql(
+ "alter table t add column s struct<x boolean, y string> default
struct(true, 'abc')")
+ checkAnswer(spark.table("t"), Row(false, null))
+ sql("insert into t(i) values (true)")
+ checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Row(true, "abc"))))
+ }
+ }
- // Negative tests: provided map element types must match their
corresponding DEFAULT
- // declarations, if applicable.
- val incompatibleDefault =
- "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
- "table column `s` has a DEFAULT value"
- Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- assert(intercept[AnalysisException] {
- sql("alter table t add column s struct<x boolean, y string>
default struct(42, 56)")
- }.getMessage.contains(incompatibleDefault))
+ // Negative tests: provided map element types must match their
corresponding DEFAULT
+ // declarations, if applicable.
+ val incompatibleDefault =
+ "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
+ "table column `s` has a DEFAULT value"
+ Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
+ assert(intercept[AnalysisException] {
+ sql("alter table t add column s struct<x boolean, y string>
default struct(42, 56)")
+ }.getMessage.contains(incompatibleDefault))
+ }
}
}
ignoreGluten("SPARK-39557 INSERT INTO statements with tables with map
defaults") {
- withSQLConf("spark.gluten.sql.complexType.scan.fallback.enabled" ->
"false") {
-
- import testImplicits._
- // Positive tests: map types are supported as default values.
- case class Config(dataSource: String, useDataFrames: Boolean = false)
- Seq(
- Config("parquet"),
- Config("parquet", useDataFrames = true),
- Config("orc"),
- Config("orc", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- sql("alter table t add column s map<boolean, string> default
map(true, 'abc')")
- checkAnswer(spark.table("t"), Row(false, null))
- sql("insert into t(i) select true")
- checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Map(true -> "abc"))))
+
+ import testImplicits._
+ // Positive tests: map types are supported as default values.
+ case class Config(dataSource: String, useDataFrames: Boolean = false)
+ Seq(
+ Config("parquet"),
+ Config("parquet", useDataFrames = true),
+ Config("orc"),
+ Config("orc", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- withTable("t") {
- sql(s"""
- create table t(
- i int,
- s struct<
- x array<
- struct<a int, b int>>,
- y array<
- map<boolean, string>>>
- default struct(
- array(
- struct(1, 2)),
- array(
- map(false, 'def', true, 'jkl'))))
- using ${config.dataSource}""")
- sql("insert into t select 1, default")
- sql("alter table t alter column s drop default")
- if (config.useDataFrames) {
- Seq((2, null)).toDF.write.insertInto("t")
- } else {
- sql("insert into t select 2, default")
- }
- sql("""
- alter table t alter column s
- set default struct(
+ sql("alter table t add column s map<boolean, string> default
map(true, 'abc')")
+ checkAnswer(spark.table("t"), Row(false, null))
+ sql("insert into t(i) select true")
+ checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Map(true -> "abc"))))
+ }
+ withTable("t") {
+ sql(s"""
+ create table t(
+ i int,
+ s struct<
+ x array<
+ struct<a int, b int>>,
+ y array<
+ map<boolean, string>>>
+ default struct(
array(
- struct(3, 4)),
+ struct(1, 2)),
array(
- map(false, 'mno', true, 'pqr')))""")
- sql("insert into t select 3, default")
- sql("""
- alter table t
- add column t array<
- map<boolean, string>>
- default array(
- map(true, 'xyz'))""")
- sql("insert into t(i, s) select 4, default")
- checkAnswer(
- spark.table("t"),
- Seq(
- Row(1, Row(Seq(Row(1, 2)), Seq(Map(false -> "def", true ->
"jkl"))), null),
- Row(2, null, null),
- Row(3, Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true ->
"pqr"))), null),
- Row(
- 4,
- Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true -> "pqr"))),
- Seq(Map(true -> "xyz")))
- )
- )
+ map(false, 'def', true, 'jkl'))))
+ using ${config.dataSource}""")
+ sql("insert into t select 1, default")
+ sql("alter table t alter column s drop default")
+ if (config.useDataFrames) {
+ Seq((2, null)).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select 2, default")
}
- }
- // Negative tests: provided map element types must match their
corresponding DEFAULT
- // declarations, if applicable.
- val incompatibleDefault =
- "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
- "table column `s` has a DEFAULT value"
- Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- assert(intercept[AnalysisException] {
- sql("alter table t add column s map<boolean, string> default
map(42, 56)")
- }.getMessage.contains(incompatibleDefault))
+ sql("""
+ alter table t alter column s
+ set default struct(
+ array(
+ struct(3, 4)),
+ array(
+ map(false, 'mno', true, 'pqr')))""")
+ sql("insert into t select 3, default")
+ sql("""
+ alter table t
+ add column t array<
+ map<boolean, string>>
+ default array(
+ map(true, 'xyz'))""")
+ sql("insert into t(i, s) select 4, default")
+ checkAnswer(
+ spark.table("t"),
+ Seq(
+ Row(1, Row(Seq(Row(1, 2)), Seq(Map(false -> "def", true ->
"jkl"))), null),
+ Row(2, null, null),
+ Row(3, Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true ->
"pqr"))), null),
+ Row(
+ 4,
+ Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true -> "pqr"))),
+ Seq(Map(true -> "xyz")))
+ )
+ )
+ }
+ }
+ // Negative tests: provided map element types must match their
corresponding DEFAULT
+ // declarations, if applicable.
+ val incompatibleDefault =
+ "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
+ "table column `s` has a DEFAULT value"
+ Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
+ assert(intercept[AnalysisException] {
+ sql("alter table t add column s map<boolean, string> default
map(42, 56)")
+ }.getMessage.contains(incompatibleDefault))
+ }
}
}
}
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
index 264a7e7836..1020294d88 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveSQLQueryCHSuite.scala
@@ -38,8 +38,7 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
testGluten("5182: Fix failed to parse post join filters") {
withSQLConf(
- "spark.sql.hive.convertMetastoreParquet" -> "false",
- "spark.gluten.sql.complexType.scan.fallback.enabled" -> "false") {
+ "spark.sql.hive.convertMetastoreParquet" -> "false") {
sql("DROP TABLE IF EXISTS test_5182_0;")
sql("DROP TABLE IF EXISTS test_5182_1;")
sql(
@@ -76,27 +75,23 @@ class GlutenHiveSQLQueryCHSuite extends
GlutenHiveSQLQuerySuiteBase {
}
testGluten("5249: Reading csv may throw Unexpected empty column") {
- withSQLConf(
- "spark.gluten.sql.complexType.scan.fallback.enabled" -> "false"
- ) {
- sql("DROP TABLE IF EXISTS test_5249;")
- sql(
- "CREATE TABLE test_5249 (name STRING, uid STRING) " +
- "ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' " +
- "STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' " +
- "OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';")
- sql("INSERT INTO test_5249 VALUES('name_1', 'id_1');")
- val df = spark.sql(
- "SELECT name, uid, count(distinct uid) total_uid_num from test_5249 " +
- "group by name, uid with cube;")
- checkAnswer(
- df,
- Seq(
- Row("name_1", "id_1", 1),
- Row("name_1", null, 1),
- Row(null, "id_1", 1),
- Row(null, null, 1)))
- }
+ sql("DROP TABLE IF EXISTS test_5249;")
+ sql(
+ "CREATE TABLE test_5249 (name STRING, uid STRING) " +
+ "ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
" +
+ "STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat' " +
+ "OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat';")
+ sql("INSERT INTO test_5249 VALUES('name_1', 'id_1');")
+ val df = spark.sql(
+ "SELECT name, uid, count(distinct uid) total_uid_num from test_5249 " +
+ "group by name, uid with cube;")
+ checkAnswer(
+ df,
+ Seq(
+ Row("name_1", "id_1", 1),
+ Row("name_1", null, 1),
+ Row(null, "id_1", 1),
+ Row(null, null, 1)))
spark.sessionState.catalog.dropTable(
TableIdentifier("test_5249"),
ignoreIfNotExists = true,
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
index 0437e29a7d..c71e2cdd19 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/sources/GlutenInsertSuite.scala
@@ -406,195 +406,189 @@ class GlutenInsertSuite
// TODO: fix in Spark-4.0
ignoreGluten("SPARK-39557 INSERT INTO statements with tables with array
defaults") {
- withSQLConf("spark.gluten.sql.complexType.scan.fallback.enabled" ->
"false") {
- import testImplicits._
- // Positive tests: array types are supported as default values.
- case class Config(dataSource: String, useDataFrames: Boolean = false)
- Seq(
- Config("parquet"),
- Config("parquet", useDataFrames = true),
- Config("orc"),
- Config("orc", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- sql("alter table t add column s array<int> default array(1, 2)")
- checkAnswer(spark.table("t"), Row(false, null))
- sql("insert into t(i) values (true)")
- checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Seq(1, 2))))
+ import testImplicits._
+ // Positive tests: array types are supported as default values.
+ case class Config(dataSource: String, useDataFrames: Boolean = false)
+ Seq(
+ Config("parquet"),
+ Config("parquet", useDataFrames = true),
+ Config("orc"),
+ Config("orc", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
- // Negative tests: provided array element types must match their
corresponding DEFAULT
- // declarations, if applicable.
- val incompatibleDefault =
- "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
- "table column `s` has a DEFAULT value"
- Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- assert(intercept[AnalysisException] {
- sql("alter table t add column s array<int> default array('abc',
'def')")
- }.getMessage.contains(incompatibleDefault))
+ sql("alter table t add column s array<int> default array(1, 2)")
+ checkAnswer(spark.table("t"), Row(false, null))
+ sql("insert into t(i) values (true)")
+ checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true, Seq(1,
2))))
+ }
+ }
+ // Negative tests: provided array element types must match their
corresponding DEFAULT
+ // declarations, if applicable.
+ val incompatibleDefault =
+ "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
+ "table column `s` has a DEFAULT value"
+ Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
+ assert(intercept[AnalysisException] {
+ sql("alter table t add column s array<int> default array('abc',
'def')")
+ }.getMessage.contains(incompatibleDefault))
+ }
}
}
// TODO: fix in Spark-4.0
ignoreGluten("SPARK-39557 INSERT INTO statements with tables with struct
defaults") {
- withSQLConf("spark.gluten.sql.complexType.scan.fallback.enabled" ->
"false") {
-
- import testImplicits._
- // Positive tests: struct types are supported as default values.
- case class Config(dataSource: String, useDataFrames: Boolean = false)
- Seq(
- Config("parquet"),
- Config("parquet", useDataFrames = true),
- Config("orc"),
- Config("orc", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- sql(
- "alter table t add column s struct<x boolean, y string> default
struct(true, 'abc')")
- checkAnswer(spark.table("t"), Row(false, null))
- sql("insert into t(i) values (true)")
- checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Row(true, "abc"))))
+
+ import testImplicits._
+ // Positive tests: struct types are supported as default values.
+ case class Config(dataSource: String, useDataFrames: Boolean = false)
+ Seq(
+ Config("parquet"),
+ Config("parquet", useDataFrames = true),
+ Config("orc"),
+ Config("orc", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
+ sql(
+ "alter table t add column s struct<x boolean, y string> default
struct(true, 'abc')")
+ checkAnswer(spark.table("t"), Row(false, null))
+ sql("insert into t(i) values (true)")
+ checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Row(true, "abc"))))
+ }
+ }
- // Negative tests: provided map element types must match their
corresponding DEFAULT
- // declarations, if applicable.
- val incompatibleDefault =
- "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
- "table column `s` has a DEFAULT value"
- Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- assert(intercept[AnalysisException] {
- sql("alter table t add column s struct<x boolean, y string>
default struct(42, 56)")
- }.getMessage.contains(incompatibleDefault))
+ // Negative tests: provided map element types must match their
corresponding DEFAULT
+ // declarations, if applicable.
+ val incompatibleDefault =
+ "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
+ "table column `s` has a DEFAULT value"
+ Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
+ assert(intercept[AnalysisException] {
+ sql("alter table t add column s struct<x boolean, y string>
default struct(42, 56)")
+ }.getMessage.contains(incompatibleDefault))
+ }
}
}
ignoreGluten("SPARK-39557 INSERT INTO statements with tables with map
defaults") {
- withSQLConf("spark.gluten.sql.complexType.scan.fallback.enabled" ->
"false") {
-
- import testImplicits._
- // Positive tests: map types are supported as default values.
- case class Config(dataSource: String, useDataFrames: Boolean = false)
- Seq(
- Config("parquet"),
- Config("parquet", useDataFrames = true),
- Config("orc"),
- Config("orc", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- sql("alter table t add column s map<boolean, string> default
map(true, 'abc')")
- checkAnswer(spark.table("t"), Row(false, null))
- sql("insert into t(i) select true")
- checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Map(true -> "abc"))))
+
+ import testImplicits._
+ // Positive tests: map types are supported as default values.
+ case class Config(dataSource: String, useDataFrames: Boolean = false)
+ Seq(
+ Config("parquet"),
+ Config("parquet", useDataFrames = true),
+ Config("orc"),
+ Config("orc", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- withTable("t") {
- sql(s"""
- create table t(
- i int,
- s struct<
- x array<
- struct<a int, b int>>,
- y array<
- map<boolean, string>>>
- default struct(
- array(
- struct(1, 2)),
- array(
- map(false, 'def', true, 'jkl'))))
- using ${config.dataSource}""")
- sql("insert into t select 1, default")
- sql("alter table t alter column s drop default")
- if (config.useDataFrames) {
- Seq((2, null)).toDF.write.insertInto("t")
- } else {
- sql("insert into t select 2, default")
- }
- sql("""
- alter table t alter column s
- set default struct(
+ sql("alter table t add column s map<boolean, string> default
map(true, 'abc')")
+ checkAnswer(spark.table("t"), Row(false, null))
+ sql("insert into t(i) select true")
+ checkAnswer(spark.table("t"), Seq(Row(false, null), Row(true,
Map(true -> "abc"))))
+ }
+ withTable("t") {
+ sql(s"""
+ create table t(
+ i int,
+ s struct<
+ x array<
+ struct<a int, b int>>,
+ y array<
+ map<boolean, string>>>
+ default struct(
array(
- struct(3, 4)),
+ struct(1, 2)),
array(
- map(false, 'mno', true, 'pqr')))""")
- sql("insert into t select 3, default")
- sql("""
- alter table t
- add column t array<
- map<boolean, string>>
- default array(
- map(true, 'xyz'))""")
- sql("insert into t(i, s) select 4, default")
- checkAnswer(
- spark.table("t"),
- Seq(
- Row(1, Row(Seq(Row(1, 2)), Seq(Map(false -> "def", true ->
"jkl"))), null),
- Row(2, null, null),
- Row(3, Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true ->
"pqr"))), null),
- Row(
- 4,
- Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true -> "pqr"))),
- Seq(Map(true -> "xyz")))
- )
- )
+ map(false, 'def', true, 'jkl'))))
+ using ${config.dataSource}""")
+ sql("insert into t select 1, default")
+ sql("alter table t alter column s drop default")
+ if (config.useDataFrames) {
+ Seq((2, null)).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select 2, default")
}
- }
- // Negative tests: provided map element types must match their
corresponding DEFAULT
- // declarations, if applicable.
- val incompatibleDefault =
- "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
- "table column `s` has a DEFAULT value"
- Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
- config =>
- withTable("t") {
- sql(s"create table t(i boolean) using ${config.dataSource}")
- if (config.useDataFrames) {
- Seq(false).toDF.write.insertInto("t")
- } else {
- sql("insert into t select false")
- }
- assert(intercept[AnalysisException] {
- sql("alter table t add column s map<boolean, string> default
map(42, 56)")
- }.getMessage.contains(incompatibleDefault))
+ sql("""
+ alter table t alter column s
+ set default struct(
+ array(
+ struct(3, 4)),
+ array(
+ map(false, 'mno', true, 'pqr')))""")
+ sql("insert into t select 3, default")
+ sql("""
+ alter table t
+ add column t array<
+ map<boolean, string>>
+ default array(
+ map(true, 'xyz'))""")
+ sql("insert into t(i, s) select 4, default")
+ checkAnswer(
+ spark.table("t"),
+ Seq(
+ Row(1, Row(Seq(Row(1, 2)), Seq(Map(false -> "def", true ->
"jkl"))), null),
+ Row(2, null, null),
+ Row(3, Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true ->
"pqr"))), null),
+ Row(
+ 4,
+ Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true -> "pqr"))),
+ Seq(Map(true -> "xyz")))
+ )
+ )
+ }
+ }
+ // Negative tests: provided map element types must match their
corresponding DEFAULT
+ // declarations, if applicable.
+ val incompatibleDefault =
+ "Failed to execute ALTER TABLE ADD COLUMNS command because the
destination " +
+ "table column `s` has a DEFAULT value"
+ Seq(Config("parquet"), Config("parquet", useDataFrames = true)).foreach {
+ config =>
+ withTable("t") {
+ sql(s"create table t(i boolean) using ${config.dataSource}")
+ if (config.useDataFrames) {
+ Seq(false).toDF.write.insertInto("t")
+ } else {
+ sql("insert into t select false")
}
- }
+ assert(intercept[AnalysisException] {
+ sql("alter table t add column s map<boolean, string> default
map(42, 56)")
+ }.getMessage.contains(incompatibleDefault))
+ }
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
