This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b01b88ad68a2 [SPARK-46629] Fix for STRUCT type DDL not picking up
nullability and comment
b01b88ad68a2 is described below
commit b01b88ad68a264396001d99c63bf58fb56d2be68
Author: Vitalii Li <vitalii...@databricks.com>
AuthorDate: Thu Jan 18 17:28:45 2024 +0300
[SPARK-46629] Fix for STRUCT type DDL not picking up nullability and comment
### What changes were proposed in this pull request?
This change adds logic to generate correct DDL for nested fields in STRUCT.
In particular instead of generating list of fields with data type names it will
add `NOT NULL` qualifier when necessary and field comment when present.
For a table:
```
CREATE TABLE t(field STRUCT<one: STRING NOT NULL, two: DOUBLE NOT NULL
COMMENT 'comment'>);
SHOW CREATE TABLE t;
```
Before:
```
CREATE TABLE t(field STRUCT<one: STRING, two: DOUBLE>)
```
After
```
CREATE TABLE t(field STRUCT<one: STRING NOT NULL, two: DOUBLE NOT NULL
COMMENT 'comment'>)
```
Closes #41016
### Why are the changes needed?
Generate correct DDL.
### Does this PR introduce _any_ user-facing change?
No, we do not document behavior of this command for struct case.
### How was this patch tested?
New unit test.
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #44644 from vitaliili-db/SPARK-46629.
Authored-by: Vitalii Li <vitalii...@databricks.com>
Signed-off-by: Max Gekk <max.g...@gmail.com>
---
.../spark/sql/protobuf/ProtobufSerdeSuite.scala | 4 +++-
.../org/apache/spark/sql/types/StructField.scala | 9 ++++++---
.../expressions/ConditionalExpressionSuite.scala | 5 +++--
.../apache/spark/sql/types/StructTypeSuite.scala | 16 ++++++++++++++++
.../analyzer-results/ansi/literals.sql.out | 2 +-
.../sql-tests/analyzer-results/literals.sql.out | 2 +-
.../typeCoercion/native/mapconcat.sql.out | 4 ++--
.../analyzer-results/xml-functions.sql.out | 2 +-
.../sql-tests/results/ansi/literals.sql.out | 2 +-
.../resources/sql-tests/results/literals.sql.out | 2 +-
.../results/typeCoercion/native/mapconcat.sql.out | 4 ++--
.../sql-tests/results/xml-functions.sql.out | 2 +-
.../apache/spark/sql/DataFrameFunctionsSuite.scala | 2 +-
.../spark/sql/DataFrameSetOperationsSuite.scala | 3 ++-
.../apache/spark/sql/FileBasedDataSourceSuite.scala | 4 ++--
.../apache/spark/sql/GeneratorFunctionSuite.scala | 4 ++--
.../scala/org/apache/spark/sql/SQLQuerySuite.scala | 4 ++--
.../sql/errors/QueryCompilationErrorsSuite.scala | 6 +++---
.../execution/command/v2/ShowCreateTableSuite.scala | 21 +++++++++++++++++++++
.../org/apache/spark/sql/sources/InsertSuite.scala | 2 +-
20 files changed, 72 insertions(+), 28 deletions(-)
diff --git
a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufSerdeSuite.scala
b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufSerdeSuite.scala
index 56a980d05fbd..03285c73f1ff 100644
---
a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufSerdeSuite.scala
+++
b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufSerdeSuite.scala
@@ -127,9 +127,11 @@ class ProtobufSerdeSuite extends SharedSparkSession with
ProtobufTestBase {
"protobufType" -> "FieldMissingInProto",
"toType" -> toSQLType(CATALYST_STRUCT)))
- assertFailedConversionMessage(protoFile,
+ assertFailedConversionMessage(
+ protoFile,
Serializer,
BY_NAME,
+ nonnullCatalyst,
errorClass = "UNABLE_TO_CONVERT_TO_PROTOBUF_MESSAGE_TYPE",
params = Map(
"protobufType" -> "FieldMissingInProto",
diff --git
a/sql/api/src/main/scala/org/apache/spark/sql/types/StructField.scala
b/sql/api/src/main/scala/org/apache/spark/sql/types/StructField.scala
index ca15d23b601e..66f9557db213 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/types/StructField.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/types/StructField.scala
@@ -148,11 +148,14 @@ case class StructField(
.map(" COMMENT '" + _ + "'")
.getOrElse("")
+ private lazy val nullDDL = if (nullable) "" else " NOT NULL"
+
/**
* Returns a string containing a schema in SQL format. For example the
following value:
* `StructField("eventId", IntegerType)` will be converted to `eventId`: INT.
*/
- private[sql] def sql = s"${QuotingUtils.quoteIfNeeded(name)}:
${dataType.sql}$getDDLComment"
+ private[sql] def sql =
+ s"${QuotingUtils.quoteIfNeeded(name)}:
${dataType.sql}$nullDDL$getDDLComment"
/**
* Returns a string containing a schema in DDL format. For example, the
following value:
@@ -161,7 +164,7 @@ case class StructField(
* @since 2.4.0
*/
def toDDL: String = {
- val nullString = if (nullable) "" else " NOT NULL"
- s"${QuotingUtils.quoteIfNeeded(name)}
${dataType.sql}${nullString}$getDDLDefault$getDDLComment"
+ s"${QuotingUtils.quoteIfNeeded(name)} ${dataType.sql}$nullDDL" +
+ s"$getDDLDefault$getDDLComment"
}
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConditionalExpressionSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConditionalExpressionSuite.scala
index e7f7c370dcb9..7ffb32121702 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConditionalExpressionSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ConditionalExpressionSuite.scala
@@ -242,7 +242,7 @@ class ConditionalExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper
errorSubClass = "DATA_DIFF_TYPES",
messageParameters = Map(
"functionName" -> "`casewhen`",
- "dataType" -> "[\"STRUCT<x: INT>\", \"STRUCT<y: INT>\"]")))
+ "dataType" -> "[\"STRUCT<x: INT NOT NULL>\", \"STRUCT<y: INT NOT
NULL>\"]")))
val checkResult2 = CaseWhen(Seq((Literal.FalseLiteral, caseVal1),
(Literal.FalseLiteral, caseVal2)), Some(elseVal)).checkInputDataTypes()
@@ -250,7 +250,8 @@ class ConditionalExpressionSuite extends SparkFunSuite with
ExpressionEvalHelper
errorSubClass = "DATA_DIFF_TYPES",
messageParameters = Map(
"functionName" -> "`casewhen`",
- "dataType" -> "[\"STRUCT<x: INT>\", \"STRUCT<y: INT>\", \"STRUCT<z:
INT>\"]")))
+ "dataType" -> ("[\"STRUCT<x: INT NOT NULL>\", " +
+ "\"STRUCT<y: INT NOT NULL>\", \"STRUCT<z: INT NOT NULL>\"]"))))
}
test("SPARK-27917 test semantic equals of CaseWhen") {
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
index 7c28fd8e2a03..1aa0d515d8f9 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
@@ -584,4 +584,20 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper
{
ResolveDefaultColumns.existenceDefaultValues(source4)
}.getMessage.contains(error))
}
+
+ test("SPARK-46629: Test STRUCT DDL with NOT NULL round trip") {
+ val struct = StructType(
+ Seq(
+ StructField(
+ "b",
+ StructType(
+ Seq(StructField("c", StringType, nullable =
false).withComment("struct comment"))),
+ nullable = false),
+ StructField("b", StringType, nullable = false),
+ StructField("c", StringType).withComment("nullable comment")))
+ assert(
+ struct.toDDL == "b STRUCT<c: STRING NOT NULL COMMENT 'struct comment'>
NOT NULL," +
+ "b STRING NOT NULL,c STRING COMMENT 'nullable comment'")
+ assert(fromDDL(struct.toDDL) === struct)
+ }
}
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/ansi/literals.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/ansi/literals.sql.out
index 48368ca11723..83d0ff3f2edf 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/ansi/literals.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/ansi/literals.sql.out
@@ -590,7 +590,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"sqlState" : "42K09",
"messageParameters" : {
"inputSql" : "\"named_struct(a, 1, b, spark)\"",
- "inputType" : "\"STRUCT<a: INT, b: STRING>\"",
+ "inputType" : "\"STRUCT<a: INT NOT NULL, b: STRING NOT NULL>\"",
"paramIndex" : "1",
"requiredType" : "(\"NUMERIC\" or \"INTERVAL DAY TO SECOND\" or \"INTERVAL
YEAR TO MONTH\" or \"INTERVAL\")",
"sqlExpr" : "\"(+ named_struct(a, 1, b, spark))\""
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/literals.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/literals.sql.out
index 48368ca11723..83d0ff3f2edf 100644
--- a/sql/core/src/test/resources/sql-tests/analyzer-results/literals.sql.out
+++ b/sql/core/src/test/resources/sql-tests/analyzer-results/literals.sql.out
@@ -590,7 +590,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"sqlState" : "42K09",
"messageParameters" : {
"inputSql" : "\"named_struct(a, 1, b, spark)\"",
- "inputType" : "\"STRUCT<a: INT, b: STRING>\"",
+ "inputType" : "\"STRUCT<a: INT NOT NULL, b: STRING NOT NULL>\"",
"paramIndex" : "1",
"requiredType" : "(\"NUMERIC\" or \"INTERVAL DAY TO SECOND\" or \"INTERVAL
YEAR TO MONTH\" or \"INTERVAL\")",
"sqlExpr" : "\"(+ named_struct(a, 1, b, spark))\""
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/typeCoercion/native/mapconcat.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/typeCoercion/native/mapconcat.sql.out
index 753f79f3f712..f4c932fa29f9 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/typeCoercion/native/mapconcat.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/typeCoercion/native/mapconcat.sql.out
@@ -184,7 +184,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"errorClass" : "DATATYPE_MISMATCH.DATA_DIFF_TYPES",
"sqlState" : "42K09",
"messageParameters" : {
- "dataType" : "(\"MAP<INT, INT>\" or \"MAP<STRUCT<col1: STRING, col2: INT>,
STRUCT<col1: STRING, col2: INT>>\")",
+ "dataType" : "(\"MAP<INT, INT>\" or \"MAP<STRUCT<col1: STRING NOT NULL,
col2: INT NOT NULL>, STRUCT<col1: STRING NOT NULL, col2: INT NOT NULL>>\")",
"functionName" : "`map_concat`",
"sqlExpr" : "\"map_concat(int_map1, struct_map2)\""
},
@@ -208,7 +208,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"errorClass" : "DATATYPE_MISMATCH.DATA_DIFF_TYPES",
"sqlState" : "42K09",
"messageParameters" : {
- "dataType" : "(\"MAP<STRUCT<col1: STRING, col2: INT>, STRUCT<col1: STRING,
col2: INT>>\" or \"MAP<ARRAY<STRING>, ARRAY<STRING>>\")",
+ "dataType" : "(\"MAP<STRUCT<col1: STRING NOT NULL, col2: INT NOT NULL>,
STRUCT<col1: STRING NOT NULL, col2: INT NOT NULL>>\" or \"MAP<ARRAY<STRING>,
ARRAY<STRING>>\")",
"functionName" : "`map_concat`",
"sqlExpr" : "\"map_concat(struct_map1, array_map2)\""
},
diff --git
a/sql/core/src/test/resources/sql-tests/analyzer-results/xml-functions.sql.out
b/sql/core/src/test/resources/sql-tests/analyzer-results/xml-functions.sql.out
index 51cf3d976f68..3fbc8d19b94a 100644
---
a/sql/core/src/test/resources/sql-tests/analyzer-results/xml-functions.sql.out
+++
b/sql/core/src/test/resources/sql-tests/analyzer-results/xml-functions.sql.out
@@ -22,7 +22,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"sqlState" : "42K09",
"messageParameters" : {
"inputSql" : "\"array(named_struct(a, 1, b, 2))\"",
- "inputType" : "\"ARRAY<STRUCT<a: INT, b: INT>>\"",
+ "inputType" : "\"ARRAY<STRUCT<a: INT NOT NULL, b: INT NOT NULL>>\"",
"paramIndex" : "1",
"requiredType" : "\"STRUCT\"",
"sqlExpr" : "\"to_xml(array(named_struct(a, 1, b, 2)))\""
diff --git
a/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out
b/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out
index 3006d30d0a05..6e2c8a65206e 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/literals.sql.out
@@ -660,7 +660,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"sqlState" : "42K09",
"messageParameters" : {
"inputSql" : "\"named_struct(a, 1, b, spark)\"",
- "inputType" : "\"STRUCT<a: INT, b: STRING>\"",
+ "inputType" : "\"STRUCT<a: INT NOT NULL, b: STRING NOT NULL>\"",
"paramIndex" : "1",
"requiredType" : "(\"NUMERIC\" or \"INTERVAL DAY TO SECOND\" or \"INTERVAL
YEAR TO MONTH\" or \"INTERVAL\")",
"sqlExpr" : "\"(+ named_struct(a, 1, b, spark))\""
diff --git a/sql/core/src/test/resources/sql-tests/results/literals.sql.out
b/sql/core/src/test/resources/sql-tests/results/literals.sql.out
index 3006d30d0a05..6e2c8a65206e 100644
--- a/sql/core/src/test/resources/sql-tests/results/literals.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/literals.sql.out
@@ -660,7 +660,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"sqlState" : "42K09",
"messageParameters" : {
"inputSql" : "\"named_struct(a, 1, b, spark)\"",
- "inputType" : "\"STRUCT<a: INT, b: STRING>\"",
+ "inputType" : "\"STRUCT<a: INT NOT NULL, b: STRING NOT NULL>\"",
"paramIndex" : "1",
"requiredType" : "(\"NUMERIC\" or \"INTERVAL DAY TO SECOND\" or \"INTERVAL
YEAR TO MONTH\" or \"INTERVAL\")",
"sqlExpr" : "\"(+ named_struct(a, 1, b, spark))\""
diff --git
a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapconcat.sql.out
b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapconcat.sql.out
index 5907f700877f..8fd398ff87f0 100644
---
a/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapconcat.sql.out
+++
b/sql/core/src/test/resources/sql-tests/results/typeCoercion/native/mapconcat.sql.out
@@ -147,7 +147,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"errorClass" : "DATATYPE_MISMATCH.DATA_DIFF_TYPES",
"sqlState" : "42K09",
"messageParameters" : {
- "dataType" : "(\"MAP<INT, INT>\" or \"MAP<STRUCT<col1: STRING, col2: INT>,
STRUCT<col1: STRING, col2: INT>>\")",
+ "dataType" : "(\"MAP<INT, INT>\" or \"MAP<STRUCT<col1: STRING NOT NULL,
col2: INT NOT NULL>, STRUCT<col1: STRING NOT NULL, col2: INT NOT NULL>>\")",
"functionName" : "`map_concat`",
"sqlExpr" : "\"map_concat(int_map1, struct_map2)\""
},
@@ -173,7 +173,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"errorClass" : "DATATYPE_MISMATCH.DATA_DIFF_TYPES",
"sqlState" : "42K09",
"messageParameters" : {
- "dataType" : "(\"MAP<STRUCT<col1: STRING, col2: INT>, STRUCT<col1: STRING,
col2: INT>>\" or \"MAP<ARRAY<STRING>, ARRAY<STRING>>\")",
+ "dataType" : "(\"MAP<STRUCT<col1: STRING NOT NULL, col2: INT NOT NULL>,
STRUCT<col1: STRING NOT NULL, col2: INT NOT NULL>>\" or \"MAP<ARRAY<STRING>,
ARRAY<STRING>>\")",
"functionName" : "`map_concat`",
"sqlExpr" : "\"map_concat(struct_map1, array_map2)\""
},
diff --git
a/sql/core/src/test/resources/sql-tests/results/xml-functions.sql.out
b/sql/core/src/test/resources/sql-tests/results/xml-functions.sql.out
index 704addb7a930..0194a676ad3b 100644
--- a/sql/core/src/test/resources/sql-tests/results/xml-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/xml-functions.sql.out
@@ -26,7 +26,7 @@ org.apache.spark.sql.catalyst.ExtendedAnalysisException
"sqlState" : "42K09",
"messageParameters" : {
"inputSql" : "\"array(named_struct(a, 1, b, 2))\"",
- "inputType" : "\"ARRAY<STRUCT<a: INT, b: INT>>\"",
+ "inputType" : "\"ARRAY<STRUCT<a: INT NOT NULL, b: INT NOT NULL>>\"",
"paramIndex" : "1",
"requiredType" : "\"STRUCT\"",
"sqlExpr" : "\"to_xml(array(named_struct(a, 1, b, 2)))\""
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index bb024b79598e..ad78916bd94c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -1988,7 +1988,7 @@ class DataFrameFunctionsSuite extends QueryTest with
SharedSparkSession {
"sqlExpr" -> "\"reverse(struct(1, a))\"",
"paramIndex" -> "1",
"inputSql" -> "\"struct(1, a)\"",
- "inputType" -> "\"STRUCT<col1: INT, col2: STRING>\"",
+ "inputType" -> "\"STRUCT<col1: INT NOT NULL, col2: STRING NOT NULL>\"",
"requiredType" -> "(\"STRING\" or \"ARRAY\")"
),
queryContext = Array(ExpectedContext("", "", 7, 29, "reverse(struct(1,
'a'))"))
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala
index bbb1561bb695..d6cf77572731 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSetOperationsSuite.scala
@@ -1039,7 +1039,8 @@ class DataFrameSetOperationsSuite extends QueryTest
parameters = Map(
"tableOrdinalNumber" -> "second",
"columnOrdinalNumber" -> "third",
- "dataType2" -> "\"STRUCT<c1: INT, c2: INT, c3: STRUCT<c3: INT>>\"",
+ "dataType2" ->
+ "\"STRUCT<c1: INT NOT NULL, c2: INT NOT NULL, c3: STRUCT<c3: INT NOT
NULL>>\"",
"operator" -> "UNION",
"hint" -> "",
"dataType1" -> "\"STRUCT<c1: INT, c2: INT, c3: STRUCT<c3: INT, c5:
INT>>\"")
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
index fc68f27c67ad..03b8ca32f561 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/FileBasedDataSourceSuite.scala
@@ -322,7 +322,7 @@ class FileBasedDataSourceSuite extends QueryTest
errorClass = "UNSUPPORTED_DATA_TYPE_FOR_DATASOURCE",
parameters = Map(
"columnName" -> "`struct(a)`",
- "columnType" -> "\"STRUCT<a: INT>\"",
+ "columnType" -> "\"STRUCT<a: INT NOT NULL>\"",
"format" -> "Text")
)
@@ -404,7 +404,7 @@ class FileBasedDataSourceSuite extends QueryTest
errorClass = "UNSUPPORTED_DATA_TYPE_FOR_DATASOURCE",
parameters = Map(
"columnName" -> "`struct(a, b)`",
- "columnType" -> "\"STRUCT<a: INT, b: STRING>\"",
+ "columnType" -> "\"STRUCT<a: INT NOT NULL, b: STRING>\"",
"format" -> "CSV")
)
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
index 2ab651237206..25287cce368a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
@@ -334,7 +334,7 @@ class GeneratorFunctionSuite extends QueryTest with
SharedSparkSession {
parameters = Map(
"sqlExpr" -> "\"array(struct(a), struct(b))\"",
"functionName" -> "`array`",
- "dataType" -> "(\"STRUCT<a: INT>\" or \"STRUCT<b: INT>\")"),
+ "dataType" -> "(\"STRUCT<a: INT NOT NULL>\" or \"STRUCT<b: INT NOT
NULL>\")"),
context = ExpectedContext(
fragment = "array",
callSitePattern = getCurrentClassCallSitePattern))
@@ -352,7 +352,7 @@ class GeneratorFunctionSuite extends QueryTest with
SharedSparkSession {
parameters = Map(
"sqlExpr" -> "\"array(struct(a), struct(2))\"",
"functionName" -> "`array`",
- "dataType" -> "(\"STRUCT<a: INT>\" or \"STRUCT<col1: INT>\")"),
+ "dataType" -> "(\"STRUCT<a: INT NOT NULL>\" or \"STRUCT<col1: INT NOT
NULL>\")"),
context = ExpectedContext(
fragment = "array",
callSitePattern = getCurrentClassCallSitePattern))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index b603c95fb30d..414a4d7197f9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -2731,10 +2731,10 @@ class SQLQuerySuite extends QueryTest with
SharedSparkSession with AdaptiveSpark
parameters = Map(
"tableOrdinalNumber" -> "second",
"columnOrdinalNumber" -> "first",
- "dataType2" -> "\"STRUCT<a: INT>\"",
+ "dataType2" -> "\"STRUCT<a: INT NOT NULL>\"",
"operator" -> "EXCEPT",
"hint" -> "",
- "dataType1" -> "\"STRUCT<A: INT>\""),
+ "dataType1" -> "\"STRUCT<A: INT NOT NULL>\""),
context = ExpectedContext(
fragment = "SELECT struct(1 a) EXCEPT (SELECT struct(2 A))",
start = 0,
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
index 0ed58626b099..3c397bd6a4e1 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala
@@ -403,7 +403,7 @@ class QueryCompilationErrorsSuite
spark.read.schema(schema).json(spark.emptyDataset[String])
},
errorClass = "INVALID_JSON_SCHEMA_MAP_TYPE",
- parameters = Map("jsonSchema" -> "\"STRUCT<map: MAP<INT, INT>>\"")
+ parameters = Map("jsonSchema" -> "\"STRUCT<map: MAP<INT, INT> NOT
NULL>\"")
)
}
@@ -622,7 +622,7 @@ class QueryCompilationErrorsSuite
exception = e1,
errorClass = "UNSUPPORTED_DESERIALIZER.FIELD_NUMBER_MISMATCH",
parameters = Map(
- "schema" -> "\"STRUCT<a: STRING, b: INT>\"",
+ "schema" -> "\"STRUCT<a: STRING, b: INT NOT NULL>\"",
"ordinal" -> "3"))
val e2 = intercept[AnalysisException] {
@@ -631,7 +631,7 @@ class QueryCompilationErrorsSuite
checkError(
exception = e2,
errorClass = "UNSUPPORTED_DESERIALIZER.FIELD_NUMBER_MISMATCH",
- parameters = Map("schema" -> "\"STRUCT<a: STRING, b: INT>\"",
+ parameters = Map("schema" -> "\"STRUCT<a: STRING, b: INT NOT NULL>\"",
"ordinal" -> "1"))
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala
index adda9dcfffe4..f72127cbd1de 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala
@@ -160,4 +160,25 @@ class ShowCreateTableSuite extends
command.ShowCreateTableSuiteBase with Command
)
}
}
+
+ test("SPARK-46629: show struct fields with NOT NULL and comment") {
+ withNamespaceAndTable(ns, table) { t =>
+ sql(s"""
+ |CREATE TABLE $t (
+ | a struct<b: bigint COMMENT 'comment', c: struct<d: string NOT
NULL, e: string>>
+ |)
+ |USING parquet
+ |COMMENT 'This is a comment'
+ """.stripMargin)
+ val showDDL = getShowCreateDDL(t)
+ assert(
+ showDDL === Array(
+ s"CREATE TABLE $fullName (",
+ "a STRUCT<b: BIGINT COMMENT 'comment', c: STRUCT<d: STRING NOT NULL,
e: STRING>>)",
+ "USING parquet",
+ "COMMENT 'This is a comment'"
+ )
+ )
+ }
+ }
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
index 76073a108a3c..be175eaeff8a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
@@ -2077,7 +2077,7 @@ class InsertSuite extends DataSourceTest with
SharedSparkSession {
"colName" -> "`s`",
"expectedType" -> "\"STRUCT<x: BOOLEAN, y: STRING>\"",
"defaultValue" -> "struct(42, 56)",
- "actualType" -> "\"STRUCT<col1: INT, col2: INT>\""))
+ "actualType" -> "\"STRUCT<col1: INT NOT NULL, col2: INT NOT
NULL>\""))
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
