PorridgeSwim commented on code in PR #56153:
URL: https://github.com/apache/spark/pull/56153#discussion_r3313837221
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala:
##########
@@ -3882,6 +3882,17 @@ object SQLConf {
.booleanConf
.createWithDefault(false)
+ val STREAMING_DISALLOW_USER_SPECIFIED_SCHEMA_IN_TABLE_ENABLED =
+ buildConf("spark.sql.streaming.disallowUserSpecifiedSchemaInTable.enabled")
+ .doc("When true, DataStreamReader.table() throws an analysis error if
the caller " +
+ "supplied a user-specified schema via DataStreamReader.schema(...).
Catalog tables " +
+ "declare their own schema and any user-specified schema is silently
dropped, so this " +
+ "guards against misconfigurations. Set this to false to restore the
pre-4.2 behavior " +
+ "of silently ignoring the user-specified schema.")
+ .version("4.2.0")
Review Comment:
done
##########
sql/core/src/test/scala/org/apache/spark/sql/streaming/test/DataStreamTableAPISuite.scala:
##########
@@ -94,8 +94,27 @@ class DataStreamTableAPISuite extends StreamTest with
BeforeAndAfter {
}
checkError(
exception = e,
- condition = "_LEGACY_ERROR_TEMP_1189",
- parameters = Map("operation" -> "table"))
+ condition = "STREAMING_USER_SPECIFIED_SCHEMA_NOT_ALLOWED_IN_TABLE",
+ parameters = Map(
+ "config" ->
+
SQLConf.STREAMING_DISALLOW_USER_SPECIFIED_SCHEMA_IN_TABLE_ENABLED.key))
+ }
+ }
+
+ test("read: user-specified schema is silently ignored when flag is flipped
off") {
+ val tblName = "my_table"
+ withTable(tblName) {
+ spark.range(3).write.format("parquet").saveAsTable(tblName)
+ withSQLConf(
+
SQLConf.STREAMING_DISALLOW_USER_SPECIFIED_SCHEMA_IN_TABLE_ENABLED.key ->
"false") {
+ testStream(
+ spark.readStream
+ .schema(new StructType().add("a", IntegerType))
+ .table(tblName))(
+ ProcessAllAvailable(),
+ CheckAnswer(Row(0), Row(1), Row(2))
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
