This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new a8ad71dbe417 [SPARK-46998][SQL] Deprecate the SQL config `spark.sql.legacy.allowZeroIndexInFormatString` a8ad71dbe417 is described below commit a8ad71dbe417c16af13d46783e13fba0c2280268 Author: Max Gekk <max.g...@gmail.com> AuthorDate: Wed Feb 7 18:20:20 2024 -0800 [SPARK-46998][SQL] Deprecate the SQL config `spark.sql.legacy.allowZeroIndexInFormatString` ### What changes were proposed in this pull request? In the PR, I propose to deprecate the SQL config `spark.sql.legacy.allowZeroIndexInFormatString` and put it to the list `deprecatedSQLConfigs` in `SQLConf`. ### Why are the changes needed? After migration on JDK 17+, the SQL config `spark.sql.legacy.allowZeroIndexInFormatString` doesn't work anymore, and doesn't allow to use the zero index. Even users set the config to `true`, they get the error: ```java Illegal format argument index = 0 java.util.IllegalFormatArgumentIndexException: Illegal format argument index = 0 at java.base/java.util.Formatter$FormatSpecifier.index(Formatter.java:2808) at java.base/java.util.Formatter$FormatSpecifier.<init>(Formatter.java:2879) ``` ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? By running the modified test suite: ``` $ build/sbt "test:testOnly *QueryCompilationErrorsSuite" ``` ### Was this patch authored or co-authored using generative AI tooling? No. Closes #45057 from MaxGekk/deprecate-ALLOW_ZERO_INDEX_IN_FORMAT_STRING. Authored-by: Max Gekk <max.g...@gmail.com> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- docs/sql-migration-guide.md | 1 + .../src/main/scala/org/apache/spark/sql/internal/SQLConf.scala | 7 +++++-- .../org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala | 7 +++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md index cb5e697f871c..3d0c7280496a 100644 --- a/docs/sql-migration-guide.md +++ b/docs/sql-migration-guide.md @@ -38,6 +38,7 @@ license: | - `spark.sql.avro.datetimeRebaseModeInRead` instead of `spark.sql.legacy.avro.datetimeRebaseModeInRead` - Since Spark 4.0, the default value of `spark.sql.orc.compression.codec` is changed from `snappy` to `zstd`. To restore the previous behavior, set `spark.sql.orc.compression.codec` to `snappy`. - Since Spark 4.0, `SELECT (*)` is equivalent to `SELECT struct(*)` instead of `SELECT *`. To restore the previous behavior, set `spark.sql.legacy.ignoreParenthesesAroundStar` to `true`. +- Since Spark 4.0, the SQL config `spark.sql.legacy.allowZeroIndexInFormatString` is deprecated. Consider to change `strfmt` of the `format_string` function to use 1-based indexes. The first argument must be referenced by "1$", the second by "2$", etc. ## Upgrading from Spark SQL 3.4 to 3.5 diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 8f86a1c8a1f3..59db3e71a135 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -2339,7 +2339,8 @@ object SQLConf { .doc("When false, the `strfmt` in `format_string(strfmt, obj, ...)` and " + "`printf(strfmt, obj, ...)` will no longer support to use \"0$\" to specify the first " + "argument, the first argument should always reference by \"1$\" when use argument index " + - "to indicating the position of the argument in the argument list.") + "to indicating the position of the argument in the argument list. " + + "This config will be removed in the future releases.") .version("3.3") .booleanConf .createWithDefault(false) @@ -4718,7 +4719,9 @@ object SQLConf { DeprecatedConfig(ESCAPED_STRING_LITERALS.key, "4.0", "Use raw string literals with the `r` prefix instead. "), DeprecatedConfig("spark.connect.copyFromLocalToFs.allowDestLocal", "4.0", - s"Use '${ARTIFACT_COPY_FROM_LOCAL_TO_FS_ALLOW_DEST_LOCAL.key}' instead.") + s"Use '${ARTIFACT_COPY_FROM_LOCAL_TO_FS_ALLOW_DEST_LOCAL.key}' instead."), + DeprecatedConfig(ALLOW_ZERO_INDEX_IN_FORMAT_STRING.key, "4.0", "Increase indexes by 1 " + + "in `strfmt` of the `format_string` function. Refer to the first argument by \"1$\".") ) Map(configs.map { cfg => cfg.key -> cfg } : _*) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala index 7ea0c3843444..e22399c326f6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala @@ -17,6 +17,8 @@ package org.apache.spark.sql.errors +import java.util.IllegalFormatException + import org.apache.spark.{SPARK_DOC_ROOT, SparkIllegalArgumentException, SparkUnsupportedOperationException} import org.apache.spark.sql._ import org.apache.spark.sql.api.java.{UDF1, UDF2, UDF23Test} @@ -129,6 +131,11 @@ class QueryCompilationErrorsSuite context = ExpectedContext( fragment = "format_string('%0$s', 'Hello')", start = 7, stop = 36)) } + withSQLConf(SQLConf.ALLOW_ZERO_INDEX_IN_FORMAT_STRING.key -> "true") { + intercept[IllegalFormatException] { + sql("select format_string('%0$s', 'Hello')").collect() + } + } } test("INVALID_PANDAS_UDF_PLACEMENT: Using aggregate function with grouped aggregate pandas UDF") { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org