This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch branch-3.4 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push: new 20048372d6a [SPARK-44990][SQL] Reduce the frequency of get `spark.sql.legacy.nullValueWrittenAsQuotedEmptyStringCsv` 20048372d6a is described below commit 20048372d6aa7546504266ffa0567c418de660f1 Author: Jia Fan <fanjiaemi...@qq.com> AuthorDate: Wed Aug 30 10:55:01 2023 -0700 [SPARK-44990][SQL] Reduce the frequency of get `spark.sql.legacy.nullValueWrittenAsQuotedEmptyStringCsv` ### What changes were proposed in this pull request? This PR move get config `spark.sql.legacy.nullValueWrittenAsQuotedEmptyStringCsv` to lazy val of `UnivocityGenerator`. To reduce the frequency of get it. As report, it will affect performance. ### Why are the changes needed? Reduce the frequency of get `spark.sql.legacy.nullValueWrittenAsQuotedEmptyStringCsv` ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? exist test ### Was this patch authored or co-authored using generative AI tooling? No Closes #42738 from Hisoka-X/SPARK-44990_csv_null_value_config. Authored-by: Jia Fan <fanjiaemi...@qq.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> (cherry picked from commit dac750b855c35a88420b6ba1b943bf0b6f0dded1) Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- .../scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala index d124a055f63..c8eded3ccd4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/UnivocityGenerator.scala @@ -60,6 +60,8 @@ class UnivocityGenerator( options.locale, legacyFormat = FAST_DATE_FORMAT, isParsing = false) + private lazy val nullAsQuotedEmptyString = + SQLConf.get.getConf(SQLConf.LEGACY_NULL_VALUE_WRITTEN_AS_QUOTED_EMPTY_STRING_CSV) @scala.annotation.tailrec private def makeConverter(dataType: DataType): ValueConverter = dataType match { @@ -96,8 +98,7 @@ class UnivocityGenerator( while (i < row.numFields) { if (!row.isNullAt(i)) { values(i) = valueConverters(i).apply(row, i) - } else if ( - SQLConf.get.getConf(SQLConf.LEGACY_NULL_VALUE_WRITTEN_AS_QUOTED_EMPTY_STRING_CSV)) { + } else if (nullAsQuotedEmptyString) { values(i) = options.nullValue } i += 1 --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org