This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 2f3cb36cb99 [SPARK-45261][CORE] Fix `EventLogFileWriters` to handle `none` as a codec 2f3cb36cb99 is described below commit 2f3cb36cb99a4b8d1ddec74696c4ed036c5df5b2 Author: Dongjoon Hyun <dh...@apple.com> AuthorDate: Thu Sep 21 14:44:11 2023 -0700 [SPARK-45261][CORE] Fix `EventLogFileWriters` to handle `none` as a codec ### What changes were proposed in this pull request? This PR aims to support `none` as a codec instead of throwing exception. Currrently, our unit test is supposed to test it, but actually it's not tested at all. https://github.com/apache/spark/blob/892fdc532696e703b353c4758320d69162fffe8c/core/src/test/scala/org/apache/spark/deploy/history/EventLogFileReadersSuite.scala#L120-L124 ``` $ build/sbt "core/testOnly *EventLogFileReaderSuite*" ... [info] - get information, list event log files, zip log files - with codec None (33 milliseconds) [info] - get information, list event log files, zip log files - with codec Some(lz4) (125 milliseconds) ... ``` ### Why are the changes needed? ``` $ bin/spark-shell \ -c spark.eventLog.enabled=true \ -c spark.eventLog.compress=true \ -c spark.eventLog.compression.codec=none ... 23/09/21 13:26:45 ERROR SparkContext: Error initializing SparkContext. org.apache.spark.SparkIllegalArgumentException: [CODEC_SHORT_NAME_NOT_FOUND] Cannot find a short name for the codec none. ``` ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass the CIs with the revised CIs. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #43038 from dongjoon-hyun/SPARK-45261. Authored-by: Dongjoon Hyun <dh...@apple.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- core/src/main/scala/org/apache/spark/SparkContext.scala | 3 ++- .../org/apache/spark/deploy/history/EventLogFileWriters.scala | 3 ++- core/src/main/scala/org/apache/spark/internal/config/package.scala | 1 + .../scala/org/apache/spark/deploy/history/EventLogTestHelper.scala | 7 ++++--- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala index b90601a5bbb..39c8e483ebd 100644 --- a/core/src/main/scala/org/apache/spark/SparkContext.scala +++ b/core/src/main/scala/org/apache/spark/SparkContext.scala @@ -464,7 +464,8 @@ class SparkContext(config: SparkConf) extends Logging { } _eventLogCodec = { - val compress = _conf.get(EVENT_LOG_COMPRESS) + val compress = _conf.get(EVENT_LOG_COMPRESS) && + !_conf.get(EVENT_LOG_COMPRESSION_CODEC).equals("none") if (compress && isEventLogEnabled) { Some(_conf.get(EVENT_LOG_COMPRESSION_CODEC)).map(CompressionCodec.getShortName) } else { diff --git a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala index 7d44cbd9f64..418d9171842 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/EventLogFileWriters.scala @@ -54,7 +54,8 @@ abstract class EventLogFileWriter( sparkConf: SparkConf, hadoopConf: Configuration) extends Logging { - protected val shouldCompress = sparkConf.get(EVENT_LOG_COMPRESS) + protected val shouldCompress = sparkConf.get(EVENT_LOG_COMPRESS) && + !sparkConf.get(EVENT_LOG_COMPRESSION_CODEC).equals("none") protected val shouldOverwrite = sparkConf.get(EVENT_LOG_OVERWRITE) protected val outputBufferSize = sparkConf.get(EVENT_LOG_OUTPUT_BUFFER_SIZE).toInt protected val fileSystem = Utils.getHadoopFileSystem(logBaseDir, hadoopConf) diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala index 05b2624b403..3da61f6c81d 100644 --- a/core/src/main/scala/org/apache/spark/internal/config/package.scala +++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala @@ -1907,6 +1907,7 @@ package object config { "the codec.") .version("3.0.0") .stringConf + .transform(_.toLowerCase(Locale.ROOT)) .createWithDefault("zstd") private[spark] val BUFFER_SIZE = diff --git a/core/src/test/scala/org/apache/spark/deploy/history/EventLogTestHelper.scala b/core/src/test/scala/org/apache/spark/deploy/history/EventLogTestHelper.scala index a68086256d1..ea8da010859 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/EventLogTestHelper.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/EventLogTestHelper.scala @@ -41,9 +41,10 @@ object EventLogTestHelper { conf.set(EVENT_LOG_BLOCK_UPDATES, true) conf.set(EVENT_LOG_TESTING, true) conf.set(EVENT_LOG_DIR, logDir.toString) - compressionCodec.foreach { codec => - conf.set(EVENT_LOG_COMPRESS, true) - conf.set(EVENT_LOG_COMPRESSION_CODEC, codec) + conf.set(EVENT_LOG_COMPRESS, true) + compressionCodec match { + case Some(codec) => conf.set(EVENT_LOG_COMPRESSION_CODEC, codec) + case _ => conf.set(EVENT_LOG_COMPRESSION_CODEC, "None") } conf.set(EVENT_LOG_STAGE_EXECUTOR_METRICS, true) conf --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org