This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new addf0b5e5ac [SPARK-42396][BUILD] Upgrade `Apache Kafka` to 3.4.0 addf0b5e5ac is described below commit addf0b5e5ac61c59e5c0c7d3030e46c9e88cd64f Author: bjornjorgensen <bjornjorgen...@gmail.com> AuthorDate: Sat Feb 11 22:40:18 2023 -0800 [SPARK-42396][BUILD] Upgrade `Apache Kafka` to 3.4.0 ### What changes were proposed in this pull request? Upgrade Apache Kafka from 3.3.2 to 3.4.0 ### Why are the changes needed? [CVE-2023-25194](https://www.cve.org/CVERecord?id=CVE-2023-25194) ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Pass GA Closes #39969 from bjornjorgensen/kafka-3.4. Authored-by: bjornjorgensen <bjornjorgen...@gmail.com> Signed-off-by: Dongjoon Hyun <dongj...@apache.org> --- .../org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala | 9 ++++++--- pom.xml | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala b/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala index b25e6c8e459..431b549f425 100644 --- a/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala +++ b/connector/kafka-0-10/src/test/scala/org/apache/spark/streaming/kafka010/KafkaRDDSuite.scala @@ -24,7 +24,7 @@ import scala.collection.JavaConverters._ import scala.concurrent.duration._ import scala.util.Random -import kafka.log.{CleanerConfig, LogCleaner, LogConfig, UnifiedLog} +import kafka.log.{CleanerConfig, LogCleaner, LogConfig, ProducerStateManagerConfig, UnifiedLog} import kafka.server.{BrokerTopicStats, LogDirFailureChannel} import kafka.utils.Pool import org.apache.kafka.common.TopicPartition @@ -94,16 +94,19 @@ class KafkaRDDSuite extends SparkFunSuite { logProps.put(LogConfig.MinCleanableDirtyRatioProp, java.lang.Float.valueOf(0.1f)) val logDirFailureChannel = new LogDirFailureChannel(1) val topicPartition = new TopicPartition(topic, partition) + val producerIdExpirationMs = Int.MaxValue + val producerStateManagerConfig = new ProducerStateManagerConfig(producerIdExpirationMs) + val logConfig = LogConfig(logProps) val log = UnifiedLog( dir, - LogConfig(logProps), + logConfig, 0L, 0L, mockTime.scheduler, new BrokerTopicStats(), mockTime, maxTransactionTimeoutMs = 5 * 60 * 1000, // KAFKA-13221 - Int.MaxValue, + producerStateManagerConfig, Int.MaxValue, logDirFailureChannel, lastShutdownClean = false, diff --git a/pom.xml b/pom.xml index 96ba96c212a..8fdc06b335c 100644 --- a/pom.xml +++ b/pom.xml @@ -137,7 +137,7 @@ <!-- Version used for internal directory structure --> <hive.version.short>2.3</hive.version.short> <!-- note that this should be compatible with Kafka brokers version 0.10 and up --> - <kafka.version>3.3.2</kafka.version> + <kafka.version>3.4.0</kafka.version> <!-- After 10.15.1.3, the minimum required version is JDK9 --> <derby.version>10.14.2.0</derby.version> <parquet.version>1.12.3</parquet.version> --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org