This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch branch-0.4
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/branch-0.4 by this push:
new c397ded55 [CELEBORN-1298] Support Spark2.4 with Scala2.12
c397ded55 is described below
commit c397ded55d6dff273ad9f358f311c78b79219635
Author: Keyong Zhou <[email protected]>
AuthorDate: Thu Feb 29 17:58:54 2024 +0800
[CELEBORN-1298] Support Spark2.4 with Scala2.12
As title
As title
no
GA
Closes #2344 from waitinfuture/1298-1.
Lead-authored-by: Keyong Zhou <[email protected]>
Co-authored-by: waitinfuture <[email protected]>
Signed-off-by: waitinfuture <[email protected]>
(cherry picked from commit af8c159c9f655d479f6631f6fbbad810c2c6d023)
Signed-off-by: waitinfuture <[email protected]>
---
README.md | 13 ++++++++++++-
.../spark/shuffle/celeborn/CelebornShuffleReader.scala | 4 ++--
2 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/README.md b/README.md
index cba693169..0a9304c48 100644
--- a/README.md
+++ b/README.md
@@ -55,7 +55,7 @@ package apache-celeborn-${project.version}-bin.tgz will be
generated.
| | Java 8/Scala 2.11 | Java 8/Scala 2.12 | Java 11/Scala 2.12 |
Java 17/Scala 2.12 | Java 8/Scala 2.13 | Java 11/Scala 2.13 | Java 17/Scala
2.13 |
|------------|-------------------|-------------------|--------------------|--------------------|-------------------|--------------------|--------------------|
-| Spark 2.4 | ✔ | ❌ | ❌ |
❌ | ❌ | ❌ | ❌
|
+| Spark 2.4 | ✔ | ✔ | ❌ |
❌ | ❌ | ❌ | ❌
|
| Spark 3.0 | ❌ | ✔ | ✔ |
❌ | ❌ | ❌ | ❌
|
| Spark 3.1 | ❌ | ✔ | ✔ |
❌ | ❌ | ❌ | ❌
|
| Spark 3.2 | ❌ | ✔ | ✔ |
❌ | ✔ | ✔ | ❌
|
@@ -67,6 +67,17 @@ package apache-celeborn-${project.version}-bin.tgz will be
generated.
| Flink 1.17 | ❌ | ✔ | ✔ |
❌ | ❌ | ❌ | ❌
|
| Flink 1.18 | ❌ | ✔ | ✔ |
❌ | ❌ | ❌ | ❌
|
+To compile the client for Spark 2.4 with Scala 2.12, please use the following
command
+
+- Scala 2.12.8/2.12.9/2.12.10
+```shell
+./build/make-distribution.sh -DskipTests -Pspark-2.4
-Dscala.version=${scala.version} -Dscala.binary.version=2.12
-Dmaven.plugin.scala.version=3.2.2 -Dmaven.plugin.silencer.version=1.6.0
+```
+- Scala 2.12.13-2.12.18
+```shell
+./build/make-distribution.sh -DskipTests -Pspark-2.4
-Dscala.version=${scala.version} -Dscala.binary.version=2.12
+```
+
### Package Details
Build procedure will create a compressed package.
diff --git
a/client-spark/spark-2/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReader.scala
b/client-spark/spark-2/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReader.scala
index ad1a620bd..b74db710c 100644
---
a/client-spark/spark-2/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReader.scala
+++
b/client-spark/spark-2/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornShuffleReader.scala
@@ -142,7 +142,7 @@ class CelebornShuffleReader[K, C](
metricsCallback.incReadTime(
TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startFetchWait))
// ensure inputStream is closed when task completes
- context.addTaskCompletionListener(_ => inputStream.close())
+ context.addTaskCompletionListener[Unit](_ => inputStream.close())
(partitionId, inputStream)
} else {
(partitionId, CelebornInputStream.empty())
@@ -206,7 +206,7 @@ class CelebornShuffleReader[K, C](
context.taskMetrics().incDiskBytesSpilled(sorter.diskBytesSpilled)
context.taskMetrics().incPeakExecutionMemory(sorter.peakMemoryUsedBytes)
// Use completion callback to stop sorter if task was
finished/cancelled.
- context.addTaskCompletionListener(_ => {
+ context.addTaskCompletionListener[Unit](_ => {
sorter.stop()
})
CompletionIterator[Product2[K, C], Iterator[Product2[K,
C]]](sorter.iterator, sorter.stop())
