This is an automated email from the ASF dual-hosted git repository.
zaynt4606 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 34f1ab5a59 [CELEBORN-2478] Add celeborn-client build information to
Spark UI Celeborn tab
34f1ab5a59 is described below
commit 34f1ab5a598ce8fec44af6383e1c1c1bc4ef5072
Author: yew1eb <[email protected]>
AuthorDate: Wed Oct 21 11:02:41 2026 +0800
[CELEBORN-2478] Add celeborn-client build information to Spark UI Celeborn
tab
### Why are the changes needed?
The Celeborn tab in the Spark UI does not show which celeborn-client
version the application is using, which makes version-mismatch debugging
between a Spark job and the cluster unnecessarily hard. See CELEBORN-2478.
### What changes were included?
Mirrors Uniffle's "Uniffle Build Information" pattern — the info is baked
into the client at compile time and rendered by the Spark UI plugin, with **no
reporting to the master**:
- **Compile-time constants** (`org.apache.celeborn.ProjectConstants`,
`VERSION` / `REVISION` / `getGitCommitId()`) generated from both build systems:
- Maven: `templating-maven-plugin` + `buildnumber-maven-plugin` +
`properties-maven-plugin` on the new `common/src/main/java-templates` (same
recipe as Uniffle)
- SBT: the existing version source generator in
`project/CelebornBuild.scala` is extended to also emit the same Java class
using `gitHeadCommit`
- **Persistence**: `CelebornDriverPlugin.init` writes a
`CelebornBuildInfoUIData` entry into the Spark KVStore, so it survives
application end and shows up on History Server replay
- **UI**: a collapsible "Celeborn Client Build Information" table rendered
in `CelebornShufflePage`, ahead of "Celeborn Properties"
- Tests: `CelebornBuildInfoSuite` covering KVStore round-trip and constant
sanity
### How was this patch tested?
- `./build/mvn -Pspark-3.5 -pl common,client-spark/spark-3-ui -am clean
install` — build success; the generated `ProjectConstants.java` contains the
correct version and current git revision
- `./build/mvn -Pspark-3.5 -pl client-spark/spark-3-ui clean test` — 8/8
tests pass (5 existing + 3 new)
- `./build/sbt "celeborn-common/Compile/compile"` — SBT generates identical
constants
- Spotless formatting applied
<img width="2294" height="1126" alt="image"
src="https://github.com/user-attachments/assets/ef5fa366-1a71-4606-a6cc-1340664543c0";
/>
Closes #3870 from yew1eb/spark-ui-client-build-info.
Authored-by: yew1eb <[email protected]>
Signed-off-by: zhengtao <[email protected]>
AI-Contributed/Feature: 0/255
AI-Contributed/UT: 0/124
---
.github/workflows/cpp_integration.yml | 4 ++
.github/workflows/maven.yml | 22 ++++++++
.../shuffle/celeborn/CelebornBuildInfoEvent.scala | 31 ++++++++++
.../spark/shuffle/celeborn/CelebornListener.scala | 21 ++++++-
.../spark/shuffle/celeborn/CelebornPlugin.scala | 9 ++-
.../shuffle/celeborn/CelebornStatusStore.scala | 17 ++++++
.../shuffle/celeborn/ui/CelebornShufflePage.scala | 19 +++++++
.../shuffle/celeborn/CelebornBuildInfoSuite.scala | 66 ++++++++++++++++++++++
.../shuffle/celeborn/CelebornListenerSuite.scala | 58 +++++++++++++++++++
common/pom.xml | 53 +++++++++++++++++
.../org/apache/celeborn/ProjectConstants.java | 35 ++++++++++++
docker/spark/conf/spark-defaults.conf | 5 ++
pom.xml | 21 +++++++
project/CelebornBuild.scala | 18 ++++--
14 files changed, 371 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/cpp_integration.yml
b/.github/workflows/cpp_integration.yml
index ff52d41fc5..7841311c0c 100644
--- a/.github/workflows/cpp_integration.yml
+++ b/.github/workflows/cpp_integration.yml
@@ -87,6 +87,10 @@ jobs:
java-version: 8
cache: maven
check-latest: false
+ # git inside the container cannot read the workspace (different uid);
+ # the safe.directory added by actions/checkout uses a temporary HOME.
+ - name: Configure git safe directory
+ run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Compile & Install Celeborn Java
run: build/mvn clean install -DskipTests
- name: Run Java-Write Cpp-Read Hybrid Integration Test (NONE
Decompression)
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
index 17c9cfc442..3a17354f48 100644
--- a/.github/workflows/maven.yml
+++ b/.github/workflows/maven.yml
@@ -18,6 +18,28 @@ concurrency:
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
+ # Source distributions have no .git directory; verify they still build via
+ # the revisionOnScmFailure fallback.
+ source-archive:
+ runs-on: ubuntu-22.04
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup JDK 8
+ uses: actions/setup-java@v4
+ with:
+ distribution: zulu
+ java-version: 8
+ cache: maven
+ check-latest: false
+ - name: Build from source archive without git metadata
+ run: |
+ mkdir -p /tmp/src-archive
+ git archive HEAD | tar -x -C /tmp/src-archive
+ cd /tmp/src-archive
+ build/mvn -pl common -am -DskipTests package
+ grep -q 'REVISION = "N/A"' \
+
common/target/generated-sources/java-templates/org/apache/celeborn/ProjectConstants.java
+
service:
runs-on: ubuntu-22.04
strategy:
diff --git
a/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornBuildInfoEvent.scala
b/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornBuildInfoEvent.scala
new file mode 100644
index 0000000000..0107826365
--- /dev/null
+++
b/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornBuildInfoEvent.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.shuffle.celeborn
+
+import org.apache.spark.scheduler.SparkListenerEvent
+
+/**
+ * Build info of the celeborn-client bundled with the application. Recorded in
+ * the event log; the listener writes it to the KVStore both for the live UI
+ * and on History Server replay.
+ */
+private[celeborn] case class CelebornBuildInfoEvent(
+ version: String,
+ commitId: String,
+ revision: String)
+ extends SparkListenerEvent
diff --git
a/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornListener.scala
b/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornListener.scala
index 1348ee2f8f..d0cd03e3c3 100644
---
a/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornListener.scala
+++
b/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornListener.scala
@@ -30,9 +30,10 @@ import org.apache.spark.util.kvstore.KVStore
* HistoryServer replay.
*
* When `requirePluginOptIn` is true (History Server replay), collection stays
- * disabled until the application's recorded `spark.plugins` or
- * `spark.plugins.defaultList` contains [[CelebornPlugin]], and an enable
marker
- * is persisted so `setupUI` can decide whether to attach the Celeborn tab.
+ * disabled until the application opted in via its recorded `spark.plugins` or
+ * `spark.plugins.defaultList`, or until a [[CelebornBuildInfoEvent]] is
+ * replayed, and an enable marker is persisted so `setupUI` can decide whether
+ * to attach the Celeborn tab.
*/
private[celeborn] class CelebornListener(
val kvstore: KVStore,
@@ -104,6 +105,20 @@ private[celeborn] class CelebornListener(
}
}
+ override def onOtherEvent(event: SparkListenerEvent): Unit = event match {
+ case e: CelebornBuildInfoEvent =>
+ // Receiving the event also proves the application loaded the plugin.
+ kvstore.write(new CelebornBuildInfoUIData(Seq(
+ "Version" -> e.version,
+ "Commit Id" -> e.commitId,
+ "Revision" -> e.revision)))
+ if (!pluginEnabled) {
+ pluginEnabled = true
+ kvstore.write(new CelebornExtensionEnabledUIData())
+ }
+ case _ =>
+ }
+
override def onApplicationEnd(applicationEnd: SparkListenerApplicationEnd):
Unit = {
mayUpdate(force = true)
logInfo("CelebornListener: application ended, final flush completed")
diff --git
a/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornPlugin.scala
b/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornPlugin.scala
index d0e0caae50..a38a419cac 100644
---
a/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornPlugin.scala
+++
b/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornPlugin.scala
@@ -49,7 +49,14 @@ private class CelebornDriverPlugin extends DriverPlugin with
Logging {
logInfo("Initializing CelebornDriverPlugin...")
this.sc = sc
val kvStore = sc.statusStore.store
- new CelebornListener(kvStore, sc.conf).register(sc)
+ val listener = new CelebornListener(kvStore, sc.conf)
+ listener.register(sc)
+ // The event is recorded in the event log and the listener persists it into
+ // the KVStore, for the live UI and History Server replay alike.
+ sc.listenerBus.post(CelebornBuildInfoEvent(
+ version = org.apache.celeborn.ProjectConstants.VERSION,
+ commitId = org.apache.celeborn.ProjectConstants.getGitCommitId,
+ revision = org.apache.celeborn.ProjectConstants.REVISION))
java.util.Collections.emptyMap[String, String]()
}
diff --git
a/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornStatusStore.scala
b/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornStatusStore.scala
index 9a8ffe5cf1..cfb255b026 100644
---
a/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornStatusStore.scala
+++
b/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/CelebornStatusStore.scala
@@ -40,6 +40,14 @@ private[celeborn] class CelebornPropertiesUIData(
def id: String = classOf[CelebornPropertiesUIData].getName
}
+private[celeborn] class CelebornBuildInfoUIData(
+ val info: Seq[(String, String)]) {
+
+ @JsonIgnore
+ @KVIndex
+ def id: String = classOf[CelebornBuildInfoUIData].getName
+}
+
/**
* Marker persisted during History Server replay when the application's
recorded
* `spark.plugins` or `spark.plugins.defaultList` contains [[CelebornPlugin]].
@@ -74,6 +82,15 @@ private[celeborn] class CelebornStatusStore(store: KVStore) {
}
}
+ def buildInfo(): CelebornBuildInfoUIData = {
+ val kClass = classOf[CelebornBuildInfoUIData]
+ try {
+ store.read(kClass, kClass.getName)
+ } catch {
+ case _: NoSuchElementException => new CelebornBuildInfoUIData(Seq.empty)
+ }
+ }
+
def extensionEnabled(): Boolean = {
val kClass = classOf[CelebornExtensionEnabledUIData]
try {
diff --git
a/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/ui/CelebornShufflePage.scala
b/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/ui/CelebornShufflePage.scala
index 4a57652fcf..33bc4027a7 100644
---
a/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/ui/CelebornShufflePage.scala
+++
b/client-spark/spark-3-ui/src/main/scala/org/apache/spark/shuffle/celeborn/ui/CelebornShufflePage.scala
@@ -53,6 +53,7 @@ private[celeborn] class CelebornShufflePage(parent:
CelebornUITab)
private def renderBody(request: HttpServletRequest): Seq[Node] = {
val taskInfo = store.aggregatedTaskInfo()
val properties = store.celebornProperties()
+ val buildInfo = store.buildInfo()
val writeBytes = taskInfo.shuffleWriteBytes
val readBytes = taskInfo.shuffleReadBytes
@@ -100,9 +101,27 @@ private[celeborn] class CelebornShufflePage(parent:
CelebornUITab)
fixedWidth = true,
headerClasses = headerClasses)
+ val buildInfoTable = UIUtils.listingTable(
+ propertyHeader,
+ propertyRow,
+ buildInfo.info,
+ fixedWidth = true,
+ headerClasses = headerClasses)
+
val content =
<span>
{summary}
+ <span class="collapse-aggregated-celebornBuildInfo collapse-table"
+ onClick="collapseTable('collapse-aggregated-celebornBuildInfo',
+ 'aggregated-celebornBuildInfo')">
+ <h4>
+ <span class="collapse-table-arrow arrow-closed"></span>
+ <a>Celeborn Client Build Information</a>
+ </h4>
+ </span>
+ <div class="aggregated-celebornBuildInfo collapsible-table collapsed">
+ {buildInfoTable}
+ </div>
<span class="collapse-aggregated-celebornProperties collapse-table"
onClick="collapseTable('collapse-aggregated-celebornProperties',
'aggregated-celebornProperties')">
diff --git
a/client-spark/spark-3-ui/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornBuildInfoSuite.scala
b/client-spark/spark-3-ui/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornBuildInfoSuite.scala
new file mode 100644
index 0000000000..6cf22c3724
--- /dev/null
+++
b/client-spark/spark-3-ui/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornBuildInfoSuite.scala
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific
+ * language governing permissions and limitations under the License.
+ */
+
+package org.apache.spark.shuffle.celeborn
+
+import org.apache.spark.util.kvstore.InMemoryStore
+import org.junit.Assert.{assertEquals, assertFalse, assertTrue}
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.JUnit4
+
+import org.apache.celeborn.ProjectConstants
+
+@RunWith(classOf[JUnit4])
+class CelebornBuildInfoSuite {
+
+ @Test
+ def buildInfoRoundTrip(): Unit = {
+ val store = new InMemoryStore()
+ val statusStore = new CelebornStatusStore(store)
+
+ // Before the plugin persists anything, the UI falls back to empty build
info.
+ assertTrue(statusStore.buildInfo().info.isEmpty)
+
+ // Mirrors what CelebornListener.onOtherEvent persists.
+ store.write(new CelebornBuildInfoUIData(Seq(
+ "Version" -> ProjectConstants.VERSION,
+ "Commit Id" -> ProjectConstants.getGitCommitId,
+ "Revision" -> ProjectConstants.REVISION)))
+
+ val info = statusStore.buildInfo().info
+ val infoMap = info.toMap
+ assertEquals(3, info.size)
+ assertEquals(ProjectConstants.VERSION, infoMap("Version"))
+ assertEquals(ProjectConstants.REVISION, infoMap("Revision"))
+ }
+
+ @Test
+ def commitIdIsShortRevision(): Unit = {
+ val expected =
+ if (ProjectConstants.REVISION.length >= 8)
ProjectConstants.REVISION.substring(0, 8)
+ else ProjectConstants.REVISION
+ assertEquals(expected, ProjectConstants.getGitCommitId)
+ }
+
+ @Test
+ def projectConstantsArePopulated(): Unit = {
+ assertFalse(ProjectConstants.VERSION.isEmpty)
+ assertFalse(ProjectConstants.REVISION.isEmpty)
+ }
+}
diff --git
a/client-spark/spark-3-ui/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornListenerSuite.scala
b/client-spark/spark-3-ui/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornListenerSuite.scala
index ed9ff78ad4..21af11c0cc 100644
---
a/client-spark/spark-3-ui/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornListenerSuite.scala
+++
b/client-spark/spark-3-ui/src/test/scala/org/apache/spark/shuffle/celeborn/CelebornListenerSuite.scala
@@ -167,4 +167,62 @@ class CelebornListenerSuite {
// Non-Celeborn properties are filtered out.
assertFalse(props.contains("spark.executor.memory"))
}
+
+ @Test
+ def buildInfoRestoredOnReplayFromRecordedEvent(): Unit = {
+ // History Server replay: the recorded event restores the build info from
+ // the application itself and acts as the opt-in marker.
+ val store = new InMemoryStore()
+ val statusStore = new CelebornStatusStore(store)
+ val listener = new CelebornListener(store, new SparkConf(),
requirePluginOptIn = true)
+ assertTrue(statusStore.buildInfo().info.isEmpty)
+
+ listener.onOtherEvent(CelebornBuildInfoEvent(
+ version = "1.0.0-test",
+ commitId = "abcdef12",
+ revision = "abcdef1234567890"))
+
+ assertTrue(statusStore.extensionEnabled())
+ val info = statusStore.buildInfo().info.toMap
+ assertEquals(3, info.size)
+ assertEquals("1.0.0-test", info("Version"))
+ assertEquals("abcdef12", info("Commit Id"))
+ assertEquals("abcdef1234567890", info("Revision"))
+ assertTrue(statusStore.celebornProperties().info.isEmpty)
+ }
+
+ @Test
+ def buildInfoEnablesCollectionOnReplay(): Unit = {
+ // Metrics are collected once the build info event enabled the plugin.
+ val store = new InMemoryStore()
+ val statusStore = new CelebornStatusStore(store)
+ val listener = new CelebornListener(store, new SparkConf(),
requirePluginOptIn = true)
+
+ listener.onOtherEvent(CelebornBuildInfoEvent(
+ version = "1.0.0-test",
+ commitId = "abcdef12",
+ revision = "abcdef1234567890"))
+ listener.onTaskEnd(newTaskEnd(100L, 10L, 1L, 5L))
+
+ assertEquals(100L, statusStore.aggregatedTaskInfo().shuffleWriteBytes)
+ }
+
+ @Test
+ def buildInfoPersistedByLiveListenerFromPostedEvent(): Unit = {
+ // Live path uses the same code path as History Server replay.
+ val store = new InMemoryStore()
+ val statusStore = new CelebornStatusStore(store)
+ val listener = new CelebornListener(store, new SparkConf())
+
+ listener.onOtherEvent(CelebornBuildInfoEvent(
+ version = "1.0.0-test",
+ commitId = "abcdef12",
+ revision = "abcdef1234567890"))
+
+ val info = statusStore.buildInfo().info.toMap
+ assertEquals(3, info.size)
+ assertEquals("1.0.0-test", info("Version"))
+ assertEquals("abcdef12", info("Commit Id"))
+ assertEquals("abcdef1234567890", info("Revision"))
+ }
}
diff --git a/common/pom.xml b/common/pom.xml
index 7f8d59b897..4f60b501e0 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -192,6 +192,59 @@
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
</plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>buildnumber-maven-plugin</artifactId>
+ <configuration>
+
<outputDirectory>${project.build.directory}/generated/build-metadata</outputDirectory>
+ <outputName>build.properties</outputName>
+ <revisionPropertyName>git.revision</revisionPropertyName>
+ <!-- Revision fallback when git metadata is unavailable, matching the
+ SBT fallback in project/CelebornBuild.scala. -->
+ <revisionOnScmFailure>N/A</revisionOnScmFailure>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>create-metadata</goal>
+ </goals>
+ <phase>initialize</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>properties-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>read-project-properties</goal>
+ </goals>
+ <phase>initialize</phase>
+ <configuration>
+ <files>
+
<file>${project.build.directory}/generated/build-metadata/build.properties</file>
+ </files>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>templating-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>filter-src</id>
+ <goals>
+ <goal>filter-sources</goal>
+ </goals>
+ <configuration>
+
<sourceDirectory>${basedir}/src/main/java-templates</sourceDirectory>
+
<outputDirectory>${project.build.directory}/generated-sources/java-templates</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
<extensions>
<extension>
diff --git
a/common/src/main/java-templates/org/apache/celeborn/ProjectConstants.java
b/common/src/main/java-templates/org/apache/celeborn/ProjectConstants.java
new file mode 100644
index 0000000000..3e6837e5e3
--- /dev/null
+++ b/common/src/main/java-templates/org/apache/celeborn/ProjectConstants.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific
+ * language governing permissions and limitations under the License.
+ */
+
+package org.apache.celeborn;
+
+public final class ProjectConstants {
+ /* Project version, specified in maven property. **/
+ public static final String VERSION = "${project.version}";
+ /* The latest git revision of at the time of building**/
+ public static final String REVISION = "${git.revision}";
+
+ private ProjectConstants() {} // prevent instantiation
+
+ public static String getGitCommitId() {
+ if (ProjectConstants.REVISION.length() >= 8) {
+ return ProjectConstants.REVISION.substring(0, 8);
+ }
+ return ProjectConstants.REVISION;
+ }
+}
diff --git a/docker/spark/conf/spark-defaults.conf
b/docker/spark/conf/spark-defaults.conf
index 583f7b1b5a..9692f5ab6f 100644
--- a/docker/spark/conf/spark-defaults.conf
+++ b/docker/spark/conf/spark-defaults.conf
@@ -26,6 +26,11 @@
# Route shuffle through Celeborn. The shuffle manager class name changed in
# Celeborn 0.3.0 (was RssShuffleManager); this repo is >= 0.3.0.
spark.shuffle.manager
org.apache.spark.shuffle.celeborn.SparkShuffleManager
+
+# Attach the Celeborn UI plugin so the "Celeborn" tab (shuffle stats +
+# celeborn-client build information) shows up in the Spark application UI.
+spark.plugins
org.apache.spark.shuffle.celeborn.CelebornPlugin
+
# Kryo is required: the shaded client relocates packages, which the Java
# serializer cannot handle.
spark.serializer
org.apache.spark.serializer.KryoSerializer
diff --git a/pom.xml b/pom.xml
index 878646622d..45fb65a35b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -140,7 +140,10 @@
<shading.prefix>org.apache.celeborn.shaded</shading.prefix>
<maven.plugin.antrun.version>3.0.0</maven.plugin.antrun.version>
+ <maven.plugin.buildnumber.version>1.4</maven.plugin.buildnumber.version>
<maven.plugin.clean.version>3.2.0</maven.plugin.clean.version>
+ <maven.plugin.properties.version>1.1.0</maven.plugin.properties.version>
+ <maven.plugin.templating.version>1.0.0</maven.plugin.templating.version>
<maven.plugin.compiler.version>3.12.1</maven.plugin.compiler.version>
<maven.plugin.dependency.version>3.1.1</maven.plugin.dependency.version>
<maven.plugin.flatten.version>1.3.0</maven.plugin.flatten.version>
@@ -1002,6 +1005,24 @@
</executions>
</plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>buildnumber-maven-plugin</artifactId>
+ <version>${maven.plugin.buildnumber.version}</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>properties-maven-plugin</artifactId>
+ <version>${maven.plugin.properties.version}</version>
+ </plugin>
+
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>templating-maven-plugin</artifactId>
+ <version>${maven.plugin.templating.version}</version>
+ </plugin>
+
<plugin>
<groupId>org.xolstice.maven.plugins</groupId>
<artifactId>protobuf-maven-plugin</artifactId>
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index cfaba0ec51..8595f62f58 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -725,16 +725,26 @@ object CelebornCommon {
Seq(Dependencies.jdkTools) else Nil),
Compile / sourceGenerators += Def.task {
- val file = (Compile / sourceManaged).value / "org" / "apache" /
"celeborn" / "package.scala"
- streams.value.log.info(s"geneate version information file
${file.toPath}")
- IO.write(file,
+ val dir = (Compile / sourceManaged).value / "org" / "apache" /
"celeborn"
+ val packageFile = dir / "package.scala"
+ streams.value.log.info(s"geneate version information file
${packageFile.toPath}")
+ IO.write(packageFile,
s"""package org.apache
|
|package object celeborn {
| val VERSION = "${version.value}"
|}
|""".stripMargin)
- Seq(file)
+ // Filter the maven template (src/main/java-templates, processed by
+ // templating-maven-plugin in maven builds) the same way, so both build
+ // systems produce the same compile-time build info constants.
+ val constantsFile = dir / "ProjectConstants.java"
+ val template = baseDirectory.value / "src" / "main" / "java-templates"
/
+ "org" / "apache" / "celeborn" / "ProjectConstants.java"
+ IO.write(constantsFile, IO.read(template)
+ .replace("${project.version}", version.value)
+ .replace("${git.revision}", gitHeadCommit.value.getOrElse("N/A")))
+ Seq(packageFile, constantsFile)
// generate version task depends on PB generate to avoid concurrency
generate source files
}.dependsOn(Compile / PB.generate),
