This is an automated email from the ASF dual-hosted git repository.
baibaichen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new a3e5e842ff [VL] Fix driver endpoint startup initialization race
(#13069)
a3e5e842ff is described below
commit a3e5e842ff5859d5b1ab59491b7692b7db2c1c49
Author: Chang Chen <[email protected]>
AuthorDate: Mon Sep 21 14:32:10 2026 +0800
[VL] Fix driver endpoint startup initialization race (#13069)
---
.../apache/spark/rpc/GlutenDriverEndpoint.scala | 5 +-
.../spark/rpc/GlutenDriverEndpointSuite.scala | 53 ++++++++++++++++++++++
2 files changed, 55 insertions(+), 3 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/spark/rpc/GlutenDriverEndpoint.scala
b/backends-velox/src/main/scala/org/apache/spark/rpc/GlutenDriverEndpoint.scala
index 6503bbe6ad..e8b96b9697 100644
---
a/backends-velox/src/main/scala/org/apache/spark/rpc/GlutenDriverEndpoint.scala
+++
b/backends-velox/src/main/scala/org/apache/spark/rpc/GlutenDriverEndpoint.scala
@@ -37,8 +37,7 @@ class GlutenDriverEndpoint extends IsolatedRpcEndpoint with
Logging {
protected val totalRegisteredExecutors = new AtomicInteger(0)
- private val driverEndpoint: RpcEndpointRef =
- rpcEnv.setupEndpoint(GlutenRpcConstants.GLUTEN_DRIVER_ENDPOINT_NAME, this)
+ rpcEnv.setupEndpoint(GlutenRpcConstants.GLUTEN_DRIVER_ENDPOINT_NAME, this)
// TODO(yuan): get thread cnt from spark context
override def threadCount(): Int = 1
@@ -90,7 +89,7 @@ class GlutenDriverEndpoint extends IsolatedRpcEndpoint with
Logging {
}
override def onStart(): Unit = {
- logInfo(s"Initialized GlutenDriverEndpoint, address:
${driverEndpoint.address.toString()}.")
+ logInfo(s"Initialized GlutenDriverEndpoint, address: ${self.address}.")
}
}
diff --git
a/gluten-ut/spark41/src/test/backends-velox/org/apache/spark/rpc/GlutenDriverEndpointSuite.scala
b/gluten-ut/spark41/src/test/backends-velox/org/apache/spark/rpc/GlutenDriverEndpointSuite.scala
new file mode 100644
index 0000000000..e277005070
--- /dev/null
+++
b/gluten-ut/spark41/src/test/backends-velox/org/apache/spark/rpc/GlutenDriverEndpointSuite.scala
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.rpc
+
+import org.apache.spark.{SparkEnv, SparkFunSuite}
+
+import org.mockito.ArgumentMatchers.{any, eq => eqTo}
+import org.mockito.Mockito.{mock, verify, when}
+
+class GlutenDriverEndpointSuite extends SparkFunSuite {
+ test("onStart can run before setupEndpoint returns") {
+ val previousEnv = SparkEnv.get
+ val sparkEnv = mock(classOf[SparkEnv])
+ val rpcEnv = mock(classOf[RpcEnv])
+ val endpointRef = mock(classOf[RpcEndpointRef])
+ when(sparkEnv.rpcEnv).thenReturn(rpcEnv)
+ when(endpointRef.address).thenReturn(RpcAddress("localhost", 12345))
+ when(rpcEnv.setupEndpoint(
+ eqTo(GlutenRpcConstants.GLUTEN_DRIVER_ENDPOINT_NAME),
+ any[RpcEndpoint]))
+ .thenAnswer {
+ invocation =>
+ val endpoint = invocation.getArgument[RpcEndpoint](1)
+ when(rpcEnv.endpointRef(endpoint)).thenReturn(endpointRef)
+ // Spark registers self before onStart, which may precede
setupEndpoint returning.
+ endpoint.onStart()
+ endpointRef
+ }
+
+ try {
+ SparkEnv.set(sparkEnv)
+ val endpoint = new GlutenDriverEndpoint
+ assert(endpoint.self eq endpointRef)
+ verify(endpointRef).address
+ } finally {
+ SparkEnv.set(previousEnv)
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
