This is an automated email from the ASF dual-hosted git repository.
yangzy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3db705794b [VL] Manually register velox conf on executor start (#9721)
3db705794b is described below
commit 3db705794bdd631e0658d2553f9c53e14a99db89
Author: Zouxxyy <[email protected]>
AuthorDate: Fri May 23 23:24:29 2025 +0800
[VL] Manually register velox conf on executor start (#9721)
---
.../backendsapi/velox/VeloxListenerApi.scala | 23 ++++++++----
.../gluten/backendsapi/VeloxListenerApiTest.java | 41 ++++++++++++++++++++++
2 files changed, 57 insertions(+), 7 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
index 6752a92baa..a7ee9e70cf 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
@@ -19,6 +19,7 @@ package org.apache.gluten.backendsapi.velox
import org.apache.gluten.backendsapi.ListenerApi
import
org.apache.gluten.backendsapi.arrow.ArrowBatchTypes.{ArrowJavaBatchType,
ArrowNativeBatchType}
import org.apache.gluten.config.GlutenConfig
+import org.apache.gluten.config.VeloxConfig
import org.apache.gluten.config.VeloxConfig._
import org.apache.gluten.execution.datasource.GlutenFormatFactory
import org.apache.gluten.expression.UDFMappings
@@ -210,15 +211,9 @@ class VeloxListenerApi extends ListenerApi with Logging {
}
// Initial native backend with configurations.
- var parsed = GlutenConfigUtil.parseConfig(conf.getAll.toMap)
-
- // Workaround for https://github.com/apache/incubator-gluten/issues/7837
- if (isDriver && !inLocalMode(conf)) {
- parsed += (COLUMNAR_VELOX_CACHE_ENABLED.key -> "false")
- }
NativeBackendInitializer
.forBackend(VeloxBackend.BACKEND_NAME)
- .initialize(newGlobalOffHeapMemoryListener(), parsed)
+ .initialize(newGlobalOffHeapMemoryListener(), parseConf(conf, isDriver))
// Inject backend-specific implementations to override spark classes.
GlutenFormatFactory.register(new VeloxParquetWriterInjects)
@@ -274,4 +269,18 @@ object VeloxListenerApi {
}
}
}
+
+ def parseConf(conf: SparkConf, isDriver: Boolean): Map[String, String] = {
+ // Ensure velox conf registered.
+ VeloxConfig.get
+
+ var parsed: Map[String, String] =
GlutenConfigUtil.parseConfig(conf.getAll.toMap)
+
+ // Workaround for https://github.com/apache/incubator-gluten/issues/7837
+ if (isDriver && !inLocalMode(conf)) {
+ parsed += (COLUMNAR_VELOX_CACHE_ENABLED.key -> "false")
+ }
+
+ parsed
+ }
}
diff --git
a/backends-velox/src/test/java/org/apache/gluten/backendsapi/VeloxListenerApiTest.java
b/backends-velox/src/test/java/org/apache/gluten/backendsapi/VeloxListenerApiTest.java
new file mode 100644
index 0000000000..8cb66b55f8
--- /dev/null
+++
b/backends-velox/src/test/java/org/apache/gluten/backendsapi/VeloxListenerApiTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.backendsapi;
+
+import org.apache.gluten.backendsapi.velox.VeloxListenerApi;
+
+import org.apache.spark.SparkConf;
+import org.junit.Test;
+
+import scala.collection.immutable.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class VeloxListenerApiTest {
+
+ @Test
+ public void testParseByteConfig() {
+ SparkConf conf = new SparkConf();
+ // Use conf string to prevent VeloxConfig object initialization.
+ conf.set("spark.gluten.sql.columnar.backend.velox.filePreloadThreshold",
"50MB");
+
+ Map<String, String> parsed = VeloxListenerApi.parseConf(conf, false);
+ assertEquals(
+ "52428800",
+
parsed.get("spark.gluten.sql.columnar.backend.velox.filePreloadThreshold").get());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
