This is an automated email from the ASF dual-hosted git repository.
yangzy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 0800596b5 [VL] Make jni debug workspace configurable (#6228)
0800596b5 is described below
commit 0800596b5d23c55443fd74fa6461c5264619c6b4
Author: Yang Zhang <[email protected]>
AuthorDate: Wed Jun 26 20:01:17 2024 +0800
[VL] Make jni debug workspace configurable (#6228)
---
.../apache/gluten/backendsapi/velox/VeloxListenerApi.scala | 7 +++----
.../main/java/org/apache/gluten/vectorized/JniWorkspace.java | 4 ++--
.../src/main/scala/org/apache/gluten/GlutenConfig.scala | 12 +++++++++---
3 files changed, 14 insertions(+), 9 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
index 41b56804b..81f06478c 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
@@ -161,10 +161,9 @@ class VeloxListenerApi extends ListenerApi {
private def initialize(conf: SparkConf, isDriver: Boolean): Unit = {
SparkDirectoryUtil.init(conf)
UDFResolver.resolveUdfConf(conf, isDriver = isDriver)
- val debugJni = conf.getBoolean(GlutenConfig.GLUTEN_DEBUG_MODE,
defaultValue = false) &&
- conf.getBoolean(GlutenConfig.GLUTEN_DEBUG_KEEP_JNI_WORKSPACE,
defaultValue = false)
- if (debugJni) {
- JniWorkspace.enableDebug()
+ if (conf.getBoolean(GlutenConfig.GLUTEN_DEBUG_KEEP_JNI_WORKSPACE,
defaultValue = false)) {
+ val debugDir = conf.get(GlutenConfig.GLUTEN_DEBUG_KEEP_JNI_WORKSPACE_DIR)
+ JniWorkspace.enableDebug(debugDir)
}
val loader = JniWorkspace.getDefault.libLoader
diff --git
a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniWorkspace.java
b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniWorkspace.java
index a7c12387a..810c945d3 100644
--- a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniWorkspace.java
+++ b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniWorkspace.java
@@ -75,14 +75,14 @@ public class JniWorkspace {
}
}
- public static void enableDebug() {
+ public static void enableDebug(String debugDir) {
// Preserve the JNI libraries even after process exits.
// This is useful for debugging native code if the debug symbols were
embedded in
// the libraries.
synchronized (DEFAULT_INSTANCE_INIT_LOCK) {
if (DEBUG_INSTANCE == null) {
final File tempRoot =
- Paths.get("/tmp").resolve("gluten-jni-debug-" +
UUID.randomUUID()).toFile();
+ Paths.get(debugDir).resolve("gluten-jni-debug-" +
UUID.randomUUID()).toFile();
try {
FileUtils.forceMkdir(tempRoot);
} catch (IOException e) {
diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
index cc2d6ac5f..89933cc58 100644
--- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -391,8 +391,7 @@ class GlutenConfig(conf: SQLConf) extends Logging {
conf.getConf(COLUMNAR_VELOX_MEMORY_USE_HUGE_PAGES)
def debug: Boolean = conf.getConf(DEBUG_ENABLED)
- def debugKeepJniWorkspace: Boolean =
- conf.getConf(DEBUG_ENABLED) && conf.getConf(DEBUG_KEEP_JNI_WORKSPACE)
+ def debugKeepJniWorkspace: Boolean = conf.getConf(DEBUG_KEEP_JNI_WORKSPACE)
def taskStageId: Int = conf.getConf(BENCHMARK_TASK_STAGEID)
def taskPartitionId: Int = conf.getConf(BENCHMARK_TASK_PARTITIONID)
def taskId: Long = conf.getConf(BENCHMARK_TASK_TASK_ID)
@@ -553,6 +552,7 @@ object GlutenConfig {
val GLUTEN_DEBUG_MODE = "spark.gluten.sql.debug"
val GLUTEN_DEBUG_KEEP_JNI_WORKSPACE =
"spark.gluten.sql.debug.keepJniWorkspace"
+ val GLUTEN_DEBUG_KEEP_JNI_WORKSPACE_DIR =
"spark.gluten.sql.debug.keepJniWorkspaceDir"
// Added back to Spark Conf during executor initialization
val GLUTEN_NUM_TASK_SLOTS_PER_EXECUTOR_KEY =
"spark.gluten.numTaskSlotsPerExecutor"
@@ -1580,11 +1580,17 @@ object GlutenConfig {
.createWithDefault(false)
val DEBUG_KEEP_JNI_WORKSPACE =
- buildConf(GLUTEN_DEBUG_KEEP_JNI_WORKSPACE)
+ buildStaticConf(GLUTEN_DEBUG_KEEP_JNI_WORKSPACE)
.internal()
.booleanConf
.createWithDefault(false)
+ val DEBUG_KEEP_JNI_WORKSPACE_DIR =
+ buildStaticConf(GLUTEN_DEBUG_KEEP_JNI_WORKSPACE_DIR)
+ .internal()
+ .stringConf
+ .createWithDefault("/tmp")
+
val BENCHMARK_TASK_STAGEID =
buildConf("spark.gluten.sql.benchmark_task.stageId")
.internal()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
