This is an automated email from the ASF dual-hosted git repository.
zhli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 07b18e75db [Core] Support Jvm memory shrinking for
DynamicOffHeapSizingMemoryTarget (#9585)
07b18e75db is described below
commit 07b18e75db083630b9c1f141f11a4a7894a404f7
Author: Zhen Li <[email protected]>
AuthorDate: Wed Jul 9 20:27:37 2025 +0800
[Core] Support Jvm memory shrinking for DynamicOffHeapSizingMemoryTarget
(#9585)
[Core] Support Jvm memory shrinking for DynamicOffHeapSizingMemoryTarget
---
.../execution/DynamicOffHeapSizingSuite.scala | 59 ++--
.../DynamicOffHeapSizingMemoryTarget.java | 315 +++++++++++++++++++--
.../memory/memtarget/spark/TreeMemoryConsumer.java | 4 +-
.../memtarget/spark/TreeMemoryConsumers.java | 7 +-
.../org/apache/spark/util/SparkThreadPoolUtil.java | 43 +++
.../scala/org/apache/gluten/GlutenPlugin.scala | 15 +-
.../apache/gluten/config/GlutenCoreConfig.scala | 2 +
.../spark/memory/GlobalOffHeapMemoryTarget.scala | 22 +-
.../org/apache/gluten/config/GlutenConfig.scala | 1 -
9 files changed, 405 insertions(+), 63 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingSuite.scala
index 30442c6645..645deec16e 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/DynamicOffHeapSizingSuite.scala
@@ -16,45 +16,58 @@
*/
package org.apache.gluten.execution
-import org.apache.gluten.benchmarks.RandomParquetDataGenerator
-import org.apache.gluten.tags.SkipTest
+import org.apache.gluten.memory.memtarget.DynamicOffHeapSizingMemoryTarget
import org.apache.spark.SparkConf
-@SkipTest
class DynamicOffHeapSizingSuite extends VeloxWholeStageTransformerSuite {
override protected val resourcePath: String = "/tpch-data-parquet"
override protected val fileFormat: String = "parquet"
- private val dataGenerator =
RandomParquetDataGenerator(System.currentTimeMillis())
- private val outputPath = getClass.getResource("/").getPath +
"dynamicoffheapsizing_output.parquet"
- private val AGG_SQL =
- """select f_1, count(DISTINCT f_1)
- |from tbl group
- |group by 1""".stripMargin
-
override def beforeAll(): Unit = {
super.beforeAll()
+ createTPCHNotNullTables()
}
+
override protected def sparkConf: SparkConf = {
super.sparkConf
.set("spark.shuffle.manager",
"org.apache.spark.shuffle.sort.ColumnarShuffleManager")
- .set("spark.executor.memory", "6GB")
- .set("spark.gluten.memory.dynamic.offHeap.sizing.memory.fraction", "0.8")
+ .set("spark.executor.memory", "2GB")
+ .set("spark.memory.offHeap.enabled", "false")
+ .set("spark.memory.offHeap.size", "0")
+ .set("spark.gluten.memory.dynamic.offHeap.sizing.memory.fraction",
"0.95")
.set("spark.gluten.memory.dynamic.offHeap.sizing.enabled", "true")
}
- def getRootCause(e: Throwable): Throwable = {
- if (e.getCause == null) {
- return e
- }
- getRootCause(e.getCause)
- }
-
test("Dynamic off-heap sizing") {
- System.gc()
- dataGenerator.generateRandomData(spark, Some(outputPath))
-
spark.read.format("parquet").load(outputPath).createOrReplaceTempView("tbl")
- spark.sql(AGG_SQL)
+ if (DynamicOffHeapSizingMemoryTarget.isJava9OrLater()) {
+ val query =
+ """
+ | select l_quantity, c_acctbal, o_orderdate, p_type, n_name,
s_suppkey
+ | from customer, orders, lineitem, part, supplier, nation
+ | where c_custkey = o_custkey and o_orderkey = l_orderkey and
l_partkey = p_partkey
+ | and l_suppkey = s_suppkey and s_nationkey = n_nationkey
+ | order by c_acctbal desc, o_orderdate, s_suppkey, n_name, p_type,
l_quantity
+ | limit 1
+ """.stripMargin
+ var totalMemory = Runtime.getRuntime().totalMemory()
+ var freeMemory = Runtime.getRuntime().freeMemory()
+ val maxMemory = Runtime.getRuntime().maxMemory()
+ // Ensure that the JVM memory is not too small to trigger dynamic
off-heap sizing.
+ while (!DynamicOffHeapSizingMemoryTarget.canShrinkJVMMemory(totalMemory,
freeMemory)) {
+ withSQLConf(("spark.gluten.enabled", "false")) {
+ spark.sql(query).collect()
+ }
+ totalMemory = Runtime.getRuntime().totalMemory()
+ freeMemory = Runtime.getRuntime().freeMemory()
+ }
+ val newTotalMemory =
+ DynamicOffHeapSizingMemoryTarget.shrinkOnHeapMemory(totalMemory,
freeMemory, false)
+ assert(DynamicOffHeapSizingMemoryTarget.getTotalExplicitGCCount() > 0)
+ // Verify that the total memory is reduced after shrink.
+ assert(newTotalMemory < totalMemory)
+ // Verify that the query can run with dynamic off-heap sizing enabled.
+ runAndCompare(query)
+ }
}
}
diff --git
a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java
index c9e9aed7b9..bc17142dec 100644
---
a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java
+++
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java
@@ -16,14 +16,21 @@
*/
package org.apache.gluten.memory.memtarget;
-import org.apache.gluten.config.GlutenCoreConfig;
import org.apache.gluten.memory.SimpleMemoryUsageRecorder;
import org.apache.gluten.proto.MemoryUsageStats;
import org.apache.spark.annotation.Experimental;
+import org.apache.spark.util.SparkThreadPoolUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.management.MBeanServer;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.RuntimeMXBean;
+import java.lang.reflect.Method;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -38,26 +45,87 @@ public class DynamicOffHeapSizingMemoryTarget implements
MemoryTarget, KnownName
// memory, so we can use it as the max memory we will use.
private static final long TOTAL_MEMORY_SHARED;
+ private static final double ASYNC_GC_MAX_TOTAL_MEMORY_USAGE_RATIO = 0.85;
+ private static final double ASYNC_GC_MAX_ON_HEAP_MEMORY_RATIO = 0.65;
+ private static final double GC_MAX_HEAP_FREE_RATIO = 0.05;
+ private static final int MAX_GC_RETRY_TIMES = 3;
+
+ private static final AtomicBoolean ASYNC_GC_SUSPEND = new
AtomicBoolean(false);
+ private static final Object JVM_SHRINK_SYNC_OBJECT = new Object();
+
+ private static final int ORIGINAL_MAX_HEAP_FREE_RATIO;
+ private static final int ORIGINAL_MIN_HEAP_FREE_RATIO;
+
static {
- final long maxOnHeapSize = Runtime.getRuntime().maxMemory();
- final double fractionForSizing =
GlutenCoreConfig.get().dynamicOffHeapSizingMemoryFraction();
- // Since when dynamic off-heap sizing is enabled, we commingle on-heap
- // and off-heap memory, we set the off-heap size to the usable on-heap
size. We will
- // size it with a memory fraction, which can be aggressively set, but the
default
- // is using the same way that Spark sizes on-heap memory:
- //
- // spark.gluten.memory.dynamic.offHeap.sizing.memory.fraction *
- // (spark.executor.memory - 300MB).
- //
- // We will be careful to use the same configuration settings as Spark to
ensure
- // that we are sizing the off-heap memory in the same way as Spark sizes
on-heap memory.
- // The 300MB value, unfortunately, is hard-coded in Spark code.
- TOTAL_MEMORY_SHARED = (long) ((maxOnHeapSize - (300 * 1024 * 1024)) *
fractionForSizing);
- LOG.info("DynamicOffHeapSizingMemoryTarget MAX_MEMORY_IN_BYTES: {}",
TOTAL_MEMORY_SHARED);
+ RuntimeMXBean runtimeMxBean = ManagementFactory.getRuntimeMXBean();
+ List<String> jvmArgs = runtimeMxBean.getInputArguments();
+ int originalMaxHeapFreeRatio = 70;
+ int originalMinHeapFreeRatio = 40;
+ for (String arg : jvmArgs) {
+ if (arg.startsWith("-XX:MaxHeapFreeRatio=")) {
+ String valuePart = arg.substring(arg.indexOf('=') + 1);
+ try {
+ originalMaxHeapFreeRatio = Integer.parseInt(valuePart);
+ } catch (NumberFormatException e) {
+ LOG.warn(
+ "Failed to parse MaxHeapFreeRatio from JVM argument: {}. Using
default value: {}.",
+ arg,
+ originalMaxHeapFreeRatio);
+ }
+ } else if (arg.startsWith("-XX:MinHeapFreeRatio=")) {
+ String valuePart = arg.substring(arg.indexOf('=') + 1);
+ try {
+ originalMinHeapFreeRatio = Integer.parseInt(valuePart);
+ } catch (NumberFormatException e) {
+ LOG.warn(
+ "Failed to parse MinHeapFreeRatio from JVM argument: {}. Using
default value: {}.",
+ arg,
+ originalMinHeapFreeRatio);
+ }
+ } else if (arg == "-XX:+ExplicitGCInvokesConcurrent") {
+ // If this is set -XX:+ExplicitGCInvokesConcurrent, System.gc() does
not trigger Full GC,
+ // so explicit JVM shrinking is not effective.
+ LOG.error(
+ "Explicit JVM shrinking is not effective because
-XX:+ExplicitGCInvokesConcurrent"
+ + " is set. Please check the JVM arguments: {}. ",
+ arg);
+
+ } else if (arg == "-XX:+DisableExplicitGC") {
+ // If -XX:+DisableExplicitGC is set, calls to System.gc() are ignored,
+ // so explicit JVM shrinking will not work as intended.
+ LOG.error(
+ "Explicit JVM shrinking is disabled because -XX:+DisableExplicitGC
is set. "
+ + "System.gc() calls will be ignored and JVM shrinking will
not work. "
+ + "Please check the JVM arguments: {}. ",
+ arg);
+ }
+ }
+ ORIGINAL_MIN_HEAP_FREE_RATIO = originalMinHeapFreeRatio;
+ ORIGINAL_MAX_HEAP_FREE_RATIO = originalMaxHeapFreeRatio;
+
+ if (!isJava9OrLater()) {
+ // For JDK 8, we cannot change MaxHeapFreeRatio programmatically at
runtime.
+ LOG.error("Dynamic off-heap sizing is not supported before JDK 9.");
+ }
+
+ TOTAL_MEMORY_SHARED = Runtime.getRuntime().maxMemory();
+
+ LOG.info(
+ "Initialized DynamicOffHeapSizingMemoryTarget with MAX_MEMORY_IN_BYTES
= {}, "
+ + "ASYNC_GC_MAX_TOTAL_MEMORY_USAGE_RATIO = {}, "
+ + "ASYNC_GC_MAX_ON_HEAP_MEMORY_RATIO = {}, "
+ + "GC_MAX_HEAP_FREE_RATIO = {}.",
+ TOTAL_MEMORY_SHARED,
+ ASYNC_GC_MAX_TOTAL_MEMORY_USAGE_RATIO,
+ ASYNC_GC_MAX_ON_HEAP_MEMORY_RATIO,
+ GC_MAX_HEAP_FREE_RATIO);
}
private static final AtomicLong USED_OFF_HEAP_BYTES = new AtomicLong();
+ // Test only.
+ private static final AtomicLong TOTAL_EXPLICIT_GC_COUNT = new AtomicLong(0L);
+
private final String name =
MemoryTargetUtil.toUniqueName("DynamicOffHeapSizing");
private final SimpleMemoryUsageRecorder recorder = new
SimpleMemoryUsageRecorder();
@@ -77,21 +145,53 @@ public class DynamicOffHeapSizingMemoryTarget implements
MemoryTarget, KnownName
// Adds the total JVM memory which is the actual memory the JVM occupied
from the operating
// system into the counter.
- if (size + usedOffHeapMemory + totalHeapMemory > TOTAL_MEMORY_SHARED) {
- LOG.warn(
- String.format(
- "Failing allocation as unified memory is OOM. "
- + "Used Off-heap: %d, Used On-Heap: %d, "
- + "Free On-heap: %d, Total On-heap: %d, "
- + "Max On-heap: %d, Allocation: %d.",
- usedOffHeapMemory,
- totalHeapMemory - freeHeapMemory,
- freeHeapMemory,
- totalHeapMemory,
- TOTAL_MEMORY_SHARED,
- size));
+ if (exceedsMaxMemoryUsage(totalHeapMemory, usedOffHeapMemory, size, 1.0)) {
+ // Perform GC synchronously to shrink memory; native tasks need to wait
for this to obtain
+ // more memory.
+ synchronized (JVM_SHRINK_SYNC_OBJECT) {
+ totalHeapMemory = Runtime.getRuntime().totalMemory();
+ freeHeapMemory = Runtime.getRuntime().freeMemory();
+ if (exceedsMaxMemoryUsage(totalHeapMemory, usedOffHeapMemory, size,
1.0)) {
+ shrinkOnHeapMemory(totalHeapMemory, freeHeapMemory, false);
+ totalHeapMemory = Runtime.getRuntime().totalMemory();
+ freeHeapMemory = Runtime.getRuntime().freeMemory();
+ }
+ }
+ // Check if we can allocate the requested size again after JVM
shrinking(GC).
+ if (exceedsMaxMemoryUsage(totalHeapMemory, usedOffHeapMemory, size,
1.0)) {
+ LOG.warn(
+ String.format(
+ "Failing allocation as unified memory is OOM. "
+ + "Used Off-heap: %d, Used On-Heap: %d, "
+ + "Free On-heap: %d, Total On-heap: %d, "
+ + "Max On-heap: %d, Allocation: %d.",
+ usedOffHeapMemory,
+ totalHeapMemory - freeHeapMemory,
+ freeHeapMemory,
+ totalHeapMemory,
+ TOTAL_MEMORY_SHARED,
+ size));
- return 0;
+ return 0;
+ }
+ } else if (shouldTriggerAsyncOnHeapMemoryShrink(
+ totalHeapMemory, freeHeapMemory, usedOffHeapMemory, size)) {
+ // Proactively trigger memory shrinking in the thread pool to prevent GC
from blocking
+ // native task execution.
+ SparkThreadPoolUtil.triggerGCInThreadPool(
+ new Runnable() {
+ @Override
+ public void run() {
+ synchronized (JVM_SHRINK_SYNC_OBJECT) {
+ long totalJvmMem = Runtime.getRuntime().totalMemory();
+ long freeJvmMem = Runtime.getRuntime().freeMemory();
+ if (shouldTriggerAsyncOnHeapMemoryShrink(
+ totalJvmMem, freeJvmMem, usedOffHeapMemory, size)) {
+ shrinkOnHeapMemory(totalJvmMem, freeJvmMem, true);
+ }
+ }
+ }
+ });
}
USED_OFF_HEAP_BYTES.addAndGet(size);
@@ -125,4 +225,159 @@ public class DynamicOffHeapSizingMemoryTarget implements
MemoryTarget, KnownName
public MemoryUsageStats stats() {
return recorder.toStats();
}
+
+ public static boolean isJava9OrLater() {
+ String spec = System.getProperty("java.specification.version", "1.8");
+ // "1.8" → 8, "9" → 9, "11" → 11, etc.
+ if (spec.startsWith("1.")) {
+ spec = spec.substring(2);
+ }
+ try {
+ return Integer.parseInt(spec) >= 9;
+ } catch (NumberFormatException e) {
+ return false;
+ }
+ }
+
+ public static boolean canShrinkJVMMemory(long totalMemory, long freeMemory) {
+ // Check if the JVM memory can be shrunk by a full GC.
+ return freeMemory > totalMemory * GC_MAX_HEAP_FREE_RATIO;
+ }
+
+ public static long getTotalExplicitGCCount() {
+ return TOTAL_EXPLICIT_GC_COUNT.get();
+ }
+
+ private static boolean exceedsMaxMemoryUsage(
+ long totalOnHeapMemory, long totalOffHeapMemory, long requestedSize,
double ratio) {
+ return requestedSize + totalOffHeapMemory + totalOnHeapMemory >=
TOTAL_MEMORY_SHARED * ratio;
+ }
+
+ private static boolean shouldTriggerAsyncOnHeapMemoryShrink(
+ long totalOnHeapMemory, long freeOnHeapMemory, long totalOffHeapMemory,
long requestedSize) {
+ // If most of the memory has already been used, there's a high chance that
memory will be fully
+ // consumed. We proactively detect this situation to trigger JVM memory
shrinking using the
+ // following conditions.
+
+ boolean exceedsMaxMemoryUsageRatio =
+ exceedsMaxMemoryUsage(
+ totalOnHeapMemory,
+ totalOffHeapMemory,
+ requestedSize,
+ ASYNC_GC_MAX_TOTAL_MEMORY_USAGE_RATIO);
+ return exceedsMaxMemoryUsageRatio
+ && canShrinkJVMMemory(totalOnHeapMemory, freeOnHeapMemory)
+ // Limit GC frequency to prevent performance impact from excessive
garbage collection.
+ && totalOnHeapMemory > TOTAL_MEMORY_SHARED *
ASYNC_GC_MAX_ON_HEAP_MEMORY_RATIO
+ && (!ASYNC_GC_SUSPEND.get()
+ || freeOnHeapMemory > totalOnHeapMemory *
(ORIGINAL_MIN_HEAP_FREE_RATIO / 100.0));
+ }
+
+ private static long shrinkOnHeapMemoryInternal(
+ long totalMemory, long freeMemory, boolean isAsyncGc) {
+ long totalOffHeapMemory = USED_OFF_HEAP_BYTES.get();
+ LOG.warn(
+ String.format(
+ "Starting %sfull gc to shrink JVM memory: "
+ + "Total On-heap: %d, Free On-heap: %d, "
+ + "Total Off-heap: %d, Used On-Heap: %d, Executor memory: %d.",
+ isAsyncGc ? "async " : "",
+ totalMemory,
+ freeMemory,
+ totalOffHeapMemory,
+ (totalMemory - freeMemory),
+ TOTAL_MEMORY_SHARED));
+ // Explicitly calling System.gc() to trigger a full garbage collection.
+ // This is necessary in this context to attempt to shrink JVM memory usage
+ // when off-heap memory allocation is constrained. Use of System.gc() is
+ // generally discouraged due to its unpredictable performance impact, but
+ // here it is used as a last resort to prevent memory allocation failures.
+ System.gc();
+ long newTotalMemory = Runtime.getRuntime().totalMemory();
+ long newFreeMemory = Runtime.getRuntime().freeMemory();
+ int gcRetryTimes = 0;
+ while (!isAsyncGc
+ && gcRetryTimes < MAX_GC_RETRY_TIMES
+ && newTotalMemory >= totalMemory
+ && canShrinkJVMMemory(newTotalMemory, newFreeMemory)) {
+ // System.gc() is just a suggestion; the JVM may ignore it or perform
only a partial GC.
+ // Here, the total memory is not reduced but the free memory ratio is
bigger than the
+ // GC_MAX_HEAP_FREE_RATIO. So we need to call System.gc() again to try
to reduce the total
+ // memory.
+ // This is a workaround for the JVM's behavior of not reducing the total
memory after GC.
+ System.gc();
+ newTotalMemory = Runtime.getRuntime().totalMemory();
+ newFreeMemory = Runtime.getRuntime().freeMemory();
+ gcRetryTimes++;
+ }
+ // If the memory usage is still high after GC, we need to suspend the
async GC for a while.
+ if (isAsyncGc) {
+ ASYNC_GC_SUSPEND.set(
+ totalMemory - newTotalMemory < totalMemory *
(ORIGINAL_MIN_HEAP_FREE_RATIO / 100.0));
+ }
+
+ TOTAL_EXPLICIT_GC_COUNT.getAndAdd(1);
+ LOG.warn(
+ String.format(
+ "Finished %sfull gc to shrink JVM memory: "
+ + "Total On-heap: %d, Free On-heap: %d, "
+ + "Total Off-heap: %d, Used On-Heap: %d, Executor memory: %d, "
+ + "[GC Retry times: %d].",
+ isAsyncGc ? "async " : "",
+ newTotalMemory,
+ newFreeMemory,
+ totalOffHeapMemory,
+ (newTotalMemory - newFreeMemory),
+ TOTAL_MEMORY_SHARED,
+ gcRetryTimes));
+ return newTotalMemory;
+ }
+
+ public static long shrinkOnHeapMemory(long totalMemory, long freeMemory,
boolean isAsyncGc) {
+ boolean updateMaxHeapFreeRatio = false;
+ Object hotSpotBean = null;
+ String maxHeapFreeRatioName = "MaxHeapFreeRatio";
+ String minHeapFreeRatioName = "MinHeapFreeRatio";
+ int newValue = (int) (GC_MAX_HEAP_FREE_RATIO * 100);
+
+ try {
+ MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+ Class<?> beanClass =
Class.forName("com.sun.management.HotSpotDiagnosticMXBean");
+ hotSpotBean =
+ ManagementFactory.newPlatformMXBeanProxy(
+ mbs, "com.sun.management:type=HotSpotDiagnostic", beanClass);
+
+ Method setOption = beanClass.getMethod("setVMOption", String.class,
String.class);
+ if (newValue < ORIGINAL_MIN_HEAP_FREE_RATIO) {
+ // Adjust the MinHeapFreeRatio to avoid the violation of the
MaxHeapFreeRatio.
+ setOption.invoke(hotSpotBean, minHeapFreeRatioName,
Integer.toString(newValue));
+ }
+ if (newValue < ORIGINAL_MAX_HEAP_FREE_RATIO) {
+ setOption.invoke(hotSpotBean, maxHeapFreeRatioName,
Integer.toString(newValue));
+ updateMaxHeapFreeRatio = true;
+ LOG.info(
+ String.format(
+ "Updated VM flags: MaxHeapFreeRatio from %d to %d.",
+ ORIGINAL_MAX_HEAP_FREE_RATIO, newValue));
+ }
+ return shrinkOnHeapMemoryInternal(totalMemory, freeMemory, isAsyncGc);
+ } catch (Exception e) {
+ LOG.warn(
+ "Failed to update JVM heap free ratio via HotSpotDiagnosticMXBean:
{}", e.toString());
+ return totalMemory;
+ } finally {
+ // Reset the MaxHeapFreeRatio to the original values.
+ if (hotSpotBean != null && updateMaxHeapFreeRatio) {
+ try {
+ Class<?> beanClass =
Class.forName("com.sun.management.HotSpotDiagnosticMXBean");
+ Method setOption = beanClass.getMethod("setVMOption", String.class,
String.class);
+ setOption.invoke(
+ hotSpotBean, maxHeapFreeRatioName,
Integer.toString(ORIGINAL_MAX_HEAP_FREE_RATIO));
+ LOG.info("Reverted VM flags back.");
+ } catch (Exception ignore) {
+ // best‐effort revert
+ }
+ }
+ }
+ }
}
diff --git
a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
index 38ac7d9733..f454b55324 100644
---
a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
+++
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumer.java
@@ -53,8 +53,8 @@ public class TreeMemoryConsumer extends MemoryConsumer
implements TreeMemoryTarg
private final Map<String, TreeMemoryTarget> children = new HashMap<>();
private final String name = MemoryTargetUtil.toUniqueName("Gluten.Tree");
- TreeMemoryConsumer(TaskMemoryManager taskMemoryManager) {
- super(taskMemoryManager, taskMemoryManager.pageSizeBytes(),
MemoryMode.OFF_HEAP);
+ TreeMemoryConsumer(TaskMemoryManager taskMemoryManager, MemoryMode mode) {
+ super(taskMemoryManager, taskMemoryManager.pageSizeBytes(), mode);
}
@Override
diff --git
a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java
index a1f2106e3c..0f7c877a97 100644
---
a/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java
+++
b/gluten-core/src/main/java/org/apache/gluten/memory/memtarget/spark/TreeMemoryConsumers.java
@@ -21,6 +21,7 @@ import org.apache.gluten.memory.memtarget.Spillers;
import org.apache.gluten.memory.memtarget.TreeMemoryTarget;
import org.apache.commons.collections.map.ReferenceMap;
+import org.apache.spark.memory.MemoryMode;
import org.apache.spark.memory.TaskMemoryManager;
import org.apache.spark.util.Utils;
@@ -45,7 +46,11 @@ public final class TreeMemoryConsumers {
private final Map<Long, TreeMemoryTarget> roots = new
ConcurrentHashMap<>();
private Factory(TaskMemoryManager tmm) {
- this.sparkConsumer = new TreeMemoryConsumer(tmm);
+ MemoryMode mode =
+ GlutenCoreConfig.get().dynamicOffHeapSizingEnabled()
+ ? MemoryMode.ON_HEAP
+ : MemoryMode.OFF_HEAP;
+ this.sparkConsumer = new TreeMemoryConsumer(tmm, mode);
}
private TreeMemoryTarget ofCapacity(long capacity) {
diff --git
a/gluten-core/src/main/java/org/apache/spark/util/SparkThreadPoolUtil.java
b/gluten-core/src/main/java/org/apache/spark/util/SparkThreadPoolUtil.java
new file mode 100644
index 0000000000..25cd54568f
--- /dev/null
+++ b/gluten-core/src/main/java/org/apache/spark/util/SparkThreadPoolUtil.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.util;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class SparkThreadPoolUtil {
+ private static final ExecutorService GC_THREAD_POOL =
+ ThreadUtils.newDaemonSingleThreadExecutor("gc-thread-pool");
+
+ private static final AtomicBoolean PERFORMING_GC = new AtomicBoolean(false);
+
+ public static void triggerGCInThreadPool(Runnable callback) {
+ if (PERFORMING_GC.compareAndSet(false, true)) {
+ GC_THREAD_POOL.submit(
+ new Runnable() {
+ @Override
+ public void run() {
+ try {
+ callback.run();
+ } finally {
+ PERFORMING_GC.set(false);
+ }
+ }
+ });
+ }
+ }
+}
diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
index 19bddf0fd7..4af5880afb 100644
--- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
+++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
@@ -109,7 +109,20 @@ private object GlutenDriverPlugin extends Logging {
checkOffHeapSettings(conf)
// Get the off-heap size set by user.
- val offHeapSize =
conf.getSizeAsBytes(GlutenCoreConfig.SPARK_OFFHEAP_SIZE_KEY)
+ val offHeapSize =
+ if (conf.getBoolean(GlutenCoreConfig.DYNAMIC_OFFHEAP_SIZING_ENABLED.key,
false)) {
+ val onHeapSize: Long =
+ if (conf.contains(GlutenCoreConfig.SPARK_ONHEAP_SIZE_KEY)) {
+ conf.getSizeAsBytes(GlutenCoreConfig.SPARK_ONHEAP_SIZE_KEY)
+ } else {
+ // 1GB default
+ 1024 * 1024 * 1024
+ }
+ ((onHeapSize - (300 * 1024 * 1024)) *
+
conf.getDouble(GlutenCoreConfig.DYNAMIC_OFFHEAP_SIZING_MEMORY_FRACTION.key,
0.6d)).toLong
+ } else {
+ conf.getSizeAsBytes(GlutenCoreConfig.SPARK_OFFHEAP_SIZE_KEY)
+ }
// Set off-heap size in bytes.
conf.set(GlutenCoreConfig.COLUMNAR_OFFHEAP_SIZE_IN_BYTES, offHeapSize)
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/config/GlutenCoreConfig.scala
b/gluten-core/src/main/scala/org/apache/gluten/config/GlutenCoreConfig.scala
index f3912007cd..b3112f2990 100644
--- a/gluten-core/src/main/scala/org/apache/gluten/config/GlutenCoreConfig.scala
+++ b/gluten-core/src/main/scala/org/apache/gluten/config/GlutenCoreConfig.scala
@@ -72,6 +72,8 @@ object GlutenCoreConfig {
val SPARK_OFFHEAP_SIZE_KEY = "spark.memory.offHeap.size"
val SPARK_OFFHEAP_ENABLED_KEY = "spark.memory.offHeap.enabled"
+ val SPARK_ONHEAP_SIZE_KEY = "spark.executor.memory"
+
val GLUTEN_ENABLED =
buildConf("spark.gluten.enabled")
.internal()
diff --git
a/gluten-core/src/main/scala/org/apache/spark/memory/GlobalOffHeapMemoryTarget.scala
b/gluten-core/src/main/scala/org/apache/spark/memory/GlobalOffHeapMemoryTarget.scala
index bb251f101c..c43f0c7e57 100644
---
a/gluten-core/src/main/scala/org/apache/spark/memory/GlobalOffHeapMemoryTarget.scala
+++
b/gluten-core/src/main/scala/org/apache/spark/memory/GlobalOffHeapMemoryTarget.scala
@@ -16,6 +16,7 @@
*/
package org.apache.spark.memory
+import org.apache.gluten.config.GlutenCoreConfig
import org.apache.gluten.exception.GlutenException
import org.apache.gluten.memory.{MemoryUsageRecorder,
SimpleMemoryUsageRecorder}
import org.apache.gluten.memory.memtarget.{KnownNameAndStats, MemoryTarget,
MemoryTargetUtil, MemoryTargetVisitor}
@@ -34,6 +35,9 @@ class GlobalOffHeapMemoryTarget private[memory]
with Logging {
private val targetName = MemoryTargetUtil.toUniqueName("GlobalOffHeap")
private val recorder: MemoryUsageRecorder = new SimpleMemoryUsageRecorder()
+ private val mode: MemoryMode =
+ if (GlutenCoreConfig.get.dynamicOffHeapSizingEnabled) MemoryMode.ON_HEAP
+ else MemoryMode.OFF_HEAP
private val FIELD_MEMORY_MANAGER: Field = {
val f =
@@ -54,10 +58,7 @@ class GlobalOffHeapMemoryTarget private[memory]
.map {
mm =>
val succeeded =
- mm.acquireStorageMemory(
- BlockId(s"test_${UUID.randomUUID()}"),
- size,
- MemoryMode.OFF_HEAP)
+ mm.acquireStorageMemory(BlockId(s"test_${UUID.randomUUID()}"),
size, mode)
if (succeeded) {
recorder.inc(size)
@@ -65,6 +66,17 @@ class GlobalOffHeapMemoryTarget private[memory]
} else {
// OOM.
// Throw OOM.
+ if (mode == MemoryMode.ON_HEAP) {
+ val storageUsed = mm.onHeapStorageMemoryUsed
+ val executionUsed = mm.onHeapExecutionMemoryUsed
+ val onHeapMemoryTotal = storageUsed + executionUsed
+ logError(
+ s"Spark on-heap memory is exhausted. " +
+ s"Requested: $size, " +
+ s"Storage: $storageUsed / $onHeapMemoryTotal, " +
+ s"Execution: $executionUsed / $onHeapMemoryTotal")
+ return 0;
+ }
val storageUsed = mm.offHeapStorageMemoryUsed
val executionUsed = mm.offHeapExecutionMemoryUsed
val offHeapMemoryTotal = storageUsed + executionUsed
@@ -82,7 +94,7 @@ class GlobalOffHeapMemoryTarget private[memory]
memoryManagerOption()
.map {
mm =>
- mm.releaseStorageMemory(size, MemoryMode.OFF_HEAP)
+ mm.releaseStorageMemory(size, mode)
recorder.inc(-size)
size
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
index 2e4888e032..793a180392 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
@@ -418,7 +418,6 @@ object GlutenConfig {
private val GLUTEN_CONFIG_PREFIX = "spark.gluten.sql.columnar.backend."
// Private Spark configs.
- val SPARK_ONHEAP_SIZE_KEY = "spark.executor.memory"
val SPARK_OVERHEAD_SIZE_KEY = "spark.executor.memoryOverhead"
val SPARK_OVERHEAD_FACTOR_KEY = "spark.executor.memoryOverheadFactor"
val SPARK_REDACTION_REGEX = "spark.redaction.regex"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]