zhli1142015 commented on code in PR #9585:
URL: https://github.com/apache/incubator-gluten/pull/9585#discussion_r2111635025
##########
gluten-core/src/main/java/org/apache/gluten/memory/memtarget/DynamicOffHeapSizingMemoryTarget.java:
##########
@@ -38,26 +45,87 @@ public class DynamicOffHeapSizingMemoryTarget implements
MemoryTarget, KnownName
// memory, so we can use it as the max memory we will use.
private static final long TOTAL_MEMORY_SHARED;
+ private static final double ASYNC_GC_MAX_TOTAL_MEMORY_USAGE_RATIO = 0.85;
+ private static final double ASYNC_GC_MAX_ON_HEAP_MEMORY_RATIO = 0.65;
+ private static final double GC_MAX_HEAP_FREE_RATIO = 0.05;
+ private static final int MAX_GC_RETRY_TIMES = 3;
+
+ private static final AtomicBoolean ASYNC_GC_SUSPEND = new
AtomicBoolean(false);
+ private static final Object JVM_SHRINK_SYNC_OBJECT = new Object();
+
+ private static final int ORIGINAL_MAX_HEAP_FREE_RATIO;
+ private static final int ORIGINAL_MIN_HEAP_FREE_RATIO;
+
static {
- final long maxOnHeapSize = Runtime.getRuntime().maxMemory();
- final double fractionForSizing =
GlutenConfig.get().dynamicOffHeapSizingMemoryFraction();
- // Since when dynamic off-heap sizing is enabled, we commingle on-heap
- // and off-heap memory, we set the off-heap size to the usable on-heap
size. We will
- // size it with a memory fraction, which can be aggressively set, but the
default
- // is using the same way that Spark sizes on-heap memory:
- //
- // spark.gluten.memory.dynamic.offHeap.sizing.memory.fraction *
- // (spark.executor.memory - 300MB).
- //
- // We will be careful to use the same configuration settings as Spark to
ensure
- // that we are sizing the off-heap memory in the same way as Spark sizes
on-heap memory.
- // The 300MB value, unfortunately, is hard-coded in Spark code.
- TOTAL_MEMORY_SHARED = (long) ((maxOnHeapSize - (300 * 1024 * 1024)) *
fractionForSizing);
- LOG.info("DynamicOffHeapSizingMemoryTarget MAX_MEMORY_IN_BYTES: {}",
TOTAL_MEMORY_SHARED);
+ RuntimeMXBean runtimeMxBean = ManagementFactory.getRuntimeMXBean();
+ List<String> jvmArgs = runtimeMxBean.getInputArguments();
+ int originalMaxHeapFreeRatio = 70;
+ int originalMinHeapFreeRatio = 40;
+ for (String arg : jvmArgs) {
+ if (arg.startsWith("-XX:MaxHeapFreeRatio=")) {
+ String valuePart = arg.substring(arg.indexOf('=') + 1);
+ try {
+ originalMaxHeapFreeRatio = Integer.parseInt(valuePart);
+ } catch (NumberFormatException e) {
+ LOG.warn(
+ "Failed to parse MaxHeapFreeRatio from JVM argument: {}. Using
default value: {}.",
+ arg,
+ originalMaxHeapFreeRatio);
+ }
+ } else if (arg.startsWith("-XX:MinHeapFreeRatio=")) {
+ String valuePart = arg.substring(arg.indexOf('=') + 1);
+ try {
+ originalMinHeapFreeRatio = Integer.parseInt(valuePart);
+ } catch (NumberFormatException e) {
+ LOG.warn(
+ "Failed to parse MinHeapFreeRatio from JVM argument: {}. Using
default value: {}.",
+ arg,
+ originalMinHeapFreeRatio);
+ }
+ } else if (arg == "-XX:+ExplicitGCInvokesConcurrent") {
+ // If this is set -XX:+ExplicitGCInvokesConcurrent, System.gc() does
not trigger Full GC,
+ // so explicit JVM shrinking is not effective.
+ LOG.error(
+ "Explicit JVM shrinking is not effective because
-XX:+ExplicitGCInvokesConcurrent"
+ + " is set. Please check the JVM arguments: {}. ",
+ arg);
+
+ } else if (arg == "-XX:+DisableExplicitGC") {
+ // If -XX:+DisableExplicitGC is set, calls to System.gc() are ignored,
+ // so explicit JVM shrinking will not work as intended.
+ LOG.error(
+ "Explicit JVM shrinking is disabled because -XX:+DisableExplicitGC
is set. "
+ + "System.gc() calls will be ignored and JVM shrinking will
not work. "
+ + "Please check the JVM arguments: {}. ",
+ arg);
+ }
+ }
+ ORIGINAL_MIN_HEAP_FREE_RATIO = originalMinHeapFreeRatio;
+ ORIGINAL_MAX_HEAP_FREE_RATIO = originalMaxHeapFreeRatio;
+
+ if (!isJava9OrLater()) {
Review Comment:
Sure, updated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
