This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 8c044fcfb7d Introducing new features to IoTDB JVM GC options (#12088)
8c044fcfb7d is described below
commit 8c044fcfb7d0f8ee484254c1a11f23f9e2a01851
Author: Peng Junzhi <[email protected]>
AuthorDate: Fri Mar 22 03:52:38 2024 -0500
Introducing new features to IoTDB JVM GC options (#12088)
---
.../src/assembly/resources/conf/datanode-env.bat | 37 ++++++++++++++++++--
.../src/assembly/resources/conf/datanode-env.sh | 39 ++++++++++++++++++++--
.../assembly/resources/sbin/start-standalone.bat | 3 ++
.../assembly/resources/sbin/start-standalone.sh | 2 ++
4 files changed, 77 insertions(+), 4 deletions(-)
diff --git a/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.bat
b/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.bat
index d1ec4b1c117..086a7f37ea0 100644
--- a/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.bat
+++ b/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.bat
@@ -121,9 +121,36 @@ set IOTDB_HEAP_OPTS=-Xmx%ON_HEAP_MEMORY%
-Xms%ON_HEAP_MEMORY%
set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -XX:MaxDirectMemorySize=%OFF_HEAP_MEMORY%
set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS%
-Djdk.nio.maxCachedBufferSize=%MAX_CACHED_BUFFER_SIZE%
set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -XX:+CrashOnOutOfMemoryError
+set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -XX:+UseAdaptiveSizePolicy
+set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -Xss512k
+@REM options below try to optimize safepoint stw time.
+set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -XX:+UnlockDiagnosticVMOptions
+set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -XX:GuaranteedSafepointInterval=0
+set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -XX:-UseBiasedLocking
+@REM these two options print safepoints with pauses longer than 1000ms to the
standard output. You can see these logs via redirection when starting in the
background like "start-datanode.sh > log_datanode_safepoint.txt"
+set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -XX:SafepointTimeoutDelay=1000
+set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -XX:+SafepointTimeout
+
+@REM option below tries to optimize safepoint stw time for large counted loop.
+@REM NOTE: it may have an impact on JIT's black-box optimization.
+@REM set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -XX:+UseCountedLoopSafepoints
+
+@REM When the GC time is too long, if there are remaining CPU resources, you
can try to turn on and increase options below.
+@REM for /F "tokens=2 delims==" %%I in ('wmic cpu get NumberOfCores /value')
do (
+@REM set "CPU_PROCESSOR_NUM=%%I"
+@REM )
+@REM set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS%
-XX:ParallelGCThreads=%CPU_PROCESSOR_NUM%
+
+@REM if there are much of stw time of reference process in GC log, you can
turn on option below.
+@REM NOTE: it may have an impact on application's throughput.
+@REM set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -XX:+ParallelRefProcEnabled
+
+@REM this option can reduce the overhead caused by memory allocation, page
fault interrupts, etc. during JVM operation.
+@REM NOTE: it may reduce memory utilization and trigger OOM killer when memory
is tight.
+@REM set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -XX:+AlwaysPreTouch
@REM if you want to dump the heap memory while OOM happening, you can use the
following command, remember to replace /tmp/heapdump.hprof with your own file
path and the folder where this file is located needs to be created in advance
-@REM IOTDB_JMX_OPTS=%IOTDB_HEAP_OPTS% -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=\tmp\datanode_heapdump.hprof
+@REM set IOTDB_JMX_OPTS=%IOTDB_HEAP_OPTS% -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=\tmp\datanode_heapdump.hprof
@REM You can put your env variable here
@REM set JAVA_HOME=%JAVA_HOME%
@@ -132,10 +159,16 @@ set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS%
-XX:+CrashOnOutOfMemoryError
IF "%1" equ "printgc" (
IF "%JAVA_VERSION%" == "8" (
md %IOTDB_HOME%\logs
- set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS%
-Xloggc:"%IOTDB_HOME%\logs\gc.log" -XX:+PrintGCDateStamps -XX:+PrintGCDetails
-XX:+PrintGCApplicationStoppedTime -XX:+PrintPromotionFailure
-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M
+ set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS%
-Xloggc:"%IOTDB_HOME%\logs\gc.log" -XX:+PrintGCDateStamps -XX:+PrintGCDetails
-XX:+PrintGCApplicationStoppedTime -XX:+PrintPromotionFailure
-XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M
+ @REM For more detailed GC information, you can uncomment option
below.
+ @REM NOTE: more detailed GC information may bring larger GC log files.
+ @REM set IOTDB_JMX_OPTS=%IOTDB_JMX_OPTS%
-Xloggc:"%IOTDB_HOME%\logs\gc.log" -XX:+PrintGCDateStamps -XX:+PrintGCDetails
-XX:+PrintGCApplicationStoppedTime -XX:+PrintPromotionFailure
-XX:+UseGCLogFileRotation -XX:+PrintTenuringDistribution -XX:+PrintHeapAtGC
-XX:+PrintReferenceGC -XX:+PrintSafepointStatistics
-XX:PrintSafepointStatisticsCount=1 -XX:NumberOfGCLogFiles=10
-XX:GCLogFileSize=100M
) ELSE (
md %IOTDB_HOME%\logs
set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS%
-Xlog:gc=info,heap*=trace,age*=debug,safepoint=info,promotion*=trace:file="%IOTDB_HOME%\logs\gc.log":time,uptime,pid,tid,level:filecount=10,filesize=10485760
+ @REM For more detailed GC information, you can uncomment option
below.
+ @REM NOTE: more detailed GC information may bring larger GC log files.
+ @REM set IOTDB_JMX_OPTS=%IOTDB_JMX_OPTS%
-Xlog:gc*=debug,heap*=debug,age*=trace,metaspace*=info,safepoint*=debug,promotion*=info:file="%IOTDB_HOME%\logs\gc.log":time,uptime,pid,tid,level,tags:filecount=10,filesize=100M
)
)
diff --git a/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh
b/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh
index 50282ccec3d..e25fc34a47e 100755
--- a/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh
+++ b/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh
@@ -187,7 +187,10 @@ if [ "${version_arr[0]}" = "1" ] ; then
# only add -Xlog:gc if it's not mentioned in jvm-server.options file
mkdir -p ${IOTDB_HOME}/logs
if [ "$#" -ge "1" -a "$1" == "printgc" ]; then
- IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xloggc:${IOTDB_HOME}/logs/gc.log
-XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCApplicationStoppedTime
-XX:+PrintPromotionFailure -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10
-XX:GCLogFileSize=10M"
+ IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xloggc:${IOTDB_HOME}/logs/gc.log
-XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCApplicationStoppedTime
-XX:+PrintPromotionFailure -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10
-XX:GCLogFileSize=10M"
+ # For more detailed GC information, you can uncomment option below.
+ # NOTE: more detailed GC information may bring larger GC log files.
+ # IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS
-Xloggc:${IOTDB_HOME}/logs/gc.log -XX:+PrintGCDateStamps -XX:+PrintGCDetails
-XX:+PrintGCApplicationStoppedTime -XX:+PrintPromotionFailure
-XX:+UseGCLogFileRotation -XX:+PrintTenuringDistribution -XX:+PrintHeapAtGC
-XX:+PrintReferenceGC -XX:+PrintSafepointStatistics
-XX:PrintSafepointStatisticsCount=1 -XX:NumberOfGCLogFiles=10
-XX:GCLogFileSize=100M"
fi
fi
else
@@ -201,6 +204,9 @@ else
mkdir -p ${IOTDB_HOME}/logs
if [ "$#" -ge "1" -a "$1" == "printgc" ]; then
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS
-Xlog:gc=info,heap*=info,age*=info,safepoint=info,promotion*=info:file=${IOTDB_HOME}/logs/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760"
+ # For more detailed GC information, you can uncomment option below.
+ # NOTE: more detailed GC information may bring larger GC log files.
+ # IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS
-Xlog:gc*=debug,heap*=debug,age*=trace,metaspace*=info,safepoint*=debug,promotion*=info:file=${IOTDB_HOME}/logs/gc.log:time,uptime,pid,tid,level,tags:filecount=10,filesize=100M"
fi
fi
# Add argLine for Java 11 and above, due to [JEP 396: Strongly Encapsulate
JDK Internals by Default] (https://openjdk.java.net/jeps/396)
@@ -269,8 +275,37 @@ IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xmx${ON_HEAP_MEMORY}"
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:MaxDirectMemorySize=${OFF_HEAP_MEMORY}"
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS
-Djdk.nio.maxCachedBufferSize=${MAX_CACHED_BUFFER_SIZE}"
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:+CrashOnOutOfMemoryError"
+IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:+UseAdaptiveSizePolicy"
+IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xss512k"
+# options below try to optimize safepoint stw time.
+IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:+UnlockDiagnosticVMOptions"
+IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:GuaranteedSafepointInterval=0"
+IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:-UseBiasedLocking"
+# these two options print safepoints with pauses longer than 1000ms to the
standard output. You can see these logs via redirection when starting in the
background like "start-datanode.sh > log_datanode_safepoint.log"
+IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:SafepointTimeoutDelay=1000"
+IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:+SafepointTimeout"
+
+# option below tries to optimize safepoint stw time for large counted loop.
+# NOTE: it may have an impact on JIT's black-box optimization.
+# IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:+UseCountedLoopSafepoints"
+
+# when the GC time is too long, if there are remaining CPU resources, you can
try to turn on and increase options below.
+# for Linux:
+# CPU_PROCESSOR_NUM=$(nproc)
+# for MacOS:
+# CPU_PROCESSOR_NUM=$(sysctl -n hw.ncpu)
+# IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:ParallelGCThreads=${CPU_PROCESSOR_NUM}"
+
+# if there are much of stw time of reference process in GC log, you can turn
on option below.
+# NOTE: it may have an impact on application's throughput.
+# IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:+ParallelRefProcEnabled"
+
+# this option can reduce the overhead caused by memory allocation, page fault
interrupts, etc. during JVM operation.
+# NOTE: it may reduce memory utilization and trigger OOM killer when memory is
tight.
+# IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:+AlwaysPreTouch"
+
# if you want to dump the heap memory while OOM happening, you can use the
following command, remember to replace /tmp/heapdump.hprof with your own file
path and the folder where this file is located needs to be created in advance
-#IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=/tmp/datanode_heapdump.hprof"
+# IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=/tmp/datanode_heapdump.hprof"
echo "DataNode on heap memory size = ${ON_HEAP_MEMORY}B, off heap memory size
= ${OFF_HEAP_MEMORY}B"
diff --git
a/iotdb-core/node-commons/src/assembly/resources/sbin/start-standalone.bat
b/iotdb-core/node-commons/src/assembly/resources/sbin/start-standalone.bat
index 06688208e6d..4a10d54f6bb 100644
--- a/iotdb-core/node-commons/src/assembly/resources/sbin/start-standalone.bat
+++ b/iotdb-core/node-commons/src/assembly/resources/sbin/start-standalone.bat
@@ -40,6 +40,9 @@ IF EXIST "%IOTDB_HOME%\sbin\start-datanode.bat" (
start cmd /c %CONFIGNODE_START_PATH%
TIMEOUT /T 5 /NOBREAK
start cmd /c %DATANODE_START_PATH%
+@REM if you have turned on "-XX:+SafepointTimeout" and
"-XX:SafepointTimeoutDelay=1000", you can use commands below instead to see
safepoint logs
+@REM SET LOG_SAFEPOINT_PATH=%IOTDB_HOME%\logs\log_datanode_safepoint.log
+@REM start cmd /c %DATANODE_START_PATH% > %LOG_SAFEPOINT_PATH%
echo "Execute start-standalone.sh finished, you can see more details in the
logs of confignode and datanode"
exit 0
diff --git
a/iotdb-core/node-commons/src/assembly/resources/sbin/start-standalone.sh
b/iotdb-core/node-commons/src/assembly/resources/sbin/start-standalone.sh
index 6d81239ab03..41c95dae33e 100644
--- a/iotdb-core/node-commons/src/assembly/resources/sbin/start-standalone.sh
+++ b/iotdb-core/node-commons/src/assembly/resources/sbin/start-standalone.sh
@@ -39,5 +39,7 @@ fi
nohup bash "$CONFIGNODE_START_PATH" >/dev/null 2>&1 &
sleep 3
nohup bash "$DATANODE_START_PATH" >/dev/null 2>&1 &
+# if you have turned on "-XX:+SafepointTimeout" and
"-XX:SafepointTimeoutDelay=1000", you can use commands below instead to see
safepoint logs
+#nohup bash "$DATANODE_START_PATH"
>"$IOTDB_HOME/logs/log_datanode_safepoint.log" 2>&1 &
echo "Execute start-standalone.sh finished, you can see more details in the
logs of confignode and datanode"
