[
https://issues.apache.org/jira/browse/FLINK-16142?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17041059#comment-17041059
]
Thomas Wozniakowski commented on FLINK-16142:
---------------------------------------------
[~arvid heise] I managed to get hold of those metaspace stats you asked for
from inside the docker container. For reference, the easiest way I found to
actually achieve this is to docker exec your way into the container, install
sdkman and then do "sdk install java 8.0.242-open". This seems to give you a
jmap command that is compatible with the running JVM.
{code:java}
root@27da7e6b6873:/usr/local/openjdk-8# jmap -clstats 1
Attaching to process ID 1, please wait...
Debugger attached successfully.
Server compiler detected.
JVM version is 25.242-b08
finding class loader instances ..done.
computing per loader stat ..done.
please wait.. computing liveness.liveness analysis may be inaccurate ...
class_loader classes bytes parent_loader alive? type
<bootstrap> 2466 4237013 null live <internal>
0x00000000e0201820 0 0 0x00000000e0170730 dead
java/util/ResourceBundle$RBClassLoader@0x000000010007c028
0x00000000e0201a20 18 30389 null dead
sun/misc/Launcher$ExtClassLoader@0x000000010000f6b0
0x00000000e0d88ed8 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130bd30 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c982c0 1 1455 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201e008 1 1457 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c620c8 1 866 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffb1f0 1 866 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d60cc0 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cec4d0 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ff9fe0 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffafe0 1 1455 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cec0e8 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130c118 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201ce28 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0ced2e0 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130c500 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d272e0 1 1457 null dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000ffe25bd0 1 1455 null dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cecef8 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffa3c8 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c98af0 1 1457 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1027138 1 873 0x00000000e0399898 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e12da110 1 1455 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130c370 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201ca40 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0dda298 1 1456 0x00000000e0d60c60 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e12da368 1 1457 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0ced088 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffb5b8 1 1457 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c98c80 1 1455 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201d648 1 1494 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e107bd48 1 1457 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffa7b0 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c98898 1 1457 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000ffe25fb8 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c968a8 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c4faa0 0 0 0x00000000e0170730 dead
com/esotericsoftware/reflectasm/AccessClassLoader@0x000000010074c7f0
0x00000000e130c758 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d22ab8 1 1455 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cecca0 1 1455 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e207c868 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0307c40 1 1455 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cec8b8 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffad88 1 1457 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130b948 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201de78 1 1456 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e12da750 1 1455 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d610a8 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d87458 1 1457 0x00000000e0d60c60 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130c9b0 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000ffe25d60 1 1455 null dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0ceca48 1 1458 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d60e50 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130b7b8 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffa170 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130bba0 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201cc98 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0dda040 1 1457 0x00000000e0d60c60 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0399898 1 1068 0x00000000e0170730 dead
sun/reflect/misc/MethodUtil@0x0000000100408c58
0x00000000e201d8a0 1 1455 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffa558 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000ffe26148 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e12d9f80 1 1455 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cec660 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d61478 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201c8b0 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201dab0 1 1455 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cec278 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d60c60 2314 3455226 0x00000000e0170730 dead
org/apache/flink/util/ChildFirstClassLoader@0x0000000100448338
0x00000000e107c1b0 1 1455 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffb748 1 1457 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130bf88 1 1458 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201d4b8 1 1455 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1024fb8 1 866 0x00000000e0d60c60 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0ced470 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1025dc0 1 1455 0x00000000e0d60c60 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c62000 1 1470 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cec408 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130bdf8 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130c1e0 1 1455 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201e0d0 1 1455 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0ced218 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffb128 1 1470 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e207c6d8 1 1458 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c98a28 1 1457 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffaf18 1 1457 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e12d11c0 1 866 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e12da5c0 1 1457 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d61238 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0805c70 1 1455 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c96a38 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e2018af0 1 1455 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201cef0 1 1455 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e12da1d8 1 1494 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130c5c8 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cece30 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffa300 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e039a938 1 1458 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130c438 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201cb08 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e133e238 1 1455 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0dda1d0 1 1455 0x00000000e0d60c60 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cecfc0 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130c820 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201d710 1 1457 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000ffe25ef0 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cecbd8 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e107be10 1 1455 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffa6e8 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130b628 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c987d0 1 1494 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e12d1030 1 1482 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e12da430 1 1457 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130ba10 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201c720 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1fa86d8 2415 3631935 0x00000000e0170730 dead
org/apache/flink/util/ChildFirstClassLoader@0x0000000100448338
0x00000000e0d615f0 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d671f0 1 864 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0170730 4693 7133071 0x00000000e0201a20 dead
sun/misc/Launcher$AppClassLoader@0x000000010000f318
0x00000000e107c020 1 1457 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0805ba8 1 1457 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1024e28 1 1482 0x00000000e0d60c60 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d875e8 1 1455 0x00000000e0d60c60 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d891e8 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c981f8 1 1455 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d60fe0 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1027c38 2415 3631935 0x00000000e0170730 dead
org/apache/flink/util/ChildFirstClassLoader@0x0000000100448338
0x00000000e0cec7f0 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffacc0 1 1455 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c98388 1 1455 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201df40 1 1457 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0307b78 1 1457 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130ca78 1 1455 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0790330 2416 3632765 0x00000000e0170730 dead
org/apache/flink/util/ChildFirstClassLoader@0x0000000100448338
0x00000000e0cec980 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d22b80 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cec598 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffa0a8 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130bc68 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d60d88 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e08059e0 1 1457 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130c050 1 1458 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201cd60 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000ffe26080 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e12da048 1 1455 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e07ee518 1 1456 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0ced3a8 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d613b0 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffa490 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c98bb8 1 1457 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201c978 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d273a8 1 864 null dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000ffe25c98 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cec1b0 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffb680 1 1455 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e03079b0 1 1457 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c98d48 1 1455 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201d580 1 1455 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e107bc80 1 1457 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffa878 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e12da2a0 1 1457 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cec340 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130c2a8 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0ced150 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130c690 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e12da688 1 1455 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cecd68 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d61170 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e207c7a0 1 1458 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c98960 1 1457 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffae50 1 1457 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130b880 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c96970 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130b6f0 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d34710 1 1471 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffa238 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000ffe25e28 1 1455 null dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d60f18 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201cbd0 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c3ad10 1 1456 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0dda108 1 1457 0x00000000e0d60c60 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e12d10f8 1 1470 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e12da4f8 1 1457 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130c8e8 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201d7d8 1 1457 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cecb10 1 1458 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e107bed8 1 1456 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1ffa620 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0cec728 1 864 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130bad8 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201c7e8 1 864 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e201d9e8 1 1482 0x00000000e1fa86d8 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d61738 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e107c0e8 1 1457 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e130bec0 1 864 0x00000000e1027c38 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e1024ef0 1 1470 0x00000000e0d60c60 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0ced538 1 1455 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d87520 1 1457 0x00000000e0d60c60 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0d89120 1 1455 0x00000000e0170730 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
0x00000000e0c61f38 1 1482 0x00000000e0790330 dead
sun/reflect/DelegatingClassLoader@0x0000000100009c70
total = 202 16930 25988254 N/A alive=1, dead=201
N/A
{code}
> Memory Leak causes Metaspace OOM error on repeated job submission
> -----------------------------------------------------------------
>
> Key: FLINK-16142
> URL: https://issues.apache.org/jira/browse/FLINK-16142
> Project: Flink
> Issue Type: Bug
> Components: Client / Job Submission
> Affects Versions: 1.10.0
> Reporter: Thomas Wozniakowski
> Priority: Blocker
> Fix For: 1.10.1, 1.11.0
>
>
> Hi Guys,
> We've just tried deploying 1.10.0 as it has lots of shiny stuff that fits our
> use-case exactly (RocksDB state backend running in a containerised cluster).
> Unfortunately, it seems like there is a memory leak somewhere in the job
> submission logic. We are getting this error:
> {code:java}
> 2020-02-18 10:22:10,020 INFO
> org.apache.flink.runtime.executiongraph.ExecutionGraph - OPERATOR_NAME
> switched from RUNNING to FAILED.
> java.lang.OutOfMemoryError: Metaspace
> at java.lang.ClassLoader.defineClass1(Native Method)
> at java.lang.ClassLoader.defineClass(ClassLoader.java:757)
> at java.security.SecureClassLoader.defineClass(SecureClassLoader.java:142)
> at java.net.URLClassLoader.defineClass(URLClassLoader.java:468)
> at java.net.URLClassLoader.access$100(URLClassLoader.java:74)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:369)
> at java.net.URLClassLoader$1.run(URLClassLoader.java:363)
> at java.security.AccessController.doPrivileged(Native Method)
> at java.net.URLClassLoader.findClass(URLClassLoader.java:362)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:419)
> at
> org.apache.flink.util.ChildFirstClassLoader.loadClass(ChildFirstClassLoader.java:60)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:352)
> at
> org.apache.flink.kinesis.shaded.com.amazonaws.jmx.SdkMBeanRegistrySupport.registerMetricAdminMBean(SdkMBeanRegistrySupport.java:27)
> at
> org.apache.flink.kinesis.shaded.com.amazonaws.metrics.AwsSdkMetrics.registerMetricAdminMBean(AwsSdkMetrics.java:398)
> at
> org.apache.flink.kinesis.shaded.com.amazonaws.metrics.AwsSdkMetrics.<clinit>(AwsSdkMetrics.java:359)
> at
> org.apache.flink.kinesis.shaded.com.amazonaws.AmazonWebServiceClient.requestMetricCollector(AmazonWebServiceClient.java:728)
> at
> org.apache.flink.kinesis.shaded.com.amazonaws.AmazonWebServiceClient.isRMCEnabledAtClientOrSdkLevel(AmazonWebServiceClient.java:660)
> at
> org.apache.flink.kinesis.shaded.com.amazonaws.AmazonWebServiceClient.isRequestMetricsEnabled(AmazonWebServiceClient.java:652)
> at
> org.apache.flink.kinesis.shaded.com.amazonaws.AmazonWebServiceClient.createExecutionContext(AmazonWebServiceClient.java:611)
> at
> org.apache.flink.kinesis.shaded.com.amazonaws.AmazonWebServiceClient.createExecutionContext(AmazonWebServiceClient.java:606)
> at
> org.apache.flink.kinesis.shaded.com.amazonaws.services.kinesis.AmazonKinesisClient.executeListShards(AmazonKinesisClient.java:1534)
> at
> org.apache.flink.kinesis.shaded.com.amazonaws.services.kinesis.AmazonKinesisClient.listShards(AmazonKinesisClient.java:1528)
> at
> org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxy.listShards(KinesisProxy.java:439)
> at
> org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxy.getShardsOfStream(KinesisProxy.java:389)
> at
> org.apache.flink.streaming.connectors.kinesis.proxy.KinesisProxy.getShardList(KinesisProxy.java:279)
> at
> org.apache.flink.streaming.connectors.kinesis.internals.KinesisDataFetcher.discoverNewShardsToSubscribe(KinesisDataFetcher.java:686)
> at
> org.apache.flink.streaming.connectors.kinesis.FlinkKinesisConsumer.run(FlinkKinesisConsumer.java:287)
> at
> org.apache.flink.streaming.api.operators.StreamSource.run(StreamSource.java:100)
> at
> org.apache.flink.streaming.api.operators.StreamSource.run(StreamSource.java:63)
> {code}
> (The only change in the above text is the OPERATOR_NAME text where I removed
> some of the internal specifics of our system).
> This will reliably happen on a fresh cluster after submitting and cancelling
> our job 3 times.
> We are using the presto-s3 plugin, the CEP library and the Kinesis connector.
> Please let me know what other diagnostics would be useful.
> Tom
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
