After upgrading from 2.7.1 version to 2.10.0 version ignite nodes facing huge full GC operations after 24-36 hours after node start.
We try to increase heap size but no luck, here is the start configuration for nodes; JVM_OPTS="$JVM_OPTS -Xms12g -Xmx12g -server -javaagent:/etc/prometheus/jmx_prometheus_javaagent-0.14.0.jar=8090:/etc/prometheus/jmx.yml -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=49165 -Dcom.sun.management.jmxremote.host=localhost -XX:MaxMetaspaceSize=256m -XX:MaxDirectMemorySize=1g -DIGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK=true -DIGNITE_WAL_MMAP=true -DIGNITE_BPLUS_TREE_LOCK_RETRIES=100000 -Djava.net.preferIPv4Stack=true" JVM_OPTS="$JVM_OPTS -XX:+AlwaysPreTouch -XX:+UseG1GC -XX:+ScavengeBeforeFullGC -XX:+DisableExplicitGC -XX:+UseStringDeduplication -Xloggc:/var/log/apache-ignite/gc.log -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -XX:+PrintGCCause -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M" here is the 80 hours of GC analyize report: https://gceasy.io/my-gc-report.jsp?p=c2hhcmVkLzIwMjEvMDgvMzEvLS1nYy5sb2cuMC5jdXJyZW50LnppcC0tNS01MS0yOQ==&channel=WEB do we need more heap size or is there a BUG that we need to be aware? here is the node configuration: <?xml version="1.0" encoding="UTF-8"?> <beans xmlns="http://www.springframework.org/schema/beans"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation=" http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd";> <bean id="ignite.cfg" class="org.apache.ignite.configuration.IgniteConfiguration"> <property name="gridLogger"> <bean class="org.apache.ignite.logger.log4j2.Log4J2Logger"> <constructor-arg type="java.lang.String" value="/etc/apache-ignite/ignite-log4j2.xml"/> </bean> </property> <property name="communicationSpi"> <bean class="org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi"> <property name="usePairedConnections" value="true"/> </bean> </property> <property name="failureDetectionTimeout" value="60000"/> <property name="systemThreadPoolSize" value="128"/> <property name="publicThreadPoolSize" value="128"/> <property name="queryThreadPoolSize" value="128"/> <property name="serviceThreadPoolSize" value="128"/> <property name="stripedPoolSize" value="128"/> <property name="dataStreamerThreadPoolSize" value="4"/> <property name="rebalanceThreadPoolSize" value="16"/> <!-- Explicitly enable peer class loading. --> <property name="peerClassLoadingEnabled" value="true"/> <!-- Enable deploymentSpi, /usr/share/apache-ignite/libs/segmentify directory will be checked every 5 seconds for changed files--> <property name="deploymentSpi"> <bean class="org.apache.ignite.spi.deployment.uri.UriDeploymentSpi"> <property name="temporaryDirectoryPath" value="/tmp/temp_ignite_libs"/> <property name="uriList"> <list> <value>file://freq=5000@localhost/usr/share/apache-ignite/libs/segmentify/</value> </list> </property> </bean> </property> <property name="cacheConfiguration"> <list> <!-- Partitioned cache example configuration (Atomic mode). --> <bean class="org.apache.ignite.configuration.CacheConfiguration"> <property name="name" value="default"/> <property name="atomicityMode" value="ATOMIC"/> <property name="backups" value="1"/> </bean> </list> </property> <!-- Explicitly configure TCP discovery SPI to provide list of initial nodes. --> <property name="discoverySpi"> <bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi"> <property name="networkTimeout" value="60000"/> <property name="ipFinder"> <bean class="org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder"> <property name="addresses"> <list> <!-- THERE ARE 10 NODES --> </list> </property> </bean> </property> </bean> </property> <!-- Enabling Apache Ignite native persistence. --> <property name="dataStorageConfiguration"> <bean class="org.apache.ignite.configuration.DataStorageConfiguration"> <property name="defaultDataRegionConfiguration"> <bean class="org.apache.ignite.configuration.DataRegionConfiguration"> <property name="persistenceEnabled" value="true"/> <property name="checkpointPageBufferSize" value="#{ 2L * 1024 * 1024 * 1024}"/> <property name="maxSize" value="#{ 40L * 1024 * 1024 * 1024 }"/> </bean> </property> <property name="storagePath" value="/srv/ignite/persist"/> <property name="walPath" value="/srv/ignite/wal"/> <property name="walArchivePath" value="/srv/ignite/wal"/> <property name="walMode" value="LOG_ONLY"/> <property name="walSegmentSize" value="#{ 256L * 1024 * 1024 }"/> <property name="walFlushFrequency" value="5000"/> <property name="maxWalArchiveSize" value="#{ 512L * 1024 * 1024 }"/> <property name="writeThrottlingEnabled" value="true"/> <property name="checkpointFrequency" value="300000"/> <property name="checkpointWriteOrder" value="SEQUENTIAL" /> </bean> </property> </bean> -- <https://www.segmentify.com/>İbrahim Halil AltunSenior Software Engineer+90 536 3327510 • segmentify.com → <https://www.segmentify.com/>UK • Germany • Turkey <https://www.segmentify.com/ecommerce-growth-show> <https://www.g2.com/products/segmentify/reviews>
