Ignite has been deployed on the kubernets , there are 3 replicas of server
pod. The pods were up and running fine for 9 days. We have created 180
inventory tables and 204 transactional tables. The data has been
inserted using the PyIgnite client using the cache.put() method. This is a
very slow operation because PyIgnite is very slow. Each insert is
committed one at a time, so it is not able to do bulk-style inserts. The
PyIgnite was inserting about 20 of the inventory tables simultaneously (20
different threads/processes).
The cluster was nowhere stable after 9days, one of the pod crashed and
failed to recover. Below is the error log:
{"type":"log","host":"ignite-cluster-ignite-esoc-2","level":"ERROR","system":"ignite-service","time":"2019-08-16T17:13:34,769Z","logger":"GridCachePartitionExchangeManager","timezone":"UTC","log":"Failed
to process custom exchange task: ClientCacheChangeDummyDiscoveryMessage
[reqId=6b5f6c50-a8c9-4b04-a461-49bfd0112eb0, cachesToClose=null,
startCaches=[BgwService]] java.lang.NullPointerException| at
org.apache.ignite.internal.processors.cache.CacheAffinitySharedManager.processClientCachesChanges(CacheAffinitySharedManager.java:635)|
at
org.apache.ignite.internal.processors.cache.GridCacheProcessor.processCustomExchangeTask(GridCacheProcessor.java:391)|
at
org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.processCustomTask(GridCachePartitionExchangeManager.java:2475)|
at
org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body0(GridCachePartitionExchangeManager.java:2620)|
at
org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$ExchangeWorker.body(GridCachePartitionExchangeManager.java:2539)|
at
org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:120)|
at java.lang.Thread.run(Thread.java:748)"}
{"type":"log","host":"ignite-cluster-ignite-esoc-2","level":"WARN","system":"ignite-service","time":"2019-08-16T17:13:36,724Z","logger":"GridCacheDatabaseSharedManager","timezone":"UTC","log":"Ignite
node stopped in the middle of checkpoint. Will restore memory state and
finish checkpoint on node start."}
The error report file and ignite-config.xml has been attached for your info.
Heap Memory and RAM Configurations are as below on each of the ignite
server container:
Heap Memory: 32gb
RAM: 64GB
Default memory region:
cpu: 4
Persistence volume
wal_storage_size: 10GB
persistence_storage_size: 10GB
Thanks
With Regards
Radha
ignite-config.xml:
----
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xmlns:util="http://www.springframework.org/schema/util";
xsi:schemaLocation="
http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/util
http://www.springframework.org/schema/util/spring-util.xsd";>
<bean class="org.apache.ignite.configuration.IgniteConfiguration">
<property name="WorkDirectory" value="/opt/ignite/persistence//work"/>
<property name="connectorConfiguration">
<bean class="org.apache.ignite.configuration.ConnectorConfiguration">
<property name="jettyPath" value="/opt/ignite/conf/jetty-server.xml" />
</bean>
</property>
<property name="peerClassLoadingEnabled" value="true"/>
<property name="dataStorageConfiguration">
<bean class="org.apache.ignite.configuration.DataStorageConfiguration">
<!-- Enable metrics for Ignite persistence -->
<property name="metricsEnabled" value="true"/>
<property name="defaultDataRegionConfiguration">
<bean class="org.apache.ignite.configuration.DataRegionConfiguration">
<property name="name" value="Default_Region"/>
<property name="pageEvictionMode" value="RANDOM_2_LRU"/>
<property name="initialSize" value="#{20 * 1024 * 1024}"/>
<property name="maxSize" value="#{60L * 1024 * 1024 * 1024}"/>
<!-- Enabling Apache Ignite Persistent Store. -->
<property name="persistenceEnabled" value="true"/>
<!-- Enable metrics for this data region -->
<property name="metricsEnabled" value="true"/>
</bean>
</property>
<property name="storagePath" value="/opt/ignite/persistence/"/>
<property name="walPath" value="/opt/ignite/wal/"/>
<property name="walArchivePath" value="/opt/ignite/wal/archive"/>
<property name="walMode" value="FSYNC"/>
</bean>
</property>
<property name="authenticationEnabled" value="false"/>
<property name="gridLogger">
<bean class="org.apache.ignite.logger.log4j2.Log4J2Logger">
<constructor-arg type="java.lang.String" value="/etc/ignite/log4j2/ignite-log4j2.xml"/>
</bean>
</property>
<property name="discoverySpi">
<bean class="org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi">
<property name="ipFinder">
<bean
class="org.apache.ignite.spi.discovery.tcp.ipfinder.kubernetes.TcpDiscoveryKubernetesIpFinder">
<property name="serviceName" value="ignite-service"/>
<property name="namespace" value="default"/>
</bean>
</property>
</bean>
</property>
<!--UPDATE THE IGNITE CACHE CONFIGURATIONS HERE-->
<property name="sqlSchemas">
<list>
<value>Trans</value>
<value>Info</value>
<value>Msg</value>
</list>
</property>
<property name="cacheConfiguration">
<list>
<bean id="cache-template-bean" abstract="true" class="org.apache.ignite.configuration.CacheConfiguration">
<property name="name" value="templateabc1*"/>
<property name="cacheMode" value="PARTITIONED"/>
<property name="backups" value="1"/>
<property name="groupName" value="groupabc1"/>
<property name="expiryPolicyFactory">
<bean class="javax.cache.expiry.CreatedExpiryPolicy" factory-method="factoryOf">
<constructor-arg>
<bean class="javax.cache.expiry.Duration">
<constructor-arg value="MINUTES"/>
<constructor-arg value="75"/>
</bean>
</constructor-arg>
</bean>
</property>
</bean>
<bean id="cache-template-bean" abstract="true" class="org.apache.ignite.configuration.CacheConfiguration">
<property name="name" value="templateabc6*"/>
<property name="cacheMode" value="PARTITIONED"/>
<property name="backups" value="1"/>
<property name="groupName" value="groupabc6"/>
<property name="expiryPolicyFactory">
<bean class="javax.cache.expiry.CreatedExpiryPolicy" factory-method="factoryOf">
<constructor-arg>
<bean class="javax.cache.expiry.Duration">
<constructor-arg value="HOURS"/>
<constructor-arg value="25"/>
</bean>
</constructor-arg>
</bean>
</property>
</bean>
<bean id="cache-template-bean" abstract="true" class="org.apache.ignite.configuration.CacheConfiguration">
<property name="name" value="templateabc16*"/>
<property name="cacheMode" value="PARTITIONED"/>
<property name="backups" value="1"/>
<property name="groupName" value="groupabc16"/>
<property name="expiryPolicyFactory">
<bean class="javax.cache.expiry.CreatedExpiryPolicy" factory-method="factoryOf">
<constructor-arg>
<bean class="javax.cache.expiry.Duration">
<constructor-arg value="HOURS"/>
<constructor-arg value="25"/>
</bean>
</constructor-arg>
</bean>
</property>
</bean>
<bean id="cache-template-bean" abstract="true" class="org.apache.ignite.configuration.CacheConfiguration">
<property name="name" value="templateTrans*"/>
<property name="cacheMode" value="PARTITIONED"/>
<property name="backups" value="1"/>
<property name="groupName" value="groupTrans"/>
</bean>
</list>
</property>
</bean>
</beans>
