[ 
https://issues.apache.org/jira/browse/IGNITE-3338?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
 ]

Ksenia Rybakova updated IGNITE-3338:
------------------------------------
    Description: 
The following exceptions occur during load test with eviction configured:

{noformat}
ERROR: The benchmark of random operation failed.
Type '--help' for usage.
Finishing main test [ts=1466178657166, date=Fri Jun 17 08:50:57 PDT 2016]
ERROR: Shutting down benchmark driver to unexpected exception.
Type '--help' for usage.
org.apache.ignite.cache.CachePartialUpdateException: Failed to update keys 
(retry update if possible).: [268888, 1872066, 1945790, 2405741, 3845651, 
4830233, 7284502, 8555643]
        at 
org.apache.ignite.internal.processors.cache.GridCacheUtils.convertToCacheException(GridCacheUtils.java:1467)
        at 
org.apache.ignite.internal.processors.cache.IgniteCacheProxy.cacheException(IgniteCacheProxy.java:1972)
        at 
org.apache.ignite.internal.processors.cache.IgniteCacheProxy.removeAll(IgniteCacheProxy.java:1416)
        at 
org.apache.ignite.yardstick.cache.load.IgniteCacheRandomOperationBenchmark.doRemoveAll(IgniteCacheRandomOperationBenchmark.java:809)
        at 
org.apache.ignite.yardstick.cache.load.IgniteCacheRandomOperationBenchmark.executeRandomOperation(IgniteCacheRandomOperationBenchmark.java:551)
        at 
org.apache.ignite.yardstick.cache.load.IgniteCacheRandomOperationBenchmark.executeOutOfTx(IgniteCacheRandomOperationBenchmark.java:509)
        at 
org.apache.ignite.yardstick.cache.load.IgniteCacheRandomOperationBenchmark.test(IgniteCacheRandomOperationBenchmark.java:156)
        at 
org.yardstickframework.impl.BenchmarkRunner$2.run(BenchmarkRunner.java:176)
        at java.lang.Thread.run(Thread.java:745)
Caused by: class 
org.apache.ignite.internal.processors.cache.CachePartialUpdateCheckedException: 
Failed to update keys (retry update if possible).: [268888, 1872066, 1945790, 
2405741, 3845651, 4830233, 7284502, 8555643]
        at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicUpdateFuture.onResult(GridNearAtomicUpdateFuture.java:311)
        at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processNearAtomicUpdateResponse(GridDhtAtomicCache.java:2958)
        at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$700(GridDhtAtomicCache.java:129)
        at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$6.apply(GridDhtAtomicCache.java:266)
        at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$6.apply(GridDhtAtomicCache.java:264)
        at 
org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:624)
        at 
org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:322)
        at 
org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:246)
        at 
org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$000(GridCacheIoManager.java:83)
        at 
org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:205)
        at 
org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1219)
        at 
org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:847)
        at 
org.apache.ignite.internal.managers.communication.GridIoManager.access$1700(GridIoManager.java:105)
        at 
org.apache.ignite.internal.managers.communication.GridIoManager$5.run(GridIoManager.java:810)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        ... 1 more
        Suppressed: class org.apache.ignite.IgniteCheckedException: Failed to 
update keys on primary node.
                at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicUpdateResponse.addFailedKeys(GridNearAtomicUpdateResponse.java:369)
                at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal0(GridDhtAtomicCache.java:1711)
                at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal(GridDhtAtomicCache.java:1482)
                at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processNearAtomicUpdateRequest(GridDhtAtomicCache.java:2937)
                at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$600(GridDhtAtomicCache.java:129)
                at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$5.apply(GridDhtAtomicCache.java:260)
                at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$5.apply(GridDhtAtomicCache.java:258)
                ... 12 more
                Suppressed: class 
org.apache.ignite.binary.BinaryObjectException: Failed to get field because 
type ID of passed object differs from type ID this BinaryField belongs to 
[expected=-1059871450, actual=-1558749226]
                        at 
org.apache.ignite.internal.binary.BinaryFieldImpl.fieldOrder(BinaryFieldImpl.java:92)
                        at 
org.apache.ignite.internal.binary.BinaryFieldImpl.value(BinaryFieldImpl.java:79)
                        at 
org.apache.ignite.internal.processors.query.GridQueryProcessor$BinaryProperty.fieldValue(GridQueryProcessor.java:2051)
                        at 
org.apache.ignite.internal.processors.query.GridQueryProcessor$BinaryProperty.value(GridQueryProcessor.java:2011)
                        at 
org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing$RowDescriptor.columnValue(IgniteH2Indexing.java:2543)
                        at 
org.apache.ignite.internal.processors.query.h2.opt.GridH2AbstractKeyValueRow.getValue(GridH2AbstractKeyValueRow.java:289)
                        at 
org.apache.ignite.internal.processors.query.h2.opt.GridH2IndexBase.compareRows(GridH2IndexBase.java:116)
                        at 
org.apache.ignite.internal.processors.query.h2.opt.GridH2TreeIndex.compare(GridH2TreeIndex.java:248)
                        at 
org.apache.ignite.internal.processors.query.h2.opt.GridH2TreeIndex.compare(GridH2TreeIndex.java:49)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap$2.compareTo(GridOffHeapSnapTreeMap.java:1350)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap$2.compareTo(GridOffHeapSnapTreeMap.java:1346)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2102)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.updateUnderRoot(GridOffHeapSnapTreeMap.java:2034)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.update(GridOffHeapSnapTreeMap.java:1915)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.remove(GridOffHeapSnapTreeMap.java:1884)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.remove(GridOffHeapSnapTreeMap.java:108)
                        at 
org.apache.ignite.internal.processors.query.h2.opt.GridH2TreeIndex.remove(GridH2TreeIndex.java:408)
                        at 
org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.doUpdate(GridH2Table.java:433)
                        at 
org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.update(GridH2Table.java:339)
                        at 
org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.remove(IgniteH2Indexing.java:602)
                        at 
org.apache.ignite.internal.processors.query.GridQueryProcessor.remove(GridQueryProcessor.java:1014)
                        at 
org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager.remove(GridCacheQueryManager.java:434)
                        at 
org.apache.ignite.internal.processors.cache.GridCacheMapEntry.clearIndex(GridCacheMapEntry.java:4027)
                        at 
org.apache.ignite.internal.processors.cache.GridCacheMapEntry.innerUpdate(GridCacheMapEntry.java:2509)
                        at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateSingle(GridDhtAtomicCache.java:2242)
                        at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal0(GridDhtAtomicCache.java:1642)
                        ... 17 more
{noformat}

Observation: the issue is reproduced when eviction is configured for cache with 
indexed type.

Load config:
- benchmark: IgniteCacheRandomOperationBenchmark
- preload entries: 5M
- key range: 10M
- servers: 20 (5 per host)
- clients: 1
- operations: PUT PUT_ALL GET GET_ALL INVOKE INVOKE_ALL REMOVE REMOVE_ALL 
PUT_IF_ABSENT REPLACE
- Xmx8g -Xms8g
- GC used: G1
- backups: 2
- batch size: 50
- thread count: 64
- sync mode: PRIMARY_SYNC
- cache config:
{noformat}
    <bean name="atomic" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="atomic"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="ATOMIC"/>
        <property name="swapEnabled" value="false"/>
        <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
        <property name="evictionPolicy">
            <bean 
class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
                <property name="maxSize" value="750000"/>
            </bean>
        </property>
    </bean>

    <bean name="atomic-offheap" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="atomic-offheap"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="ATOMIC"/>
        <property name="swapEnabled" value="false"/>
        <property name="memoryMode" value="OFFHEAP_TIERED"/>
    </bean>

    <bean name="atomic-offheap-values" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="atomic-offheap-values"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="ATOMIC"/>
        <property name="swapEnabled" value="false"/>
        <property name="memoryMode" value="OFFHEAP_VALUES"/>
    </bean>

    <bean name="tx" class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="tx"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="TRANSACTIONAL"/>
        <property name="swapEnabled" value="false"/>
        <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
        <property name="evictionPolicy">
            <bean 
class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
                <property name="maxSize" value="750000"/>
            </bean>
        </property>
    </bean>

    <bean name="tx-offheap" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="tx-offheap"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="TRANSACTIONAL"/>
        <property name="swapEnabled" value="false"/>
        <property name="memoryMode" value="OFFHEAP_TIERED"/>
    </bean>

    <bean name="tx-offheap-values" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="tx-offheap-values"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="TRANSACTIONAL"/>
        <property name="swapEnabled" value="false"/>
        <property name="memoryMode" value="OFFHEAP_VALUES"/>
    </bean>

    <bean name="atomic-index" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="atomic-index"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="ATOMIC"/>
        <property name="swapEnabled" value="false"/>
        <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
        <property name="evictionPolicy">
            <bean 
class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
                <property name="maxSize" value="750000"/>
            </bean>
        </property>
        <property name="indexedTypes">
            <list>
                <value>java.lang.Integer</value>
                <value>org.apache.ignite.yardstick.cache.model.Person1</value>

            </list>
        </property>
    </bean>

    <bean name="query" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="query"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="ATOMIC"/>
        <property name="swapEnabled" value="false"/>
        <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
        <property name="evictionPolicy">
            <bean 
class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
                <property name="maxSize" value="750000"/>
            </bean>
        </property>

        <!-- Configure query entities -->
        <property name="queryEntities">
            <list>
                <bean class="org.apache.ignite.cache.QueryEntity">
                    <property name="keyType" value="java.lang.Integer"/>
                    <property name="valueType" 
value="org.apache.ignite.yardstick.cache.model.Person"/>

                    <property name="fields">
                        <map>
                            <entry key="id" value="java.lang.Integer"/>
                            <entry key="orgId" value="java.lang.Integer"/>
                            <entry key="firstName" value="java.lang.String"/>
                            <entry key="lastName" value="java.lang.String"/>
                            <entry key="salary" value="java.lang.Double"/>
                        </map>
                    </property>

                    <property name="indexes">
                        <list>
                            <bean class="org.apache.ignite.cache.QueryIndex">
                                <constructor-arg value="id"/>
                            </bean>
                            <!-- Group index. -->
                            <bean class="org.apache.ignite.cache.QueryIndex">
                                <constructor-arg>
                                    <list>
                                        <value>id</value>
                                        <value>salary</value>
                                    </list>
                                </constructor-arg>
                                <constructor-arg value="SORTED"/>
                            </bean>
                        </list>
                    </property>
                </bean>
            </list>
        </property>
    </bean>

    <bean name="orgCache" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="orgCache"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="ATOMIC"/>
        <property name="swapEnabled" value="false"/>
        <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
        <property name="evictionPolicy">
            <bean 
class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
                <property name="maxSize" value="750000"/>
            </bean>
        </property>

        <!-- Configure query entities -->
        <property name="queryEntities">
            <list>
                <bean class="org.apache.ignite.cache.QueryEntity">
                    <property name="keyType" value="java.lang.Integer"/>
                    <property name="valueType" 
value="org.apache.ignite.yardstick.cache.model.Organization"/>

                    <property name="fields">
                        <map>
                            <entry key="id" value="java.lang.Integer"/>
                            <entry key="name" value="java.lang.String"/>
                        </map>
                    </property>

                    <property name="indexes">
                        <list>
                            <bean class="org.apache.ignite.cache.QueryIndex">
                                <constructor-arg value="id"/>
                            </bean>
                            <bean class="org.apache.ignite.cache.QueryIndex">
                                <constructor-arg value="name"/>
                            </bean>
                        </list>
                    </property>
                </bean>
            </list>
        </property>
    </bean>

    <bean name="compute" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="compute"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="TRANSACTIONAL"/>
        <property name="swapEnabled" value="false"/>
        <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
        <property name="evictionPolicy">
            <bean 
class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
                <property name="maxSize" value="750000"/>
            </bean>
        </property>
    </bean>

{noformat}


  was:
The following exceptions occur during load test with eviction configured:

{noformat}
ERROR: The benchmark of random operation failed.
Type '--help' for usage.
Finishing main test [ts=1466178657166, date=Fri Jun 17 08:50:57 PDT 2016]
ERROR: Shutting down benchmark driver to unexpected exception.
Type '--help' for usage.
org.apache.ignite.cache.CachePartialUpdateException: Failed to update keys 
(retry update if possible).: [268888, 1872066, 1945790, 2405741, 3845651, 
4830233, 7284502, 8555643]
        at 
org.apache.ignite.internal.processors.cache.GridCacheUtils.convertToCacheException(GridCacheUtils.java:1467)
        at 
org.apache.ignite.internal.processors.cache.IgniteCacheProxy.cacheException(IgniteCacheProxy.java:1972)
        at 
org.apache.ignite.internal.processors.cache.IgniteCacheProxy.removeAll(IgniteCacheProxy.java:1416)
        at 
org.apache.ignite.yardstick.cache.load.IgniteCacheRandomOperationBenchmark.doRemoveAll(IgniteCacheRandomOperationBenchmark.java:809)
        at 
org.apache.ignite.yardstick.cache.load.IgniteCacheRandomOperationBenchmark.executeRandomOperation(IgniteCacheRandomOperationBenchmark.java:551)
        at 
org.apache.ignite.yardstick.cache.load.IgniteCacheRandomOperationBenchmark.executeOutOfTx(IgniteCacheRandomOperationBenchmark.java:509)
        at 
org.apache.ignite.yardstick.cache.load.IgniteCacheRandomOperationBenchmark.test(IgniteCacheRandomOperationBenchmark.java:156)
        at 
org.yardstickframework.impl.BenchmarkRunner$2.run(BenchmarkRunner.java:176)
        at java.lang.Thread.run(Thread.java:745)
Caused by: class 
org.apache.ignite.internal.processors.cache.CachePartialUpdateCheckedException: 
Failed to update keys (retry update if possible).: [268888, 1872066, 1945790, 
2405741, 3845651, 4830233, 7284502, 8555643]
        at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicUpdateFuture.onResult(GridNearAtomicUpdateFuture.java:311)
        at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processNearAtomicUpdateResponse(GridDhtAtomicCache.java:2958)
        at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$700(GridDhtAtomicCache.java:129)
        at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$6.apply(GridDhtAtomicCache.java:266)
        at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$6.apply(GridDhtAtomicCache.java:264)
        at 
org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:624)
        at 
org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:322)
        at 
org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:246)
        at 
org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$000(GridCacheIoManager.java:83)
        at 
org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:205)
        at 
org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1219)
        at 
org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:847)
        at 
org.apache.ignite.internal.managers.communication.GridIoManager.access$1700(GridIoManager.java:105)
        at 
org.apache.ignite.internal.managers.communication.GridIoManager$5.run(GridIoManager.java:810)
        at 
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        at 
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        ... 1 more
        Suppressed: class org.apache.ignite.IgniteCheckedException: Failed to 
update keys on primary node.
                at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicUpdateResponse.addFailedKeys(GridNearAtomicUpdateResponse.java:369)
                at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal0(GridDhtAtomicCache.java:1711)
                at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal(GridDhtAtomicCache.java:1482)
                at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processNearAtomicUpdateRequest(GridDhtAtomicCache.java:2937)
                at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$600(GridDhtAtomicCache.java:129)
                at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$5.apply(GridDhtAtomicCache.java:260)
                at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$5.apply(GridDhtAtomicCache.java:258)
                ... 12 more
                Suppressed: class 
org.apache.ignite.binary.BinaryObjectException: Failed to get field because 
type ID of passed object differs from type ID this BinaryField belongs to 
[expected=-1059871450, actual=-1558749226]
                        at 
org.apache.ignite.internal.binary.BinaryFieldImpl.fieldOrder(BinaryFieldImpl.java:92)
                        at 
org.apache.ignite.internal.binary.BinaryFieldImpl.value(BinaryFieldImpl.java:79)
                        at 
org.apache.ignite.internal.processors.query.GridQueryProcessor$BinaryProperty.fieldValue(GridQueryProcessor.java:2051)
                        at 
org.apache.ignite.internal.processors.query.GridQueryProcessor$BinaryProperty.value(GridQueryProcessor.java:2011)
                        at 
org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing$RowDescriptor.columnValue(IgniteH2Indexing.java:2543)
                        at 
org.apache.ignite.internal.processors.query.h2.opt.GridH2AbstractKeyValueRow.getValue(GridH2AbstractKeyValueRow.java:289)
                        at 
org.apache.ignite.internal.processors.query.h2.opt.GridH2IndexBase.compareRows(GridH2IndexBase.java:116)
                        at 
org.apache.ignite.internal.processors.query.h2.opt.GridH2TreeIndex.compare(GridH2TreeIndex.java:248)
                        at 
org.apache.ignite.internal.processors.query.h2.opt.GridH2TreeIndex.compare(GridH2TreeIndex.java:49)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap$2.compareTo(GridOffHeapSnapTreeMap.java:1350)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap$2.compareTo(GridOffHeapSnapTreeMap.java:1346)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2102)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.updateUnderRoot(GridOffHeapSnapTreeMap.java:2034)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.update(GridOffHeapSnapTreeMap.java:1915)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.remove(GridOffHeapSnapTreeMap.java:1884)
                        at 
org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.remove(GridOffHeapSnapTreeMap.java:108)
                        at 
org.apache.ignite.internal.processors.query.h2.opt.GridH2TreeIndex.remove(GridH2TreeIndex.java:408)
                        at 
org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.doUpdate(GridH2Table.java:433)
                        at 
org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.update(GridH2Table.java:339)
                        at 
org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.remove(IgniteH2Indexing.java:602)
                        at 
org.apache.ignite.internal.processors.query.GridQueryProcessor.remove(GridQueryProcessor.java:1014)
                        at 
org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager.remove(GridCacheQueryManager.java:434)
                        at 
org.apache.ignite.internal.processors.cache.GridCacheMapEntry.clearIndex(GridCacheMapEntry.java:4027)
                        at 
org.apache.ignite.internal.processors.cache.GridCacheMapEntry.innerUpdate(GridCacheMapEntry.java:2509)
                        at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateSingle(GridDhtAtomicCache.java:2242)
                        at 
org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal0(GridDhtAtomicCache.java:1642)
                        ... 17 more
{noformat}

Load config:
- benchmark: IgniteCacheRandomOperationBenchmark
- preload entries: 5M
- key range: 10M
- servers: 20 (5 per host)
- clients: 1
- operations: PUT PUT_ALL GET GET_ALL INVOKE INVOKE_ALL REMOVE REMOVE_ALL 
PUT_IF_ABSENT REPLACE
- Xmx8g -Xms8g
- GC used: G1
- backups: 2
- batch size: 50
- thread count: 64
- sync mode: PRIMARY_SYNC
- cache config:
{noformat}
    <bean name="atomic" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="atomic"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="ATOMIC"/>
        <property name="swapEnabled" value="false"/>
        <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
        <property name="evictionPolicy">
            <bean 
class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
                <property name="maxSize" value="750000"/>
            </bean>
        </property>
    </bean>

    <bean name="atomic-offheap" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="atomic-offheap"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="ATOMIC"/>
        <property name="swapEnabled" value="false"/>
        <property name="memoryMode" value="OFFHEAP_TIERED"/>
    </bean>

    <bean name="atomic-offheap-values" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="atomic-offheap-values"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="ATOMIC"/>
        <property name="swapEnabled" value="false"/>
        <property name="memoryMode" value="OFFHEAP_VALUES"/>
    </bean>

    <bean name="tx" class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="tx"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="TRANSACTIONAL"/>
        <property name="swapEnabled" value="false"/>
        <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
        <property name="evictionPolicy">
            <bean 
class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
                <property name="maxSize" value="750000"/>
            </bean>
        </property>
    </bean>

    <bean name="tx-offheap" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="tx-offheap"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="TRANSACTIONAL"/>
        <property name="swapEnabled" value="false"/>
        <property name="memoryMode" value="OFFHEAP_TIERED"/>
    </bean>

    <bean name="tx-offheap-values" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="tx-offheap-values"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="TRANSACTIONAL"/>
        <property name="swapEnabled" value="false"/>
        <property name="memoryMode" value="OFFHEAP_VALUES"/>
    </bean>

    <bean name="atomic-index" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="atomic-index"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="ATOMIC"/>
        <property name="swapEnabled" value="false"/>
        <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
        <property name="evictionPolicy">
            <bean 
class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
                <property name="maxSize" value="750000"/>
            </bean>
        </property>
        <property name="indexedTypes">
            <list>
                <value>java.lang.Integer</value>
                <value>org.apache.ignite.yardstick.cache.model.Person1</value>

            </list>
        </property>
    </bean>

    <bean name="query" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="query"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="ATOMIC"/>
        <property name="swapEnabled" value="false"/>
        <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
        <property name="evictionPolicy">
            <bean 
class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
                <property name="maxSize" value="750000"/>
            </bean>
        </property>

        <!-- Configure query entities -->
        <property name="queryEntities">
            <list>
                <bean class="org.apache.ignite.cache.QueryEntity">
                    <property name="keyType" value="java.lang.Integer"/>
                    <property name="valueType" 
value="org.apache.ignite.yardstick.cache.model.Person"/>

                    <property name="fields">
                        <map>
                            <entry key="id" value="java.lang.Integer"/>
                            <entry key="orgId" value="java.lang.Integer"/>
                            <entry key="firstName" value="java.lang.String"/>
                            <entry key="lastName" value="java.lang.String"/>
                            <entry key="salary" value="java.lang.Double"/>
                        </map>
                    </property>

                    <property name="indexes">
                        <list>
                            <bean class="org.apache.ignite.cache.QueryIndex">
                                <constructor-arg value="id"/>
                            </bean>
                            <!-- Group index. -->
                            <bean class="org.apache.ignite.cache.QueryIndex">
                                <constructor-arg>
                                    <list>
                                        <value>id</value>
                                        <value>salary</value>
                                    </list>
                                </constructor-arg>
                                <constructor-arg value="SORTED"/>
                            </bean>
                        </list>
                    </property>
                </bean>
            </list>
        </property>
    </bean>

    <bean name="orgCache" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="orgCache"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="ATOMIC"/>
        <property name="swapEnabled" value="false"/>
        <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
        <property name="evictionPolicy">
            <bean 
class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
                <property name="maxSize" value="750000"/>
            </bean>
        </property>

        <!-- Configure query entities -->
        <property name="queryEntities">
            <list>
                <bean class="org.apache.ignite.cache.QueryEntity">
                    <property name="keyType" value="java.lang.Integer"/>
                    <property name="valueType" 
value="org.apache.ignite.yardstick.cache.model.Organization"/>

                    <property name="fields">
                        <map>
                            <entry key="id" value="java.lang.Integer"/>
                            <entry key="name" value="java.lang.String"/>
                        </map>
                    </property>

                    <property name="indexes">
                        <list>
                            <bean class="org.apache.ignite.cache.QueryIndex">
                                <constructor-arg value="id"/>
                            </bean>
                            <bean class="org.apache.ignite.cache.QueryIndex">
                                <constructor-arg value="name"/>
                            </bean>
                        </list>
                    </property>
                </bean>
            </list>
        </property>
    </bean>

    <bean name="compute" 
class="org.apache.ignite.configuration.CacheConfiguration">
        <property name="name" value="compute"/>
        <property name="cacheMode" value="PARTITIONED"/>
        <property name="atomicityMode" value="TRANSACTIONAL"/>
        <property name="swapEnabled" value="false"/>
        <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
        <property name="evictionPolicy">
            <bean 
class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
                <property name="maxSize" value="750000"/>
            </bean>
        </property>
    </bean>

{noformat}



> BinaryObjectException: Failed to get field because type ID of passed object 
> differs from type ID this BinaryField belongs to -  during load test with 
> eviction configured
> -------------------------------------------------------------------------------------------------------------------------------------------------------------------------
>
>                 Key: IGNITE-3338
>                 URL: https://issues.apache.org/jira/browse/IGNITE-3338
>             Project: Ignite
>          Issue Type: Bug
>          Components: general
>            Reporter: Ksenia Rybakova
>             Fix For: 1.7
>
>
> The following exceptions occur during load test with eviction configured:
> {noformat}
> ERROR: The benchmark of random operation failed.
> Type '--help' for usage.
> Finishing main test [ts=1466178657166, date=Fri Jun 17 08:50:57 PDT 2016]
> ERROR: Shutting down benchmark driver to unexpected exception.
> Type '--help' for usage.
> org.apache.ignite.cache.CachePartialUpdateException: Failed to update keys 
> (retry update if possible).: [268888, 1872066, 1945790, 2405741, 3845651, 
> 4830233, 7284502, 8555643]
>         at 
> org.apache.ignite.internal.processors.cache.GridCacheUtils.convertToCacheException(GridCacheUtils.java:1467)
>         at 
> org.apache.ignite.internal.processors.cache.IgniteCacheProxy.cacheException(IgniteCacheProxy.java:1972)
>         at 
> org.apache.ignite.internal.processors.cache.IgniteCacheProxy.removeAll(IgniteCacheProxy.java:1416)
>         at 
> org.apache.ignite.yardstick.cache.load.IgniteCacheRandomOperationBenchmark.doRemoveAll(IgniteCacheRandomOperationBenchmark.java:809)
>         at 
> org.apache.ignite.yardstick.cache.load.IgniteCacheRandomOperationBenchmark.executeRandomOperation(IgniteCacheRandomOperationBenchmark.java:551)
>         at 
> org.apache.ignite.yardstick.cache.load.IgniteCacheRandomOperationBenchmark.executeOutOfTx(IgniteCacheRandomOperationBenchmark.java:509)
>         at 
> org.apache.ignite.yardstick.cache.load.IgniteCacheRandomOperationBenchmark.test(IgniteCacheRandomOperationBenchmark.java:156)
>         at 
> org.yardstickframework.impl.BenchmarkRunner$2.run(BenchmarkRunner.java:176)
>         at java.lang.Thread.run(Thread.java:745)
> Caused by: class 
> org.apache.ignite.internal.processors.cache.CachePartialUpdateCheckedException:
>  Failed to update keys (retry update if possible).: [268888, 1872066, 
> 1945790, 2405741, 3845651, 4830233, 7284502, 8555643]
>         at 
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicUpdateFuture.onResult(GridNearAtomicUpdateFuture.java:311)
>         at 
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processNearAtomicUpdateResponse(GridDhtAtomicCache.java:2958)
>         at 
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$700(GridDhtAtomicCache.java:129)
>         at 
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$6.apply(GridDhtAtomicCache.java:266)
>         at 
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$6.apply(GridDhtAtomicCache.java:264)
>         at 
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:624)
>         at 
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:322)
>         at 
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:246)
>         at 
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$000(GridCacheIoManager.java:83)
>         at 
> org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:205)
>         at 
> org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1219)
>         at 
> org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:847)
>         at 
> org.apache.ignite.internal.managers.communication.GridIoManager.access$1700(GridIoManager.java:105)
>         at 
> org.apache.ignite.internal.managers.communication.GridIoManager$5.run(GridIoManager.java:810)
>         at 
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
>         at 
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
>         ... 1 more
>         Suppressed: class org.apache.ignite.IgniteCheckedException: Failed to 
> update keys on primary node.
>                 at 
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicUpdateResponse.addFailedKeys(GridNearAtomicUpdateResponse.java:369)
>                 at 
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal0(GridDhtAtomicCache.java:1711)
>                 at 
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal(GridDhtAtomicCache.java:1482)
>                 at 
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processNearAtomicUpdateRequest(GridDhtAtomicCache.java:2937)
>                 at 
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$600(GridDhtAtomicCache.java:129)
>                 at 
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$5.apply(GridDhtAtomicCache.java:260)
>                 at 
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$5.apply(GridDhtAtomicCache.java:258)
>                 ... 12 more
>                 Suppressed: class 
> org.apache.ignite.binary.BinaryObjectException: Failed to get field because 
> type ID of passed object differs from type ID this BinaryField belongs to 
> [expected=-1059871450, actual=-1558749226]
>                         at 
> org.apache.ignite.internal.binary.BinaryFieldImpl.fieldOrder(BinaryFieldImpl.java:92)
>                         at 
> org.apache.ignite.internal.binary.BinaryFieldImpl.value(BinaryFieldImpl.java:79)
>                         at 
> org.apache.ignite.internal.processors.query.GridQueryProcessor$BinaryProperty.fieldValue(GridQueryProcessor.java:2051)
>                         at 
> org.apache.ignite.internal.processors.query.GridQueryProcessor$BinaryProperty.value(GridQueryProcessor.java:2011)
>                         at 
> org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing$RowDescriptor.columnValue(IgniteH2Indexing.java:2543)
>                         at 
> org.apache.ignite.internal.processors.query.h2.opt.GridH2AbstractKeyValueRow.getValue(GridH2AbstractKeyValueRow.java:289)
>                         at 
> org.apache.ignite.internal.processors.query.h2.opt.GridH2IndexBase.compareRows(GridH2IndexBase.java:116)
>                         at 
> org.apache.ignite.internal.processors.query.h2.opt.GridH2TreeIndex.compare(GridH2TreeIndex.java:248)
>                         at 
> org.apache.ignite.internal.processors.query.h2.opt.GridH2TreeIndex.compare(GridH2TreeIndex.java:49)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap$2.compareTo(GridOffHeapSnapTreeMap.java:1350)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap$2.compareTo(GridOffHeapSnapTreeMap.java:1346)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2102)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.attemptUpdate(GridOffHeapSnapTreeMap.java:2217)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.updateUnderRoot(GridOffHeapSnapTreeMap.java:2034)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.update(GridOffHeapSnapTreeMap.java:1915)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.remove(GridOffHeapSnapTreeMap.java:1884)
>                         at 
> org.apache.ignite.internal.util.offheap.unsafe.GridOffHeapSnapTreeMap.remove(GridOffHeapSnapTreeMap.java:108)
>                         at 
> org.apache.ignite.internal.processors.query.h2.opt.GridH2TreeIndex.remove(GridH2TreeIndex.java:408)
>                         at 
> org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.doUpdate(GridH2Table.java:433)
>                         at 
> org.apache.ignite.internal.processors.query.h2.opt.GridH2Table.update(GridH2Table.java:339)
>                         at 
> org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing.remove(IgniteH2Indexing.java:602)
>                         at 
> org.apache.ignite.internal.processors.query.GridQueryProcessor.remove(GridQueryProcessor.java:1014)
>                         at 
> org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager.remove(GridCacheQueryManager.java:434)
>                         at 
> org.apache.ignite.internal.processors.cache.GridCacheMapEntry.clearIndex(GridCacheMapEntry.java:4027)
>                         at 
> org.apache.ignite.internal.processors.cache.GridCacheMapEntry.innerUpdate(GridCacheMapEntry.java:2509)
>                         at 
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateSingle(GridDhtAtomicCache.java:2242)
>                         at 
> org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal0(GridDhtAtomicCache.java:1642)
>                         ... 17 more
> {noformat}
> Observation: the issue is reproduced when eviction is configured for cache 
> with indexed type.
> Load config:
> - benchmark: IgniteCacheRandomOperationBenchmark
> - preload entries: 5M
> - key range: 10M
> - servers: 20 (5 per host)
> - clients: 1
> - operations: PUT PUT_ALL GET GET_ALL INVOKE INVOKE_ALL REMOVE REMOVE_ALL 
> PUT_IF_ABSENT REPLACE
> - Xmx8g -Xms8g
> - GC used: G1
> - backups: 2
> - batch size: 50
> - thread count: 64
> - sync mode: PRIMARY_SYNC
> - cache config:
> {noformat}
>     <bean name="atomic" 
> class="org.apache.ignite.configuration.CacheConfiguration">
>         <property name="name" value="atomic"/>
>         <property name="cacheMode" value="PARTITIONED"/>
>         <property name="atomicityMode" value="ATOMIC"/>
>         <property name="swapEnabled" value="false"/>
>         <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
>         <property name="evictionPolicy">
>             <bean 
> class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
>                 <property name="maxSize" value="750000"/>
>             </bean>
>         </property>
>     </bean>
>     <bean name="atomic-offheap" 
> class="org.apache.ignite.configuration.CacheConfiguration">
>         <property name="name" value="atomic-offheap"/>
>         <property name="cacheMode" value="PARTITIONED"/>
>         <property name="atomicityMode" value="ATOMIC"/>
>         <property name="swapEnabled" value="false"/>
>         <property name="memoryMode" value="OFFHEAP_TIERED"/>
>     </bean>
>     <bean name="atomic-offheap-values" 
> class="org.apache.ignite.configuration.CacheConfiguration">
>         <property name="name" value="atomic-offheap-values"/>
>         <property name="cacheMode" value="PARTITIONED"/>
>         <property name="atomicityMode" value="ATOMIC"/>
>         <property name="swapEnabled" value="false"/>
>         <property name="memoryMode" value="OFFHEAP_VALUES"/>
>     </bean>
>     <bean name="tx" 
> class="org.apache.ignite.configuration.CacheConfiguration">
>         <property name="name" value="tx"/>
>         <property name="cacheMode" value="PARTITIONED"/>
>         <property name="atomicityMode" value="TRANSACTIONAL"/>
>         <property name="swapEnabled" value="false"/>
>         <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
>         <property name="evictionPolicy">
>             <bean 
> class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
>                 <property name="maxSize" value="750000"/>
>             </bean>
>         </property>
>     </bean>
>     <bean name="tx-offheap" 
> class="org.apache.ignite.configuration.CacheConfiguration">
>         <property name="name" value="tx-offheap"/>
>         <property name="cacheMode" value="PARTITIONED"/>
>         <property name="atomicityMode" value="TRANSACTIONAL"/>
>         <property name="swapEnabled" value="false"/>
>         <property name="memoryMode" value="OFFHEAP_TIERED"/>
>     </bean>
>     <bean name="tx-offheap-values" 
> class="org.apache.ignite.configuration.CacheConfiguration">
>         <property name="name" value="tx-offheap-values"/>
>         <property name="cacheMode" value="PARTITIONED"/>
>         <property name="atomicityMode" value="TRANSACTIONAL"/>
>         <property name="swapEnabled" value="false"/>
>         <property name="memoryMode" value="OFFHEAP_VALUES"/>
>     </bean>
>     <bean name="atomic-index" 
> class="org.apache.ignite.configuration.CacheConfiguration">
>         <property name="name" value="atomic-index"/>
>         <property name="cacheMode" value="PARTITIONED"/>
>         <property name="atomicityMode" value="ATOMIC"/>
>         <property name="swapEnabled" value="false"/>
>         <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
>         <property name="evictionPolicy">
>             <bean 
> class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
>                 <property name="maxSize" value="750000"/>
>             </bean>
>         </property>
>         <property name="indexedTypes">
>             <list>
>                 <value>java.lang.Integer</value>
>                 <value>org.apache.ignite.yardstick.cache.model.Person1</value>
>             </list>
>         </property>
>     </bean>
>     <bean name="query" 
> class="org.apache.ignite.configuration.CacheConfiguration">
>         <property name="name" value="query"/>
>         <property name="cacheMode" value="PARTITIONED"/>
>         <property name="atomicityMode" value="ATOMIC"/>
>         <property name="swapEnabled" value="false"/>
>         <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
>         <property name="evictionPolicy">
>             <bean 
> class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
>                 <property name="maxSize" value="750000"/>
>             </bean>
>         </property>
>         <!-- Configure query entities -->
>         <property name="queryEntities">
>             <list>
>                 <bean class="org.apache.ignite.cache.QueryEntity">
>                     <property name="keyType" value="java.lang.Integer"/>
>                     <property name="valueType" 
> value="org.apache.ignite.yardstick.cache.model.Person"/>
>                     <property name="fields">
>                         <map>
>                             <entry key="id" value="java.lang.Integer"/>
>                             <entry key="orgId" value="java.lang.Integer"/>
>                             <entry key="firstName" value="java.lang.String"/>
>                             <entry key="lastName" value="java.lang.String"/>
>                             <entry key="salary" value="java.lang.Double"/>
>                         </map>
>                     </property>
>                     <property name="indexes">
>                         <list>
>                             <bean class="org.apache.ignite.cache.QueryIndex">
>                                 <constructor-arg value="id"/>
>                             </bean>
>                             <!-- Group index. -->
>                             <bean class="org.apache.ignite.cache.QueryIndex">
>                                 <constructor-arg>
>                                     <list>
>                                         <value>id</value>
>                                         <value>salary</value>
>                                     </list>
>                                 </constructor-arg>
>                                 <constructor-arg value="SORTED"/>
>                             </bean>
>                         </list>
>                     </property>
>                 </bean>
>             </list>
>         </property>
>     </bean>
>     <bean name="orgCache" 
> class="org.apache.ignite.configuration.CacheConfiguration">
>         <property name="name" value="orgCache"/>
>         <property name="cacheMode" value="PARTITIONED"/>
>         <property name="atomicityMode" value="ATOMIC"/>
>         <property name="swapEnabled" value="false"/>
>         <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
>         <property name="evictionPolicy">
>             <bean 
> class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
>                 <property name="maxSize" value="750000"/>
>             </bean>
>         </property>
>         <!-- Configure query entities -->
>         <property name="queryEntities">
>             <list>
>                 <bean class="org.apache.ignite.cache.QueryEntity">
>                     <property name="keyType" value="java.lang.Integer"/>
>                     <property name="valueType" 
> value="org.apache.ignite.yardstick.cache.model.Organization"/>
>                     <property name="fields">
>                         <map>
>                             <entry key="id" value="java.lang.Integer"/>
>                             <entry key="name" value="java.lang.String"/>
>                         </map>
>                     </property>
>                     <property name="indexes">
>                         <list>
>                             <bean class="org.apache.ignite.cache.QueryIndex">
>                                 <constructor-arg value="id"/>
>                             </bean>
>                             <bean class="org.apache.ignite.cache.QueryIndex">
>                                 <constructor-arg value="name"/>
>                             </bean>
>                         </list>
>                     </property>
>                 </bean>
>             </list>
>         </property>
>     </bean>
>     <bean name="compute" 
> class="org.apache.ignite.configuration.CacheConfiguration">
>         <property name="name" value="compute"/>
>         <property name="cacheMode" value="PARTITIONED"/>
>         <property name="atomicityMode" value="TRANSACTIONAL"/>
>         <property name="swapEnabled" value="false"/>
>         <property name="offHeapMaxMemory" value="#{1024L*1024L*1024L*10L}"/>
>         <property name="evictionPolicy">
>             <bean 
> class="org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy">
>                 <property name="maxSize" value="750000"/>
>             </bean>
>         </property>
>     </bean>
> {noformat}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

Reply via email to