[ignite] branch ignite-11704 updated: ignite-11704
This is an automated email from the ASF dual-hosted git repository. sboikov pushed a commit to branch ignite-11704 in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/ignite-11704 by this push: new 1da9dc3 ignite-11704 1da9dc3 is described below commit 1da9dc31b51718f595334a32e60e11f571676dd5 Author: sboikov AuthorDate: Thu Aug 1 09:43:33 2019 +0300 ignite-11704 --- .../ignite/internal/processors/cache/CacheGroupContext.java | 2 +- .../apache/ignite/internal/processors/cache/GridCacheContext.java | 8 ++-- .../ignite/internal/processors/cache/GridCacheMapEntry.java | 3 +++ .../internal/processors/cache/IgniteCacheOffheapManagerImpl.java | 2 ++ .../cache/distributed/dht/topology/GridDhtLocalPartition.java | 2 -- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java index 3a7a445..c2cba6b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java @@ -1296,7 +1296,7 @@ public class CacheGroupContext { * @return {@code True} if need create temporary tombstones entries for removed data. */ public boolean supportsTombstone() { -return !sharedGroup() && !hasAtomicCaches && !mvccEnabled && !isLocal(); +return !mvccEnabled && !isLocal(); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java index 959095c..4c51981 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java @@ -625,8 +625,12 @@ public class GridCacheContext implements Externalizable { public void cache(GridCacheAdapter cache) { this.cache = cache; -deferredDel = !grp.supportsTombstone() && (cache.isDht() || cache.isDhtAtomic() || cache.isColocated() || -(cache.isNear() && cache.configuration().getAtomicityMode() == ATOMIC)); +if (grp.supportsTombstone() && cache.configuration().getAtomicityMode() == TRANSACTIONAL) +deferredDel = false; +else { +deferredDel = (cache.isDht() || cache.isDhtAtomic() || cache.isColocated() || +(cache.isNear() && cache.configuration().getAtomicityMode() == ATOMIC)); +} } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index 1133085..db066b6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -4505,6 +4505,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme CacheDataRow row = cctx.offheap().read(this); +if (cctx.offheap().isTombstone(row)) +return; + if (row != null && (filter == null || filter.apply(row))) clo.apply(row); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index 1f3a22d..9d15358 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -628,6 +628,8 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager GridCacheVersion ver, GridDhtLocalPartition part) throws IgniteCheckedException { assert part != null; +assert !cctx.isNear(); +assert !cctx.isLocal(); dataStore(part).removeWithTombstone(cctx, key, ver, part); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index d9b34c6..f098e99 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++
[ignite] branch ignite-11704 updated: ignite-11704
This is an automated email from the ASF dual-hosted git repository. sboikov pushed a commit to branch ignite-11704 in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/ignite-11704 by this push: new a6cd27d ignite-11704 a6cd27d is described below commit a6cd27dbd0ec5a7e9d82d4904f6b30c2cb5d7b6e Author: sboikov AuthorDate: Wed Jul 31 10:58:06 2019 +0300 ignite-11704 --- .../CacheRemoveWithTombstonesLoadTest.java| 19 ++- .../ignite/testframework/MvccFeatureChecker.java | 3 +-- .../ignite/testsuites/IgniteCacheMvccTestSuite9.java | 4 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java index cb144b9..6b0aefd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java @@ -33,6 +33,7 @@ import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Assume; import org.junit.Test; import java.util.ArrayList; @@ -80,7 +81,7 @@ public class CacheRemoveWithTombstonesLoadTest extends GridCommonAbstractTest { @Override protected void beforeTestsStarted() throws Exception { super.beforeTestsStarted(); - MvccFeatureChecker.skipIfNotSupported(MvccFeatureChecker.Feature.TOMBSTONES); +Assume.assumeFalse(MvccFeatureChecker.forcedMvcc()); } /** {@inheritDoc} */ @@ -138,8 +139,8 @@ public class CacheRemoveWithTombstonesLoadTest extends GridCommonAbstractTest { Map data = new HashMap<>(); -final int KEYS = 10_000; -final int ADD_NODES = 3; +final int KEYS = persistence ? 5_000 : 10_000; +final int ADD_NODES = persistence ? 2 : 3; for (int i = 0; i < KEYS; i++) { TestKey key = new TestKey(i, new byte[rnd.nextInt(pageSize * 3)]); @@ -164,7 +165,7 @@ public class CacheRemoveWithTombstonesLoadTest extends GridCommonAbstractTest { } }); -long endTime = System.currentTimeMillis() + 5000; +long endTime = System.currentTimeMillis() + 2500; while (System.currentTimeMillis() < endTime) { for (int i = 0; i < 100; i++) { @@ -180,6 +181,8 @@ public class CacheRemoveWithTombstonesLoadTest extends GridCommonAbstractTest { cache0.put(key, val); data.put(key, val); } + +Thread.sleep(10); } } @@ -209,7 +212,7 @@ public class CacheRemoveWithTombstonesLoadTest extends GridCommonAbstractTest { } }); -long endTime = System.currentTimeMillis() + 5000; +long endTime = System.currentTimeMillis() + 2500; while (System.currentTimeMillis() < endTime) { for (int i = 0; i < 100; i++) { @@ -226,6 +229,8 @@ public class CacheRemoveWithTombstonesLoadTest extends GridCommonAbstractTest { data.put(key, val); } } + +Thread.sleep(10); } fut.get(30_000); @@ -238,6 +243,10 @@ public class CacheRemoveWithTombstonesLoadTest extends GridCommonAbstractTest { } } +/** + * @param keys Keys to check. + * @param data Expected data. + */ private void checkData(List keys, Map data) { for (Ignite node : Ignition.allGrids()) { if (!node.name().endsWith("CacheRemoveWithTombstonesLoadTest1")) diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java index dce8c8d..1daeab4 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java @@ -47,8 +47,7 @@ public class MvccFeatureChecker { EVICTION, EXPIRATION, METRICS, -INTERCEPTOR, -TOMBSTONES +INTERCEPTOR } /** diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite9.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccTestSuite9.java index 243fb4b..e5dff20 100644 ---
[ignite] branch ignite-11704 updated: ignite-11704
This is an automated email from the ASF dual-hosted git repository. sboikov pushed a commit to branch ignite-11704 in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/ignite-11704 by this push: new 901661a ignite-11704 901661a is described below commit 901661ad40815aaeaf25470c67e575fade0c9ade Author: sboikov AuthorDate: Tue Jul 30 10:57:47 2019 +0300 ignite-11704 --- .../internal/processors/cache/IgniteCacheOffheapManager.java | 11 ++- .../processors/cache/IgniteCacheOffheapManagerImpl.java | 10 +- .../processors/cache/persistence/GridCacheOffheapManager.java | 8 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index aca75b6..8b6b623 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -28,6 +28,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.topology.Grid import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot; import org.apache.ignite.internal.processors.cache.mvcc.MvccVersion; import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow; +import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter; import org.apache.ignite.internal.processors.cache.persistence.CacheSearchRow; import org.apache.ignite.internal.processors.cache.persistence.RootPage; import org.apache.ignite.internal.processors.cache.persistence.RowStore; @@ -215,7 +216,7 @@ public interface IgniteCacheOffheapManager { * @return Iterator over all versions. * @throws IgniteCheckedException If failed. */ -GridCursor mvccAllVersionsCursor(GridCacheContext cctx, KeyCacheObject key, Object x) +GridCursor mvccAllVersionsCursor(GridCacheContext cctx, KeyCacheObject key, CacheDataRowAdapter.RowData x) throws IgniteCheckedException; /** @@ -945,7 +946,7 @@ public interface IgniteCacheOffheapManager { * @return Iterator over all versions. * @throws IgniteCheckedException If failed. */ -GridCursor mvccAllVersionsCursor(GridCacheContext cctx, KeyCacheObject key, Object x) +GridCursor mvccAllVersionsCursor(GridCacheContext cctx, KeyCacheObject key, CacheDataRowAdapter.RowData x) throws IgniteCheckedException; /** @@ -980,7 +981,7 @@ public interface IgniteCacheOffheapManager { * @return Data cursor. * @throws IgniteCheckedException If failed. */ -public GridCursor cursor(Object x) throws IgniteCheckedException; +public GridCursor cursor(CacheDataRowAdapter.RowData x) throws IgniteCheckedException; /** * @param mvccSnapshot MVCC snapshot. @@ -1025,7 +1026,7 @@ public interface IgniteCacheOffheapManager { * @throws IgniteCheckedException If failed. */ public GridCursor cursor(int cacheId, KeyCacheObject lower, -KeyCacheObject upper, Object x) throws IgniteCheckedException; +KeyCacheObject upper, CacheDataRowAdapter.RowData x) throws IgniteCheckedException; /** * @param cacheId Cache ID. @@ -1040,7 +1041,7 @@ public interface IgniteCacheOffheapManager { public GridCursor cursor(int cacheId, KeyCacheObject lower, KeyCacheObject upper, -Object x, +CacheDataRowAdapter.RowData x, MvccSnapshot snapshot, boolean withTombstones) throws IgniteCheckedException; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java index 8aa5514..1f3a22d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java @@ -687,7 +687,7 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager /** {@inheritDoc} */ @Override public GridCursor mvccAllVersionsCursor(GridCacheContext cctx, -KeyCacheObject key, Object x) throws IgniteCheckedException { +KeyCacheObject key, CacheDataRowAdapter.RowData x) throws IgniteCheckedException { CacheDataStore dataStore = dataStore(cctx, key); return dataStore != null ? dataStore.mvccAllVersionsCursor(cctx, key, x) : EMPTY_CURSOR; @@ -2889,7 +2889,7 @@ public class IgniteCacheOffheapManagerImpl implements
[ignite] branch ignite-11704 updated: ignite-11704
This is an automated email from the ASF dual-hosted git repository. sboikov pushed a commit to branch ignite-11704 in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/ignite-11704 by this push: new 7d3fcdc ignite-11704 7d3fcdc is described below commit 7d3fcdc3b6b3235f69c39074afa3189776865402 Author: sboikov AuthorDate: Tue Jul 30 10:54:01 2019 +0300 ignite-11704 --- .../ignite/internal/processors/query/h2/database/H2PkHashIndex.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java index d3bff29..db85a7b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java @@ -209,7 +209,7 @@ public class H2PkHashIndex extends GridH2IndexBase { int part = store.partId(); if (partsFilter == null || partsFilter.applyPartition(part)) -cursors.add(store.cursor(cctx.cacheId())); +cursors.add(store.cursor(cctx.cacheId(), false)); } Cursor pkHashCursor = new H2PkHashIndexCursor(cursors.iterator());
[ignite] branch ignite-11704 updated: ignite-11704
This is an automated email from the ASF dual-hosted git repository. sboikov pushed a commit to branch ignite-11704 in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/ignite-11704 by this push: new ea425b4 ignite-11704 ea425b4 is described below commit ea425b4306702abf27dd84e1370a4bc273df874a Author: sboikov AuthorDate: Tue Jul 30 09:43:39 2019 +0300 ignite-11704 --- .../internal/processors/cache/GridCacheAbstractFullApiSelfTest.java | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java index 7a823d4..ef7463e 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheAbstractFullApiSelfTest.java @@ -6728,7 +6728,10 @@ public abstract class GridCacheAbstractFullApiSelfTest extends GridCacheAbstract GridCacheContext ctx = ((IgniteKernal)ignite).internalCache(DEFAULT_CACHE_NAME).context(); -GridCacheEntryEx entry = ctx.isNear() ? ctx.near().dht().peekEx(key) : ctx.cache().peekEx(key); +if (ctx.isNear()) +ctx = ctx.near().dht().context(); + +GridCacheEntryEx entry = ctx.cache().peekEx(key); if (ctx.deferredDelete() && ignite.affinity(DEFAULT_CACHE_NAME).mapKeyToPrimaryAndBackups(key).contains(((IgniteKernal)ignite).localNode())) { assertNotNull(entry);
[ignite] branch ignite-11704 updated: ignite-11704
This is an automated email from the ASF dual-hosted git repository. sboikov pushed a commit to branch ignite-11704 in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/ignite-11704 by this push: new 8035686 ignite-11704 8035686 is described below commit 8035686072bbe34af3c60e71d3c454def9cab0c5 Author: sboikov AuthorDate: Tue Jul 30 08:06:15 2019 +0300 ignite-11704 --- .../processors/cache/distributed/CacheRemoveWithTombstonesTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java index b098905..daab924 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java @@ -49,7 +49,6 @@ import java.util.concurrent.Callable; import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; -import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT; import static org.apache.ignite.cache.CacheMode.PARTITIONED; import static org.apache.ignite.cache.CacheRebalanceMode.ASYNC; import static org.apache.ignite.internal.processors.metric.impl.MetricUtils.cacheMetricsRegistryName;
[ignite] branch ignite-11704 updated: ignite-11704
This is an automated email from the ASF dual-hosted git repository. sboikov pushed a commit to branch ignite-11704 in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/ignite-11704 by this push: new 5010c38 ignite-11704 5010c38 is described below commit 5010c38efd35d6a900159a85b82360f99f24edb8 Author: sboikov AuthorDate: Tue Jul 30 08:05:09 2019 +0300 ignite-11704 --- .../internal/processors/cache/IgniteCacheOffheapManager.java | 10 +++--- .../cache/persistence/GridCacheDatabaseSharedManager.java | 2 -- .../cache/distributed/CacheRemoveWithTombstonesTest.java | 1 + 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java index e068059..aca75b6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java @@ -398,7 +398,7 @@ public interface IgniteCacheOffheapManager { /** * @param cctx Cache context. * @param key Key. - * @param partId Partition number. + * @param ver Version. * @param part Partition. * @throws IgniteCheckedException If failed. */ @@ -919,10 +919,14 @@ public interface IgniteCacheOffheapManager { * @param cctx Cache context. * @param key Key. * @param ver Version. - * @param partId Partition number. + * @param part Partition. * @throws IgniteCheckedException If failed. */ -public void removeWithTombstone(GridCacheContext cctx, KeyCacheObject key, GridCacheVersion ver, GridDhtLocalPartition part) throws IgniteCheckedException; +public void removeWithTombstone( +GridCacheContext cctx, +KeyCacheObject key, +GridCacheVersion ver, +GridDhtLocalPartition part) throws IgniteCheckedException; /** * @param cctx Cache context. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index 3fd36a0..b1a21de 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -121,8 +121,6 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRec import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor; -import org.apache.ignite.internal.processors.cache.CacheObject; -import org.apache.ignite.internal.processors.cache.CacheObjectImpl; import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; import org.apache.ignite.internal.processors.cache.ExchangeActions; import org.apache.ignite.internal.processors.cache.GridCacheContext; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java index cb4ad69..b098905 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java @@ -238,6 +238,7 @@ public class CacheRemoveWithTombstonesTest extends GridCommonAbstractTest { assertEquals(0, tombstoneMetric1.get()); } + /** * */
[ignite] branch ignite-11704 updated: ignite-11704
This is an automated email from the ASF dual-hosted git repository. sboikov pushed a commit to branch ignite-11704 in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/ignite-11704 by this push: new 10d24bd ignite-11704 10d24bd is described below commit 10d24bdc128ee7472e4d5a320086643ccb88b3f5 Author: sboikov AuthorDate: Tue Jul 30 07:29:26 2019 +0300 ignite-11704 --- .../CacheRemoveWithTombstonesLoadTest.java | 8 .../distributed/CacheRemoveWithTombstonesTest.java | 22 -- .../ignite/testframework/MvccFeatureChecker.java | 3 ++- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java index 2780132..cb144b9 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java @@ -31,6 +31,7 @@ import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.util.lang.GridAbsPredicate; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; @@ -76,6 +77,13 @@ public class CacheRemoveWithTombstonesLoadTest extends GridCommonAbstractTest { } /** {@inheritDoc} */ +@Override protected void beforeTestsStarted() throws Exception { +super.beforeTestsStarted(); + + MvccFeatureChecker.skipIfNotSupported(MvccFeatureChecker.Feature.TOMBSTONES); +} + +/** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { super.beforeTest(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java index dcd7f6c..cb4ad69 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesTest.java @@ -37,6 +37,7 @@ import org.apache.ignite.lang.IgniteBiPredicate; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.metric.LongMetric; import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.MvccFeatureChecker; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; @@ -108,14 +109,6 @@ public class CacheRemoveWithTombstonesTest extends GridCommonAbstractTest { * @throws Exception If failed. */ @Test -public void testRemoveAndRebalanceRaceTxMvcc() throws Exception { -testRemoveAndRebalanceRace(TRANSACTIONAL_SNAPSHOT, false); -} - -/** - * @throws Exception If failed. - */ -@Test public void testRemoveAndRebalanceRaceAtomic() throws Exception { testRemoveAndRebalanceRace(ATOMIC, false); } @@ -134,16 +127,6 @@ public class CacheRemoveWithTombstonesTest extends GridCommonAbstractTest { * @throws Exception If failed. */ @Test -public void testRemoveAndRebalanceRaceTxMvccWithPersistence() throws Exception { -persistence = true; - -testRemoveAndRebalanceRaceTxMvcc(); -} - -/** - * @throws Exception If failed. - */ -@Test public void testRemoveAndRebalanceRaceAtomicWithPersistence() throws Exception { persistence = true; @@ -162,6 +145,9 @@ public class CacheRemoveWithTombstonesTest extends GridCommonAbstractTest { IgniteCache cache0 = ignite0.createCache(cacheConfiguration(atomicityMode)); +if (MvccFeatureChecker.forcedMvcc()) +expTombstone = false; + LongMetric tombstoneMetric0 = ignite0.context().metric().registry( cacheMetricsRegistryName(DEFAULT_CACHE_NAME, false)).findMetric("Tombstones"); diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java index 1daeab4..dce8c8d 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/MvccFeatureChecker.java @@ -47,7 +47,8 @@ public class MvccFeatureChecker { EVICTION, EXPIRATION,
[ignite] branch ignite-11704 updated: ignite-11704
This is an automated email from the ASF dual-hosted git repository. sboikov pushed a commit to branch ignite-11704 in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/ignite-11704 by this push: new 8294495 ignite-11704 8294495 is described below commit 829449525fffdc4e0bac14c6b1905abe164f96d0 Author: sboikov AuthorDate: Fri Jul 26 11:13:19 2019 +0300 ignite-11704 --- .../dht/topology/GridDhtLocalPartition.java| 6 +- .../dht/topology/PartitionsEvictManager.java | 344 + .../IgniteCacheDatabaseSharedManager.java | 22 +- .../distributed/CacheRemoveWithTombstonesTest.java | 12 +- 4 files changed, 300 insertions(+), 84 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java index d24cfee..9097463 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/GridDhtLocalPartition.java @@ -1138,13 +1138,13 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements */ private void submitClearTombstones() { if (tombstoneCreated) - grp.shared().kernalContext().closure().runLocalSafe(this::clearTombstones, true); +grp.shared().evict().clearTombstonesAsync(grp, this); } /** * */ -private void clearTombstones() { +public void clearTombstones(EvictionContext evictionCtx) { final int stopCheckingFreq = 1000; CacheMapHolder hld = grp.sharedGroup() ? null : singleCacheEntryMap; @@ -1202,7 +1202,7 @@ public class GridDhtLocalPartition extends GridCacheConcurrentMapImpl implements cntr++; if (cntr % stopCheckingFreq == 0) { -if (ctx.kernalContext().isStopping() || state() != OWNING) +if (evictionCtx.shouldStop() || state() != OWNING) break; } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java index 826902c..b2fd276 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/topology/PartitionsEvictManager.java @@ -21,6 +21,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Queue; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -104,6 +105,14 @@ public class PartitionsEvictManager extends GridCacheSharedManagerAdapter { } } +public void clearTombstonesAsync(CacheGroupContext grp, GridDhtLocalPartition part) { +if (addAsyncTask(grp, part, TaskType.CLEAR_TOMBSTONES)) { +if (log.isDebugEnabled()) +log.debug("Partition has been scheduled for tomstones cleanup [grp=" + grp.cacheOrGroupName() ++ ", p=" + part.id() + ", state=" + part.state() + "]"); +} +} + /** * Adds partition to eviction queue and starts eviction process if permit available. * @@ -111,29 +120,56 @@ public class PartitionsEvictManager extends GridCacheSharedManagerAdapter { * @param part Partition to evict. */ public void evictPartitionAsync(CacheGroupContext grp, GridDhtLocalPartition part) { +if (addAsyncTask(grp, part, TaskType.EVICT)) { +if (log.isDebugEnabled()) +log.debug("Partition has been scheduled for eviction [grp=" + grp.cacheOrGroupName() ++ ", p=" + part.id() + ", state=" + part.state() + "]"); +} +} + +/** + * @param grp Group context. + * @param part Partition. + * @param type Task type. + * @return {@code True} if task was added. + */ +private boolean addAsyncTask(CacheGroupContext grp, GridDhtLocalPartition part, TaskType type) { GroupEvictionContext grpEvictionCtx = evictionGroupsMap.computeIfAbsent( grp.groupId(), (k) -> new GroupEvictionContext(grp)); // Check node stop. if (grpEvictionCtx.shouldStop()) -return; +return false; int bucket; +AbstractEvictionTask task; + +switch (type) { +case EVICT: +task = new
[ignite] branch ignite-11704 updated: ignite-11704
This is an automated email from the ASF dual-hosted git repository. sboikov pushed a commit to branch ignite-11704 in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/ignite-11704 by this push: new db16252 ignite-11704 db16252 is described below commit db1625276b61cba1b87d287a133b979031728c11 Author: sboikov AuthorDate: Wed Jul 24 09:58:40 2019 +0300 ignite-11704 --- .../cache/persistence/CacheDataRowAdapter.java | 58 ++ .../IgniteCacheDatabaseSharedManager.java | 11 +++- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java index 5216e21..c0a421e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java @@ -262,33 +262,8 @@ public class CacheDataRowAdapter implements CacheDataRow { incomplete = readIncomplete(incomplete, sharedCtx, coctx, pageMem, grpId, pageAddr, itemId, io, rowData, readCacheId, skipVer); -if (incomplete == null) { -if (rowData == TOMBSTONES && val != null && !sharedCtx.database().isTombstone(this)) { -// TODO IGNITE-11704. -ver = null; -key = null; -val = null; -verReady = true; -} - +if (incomplete == null || (rowData == KEY_ONLY && key != null)) return; -} - -if (rowData == KEY_ONLY) { -if (key != null) -return; -} -else if (rowData == TOMBSTONES) { -// TODO IGNITE-11704. -if (val != null && !sharedCtx.database().isTombstone(this)) { -ver = null; -key = null; -val = null; -verReady = true; - -return; -} -} nextLink = incomplete.getNextLink(); } @@ -377,9 +352,7 @@ public class CacheDataRowAdapter implements CacheDataRow { buf.position(off); buf.limit(off + payloadSize); -boolean keyOnly = rowData == RowData.KEY_ONLY; - -incomplete = readFragment(sharedCtx, coctx, buf, keyOnly, readCacheId, incomplete, skipVer); +incomplete = readFragment(sharedCtx, coctx, buf, rowData, readCacheId, incomplete, skipVer); if (incomplete != null) incomplete.setNextLink(nextLink); @@ -416,11 +389,13 @@ public class CacheDataRowAdapter implements CacheDataRow { GridCacheSharedContext sharedCtx, CacheObjectContext coctx, ByteBuffer buf, -boolean keyOnly, +RowData rowData, boolean readCacheId, IncompleteObject incomplete, boolean skipVer ) throws IgniteCheckedException { +boolean tombstones = rowData == TOMBSTONES; + if (readCacheId && cacheId == 0) { incomplete = readIncompleteCacheId(buf, incomplete); @@ -442,6 +417,12 @@ public class CacheDataRowAdapter implements CacheDataRow { // Read key. if (key == null) { +if (tombstones && sharedCtx.database().isTombstone(buf, key, (IncompleteCacheObject)incomplete) == Boolean.FALSE) { +verReady = true; + +return null; +} + incomplete = readIncompleteKey(coctx, buf, (IncompleteCacheObject)incomplete); if (key == null) { @@ -449,7 +430,7 @@ public class CacheDataRowAdapter implements CacheDataRow { return incomplete; // Need to finish reading the key. } -if (keyOnly) +if (rowData == RowData.KEY_ONLY) return null; // Key is ready - we are done! incomplete = null; @@ -468,6 +449,13 @@ public class CacheDataRowAdapter implements CacheDataRow { // Read value. if (val == null) { +if (tombstones && sharedCtx.database().isTombstone(buf, key, (IncompleteCacheObject)incomplete) == Boolean.FALSE) { +key = null; +verReady = true; + +return null; +} + incomplete =
[ignite] branch ignite-11704 updated: ignite-11704
This is an automated email from the ASF dual-hosted git repository. sboikov pushed a commit to branch ignite-11704 in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/ignite-11704 by this push: new 0a574a6 ignite-11704 0a574a6 is described below commit 0a574a68385cf250b928972654b0486e2efe67a4 Author: sboikov AuthorDate: Tue Jul 23 10:42:28 2019 +0300 ignite-11704 --- .../cache/persistence/CacheDataRowAdapter.java | 13 +- .../CacheRemoveWithTombstonesLoadTest.java | 347 + .../ignite/testsuites/IgniteCacheTestSuite9.java | 2 + 3 files changed, 360 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java index 1be2bb5..5216e21 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java @@ -260,10 +260,19 @@ public class CacheDataRowAdapter implements CacheDataRow { int itemId = itemId(nextLink); incomplete = readIncomplete(incomplete, sharedCtx, coctx, pageMem, -grpId, pageAddr, itemId, io, rowData, readCacheId, skipVer); +grpId, pageAddr, itemId, io, rowData, readCacheId, skipVer); + +if (incomplete == null) { +if (rowData == TOMBSTONES && val != null && !sharedCtx.database().isTombstone(this)) { +// TODO IGNITE-11704. +ver = null; +key = null; +val = null; +verReady = true; +} -if (incomplete == null) return; +} if (rowData == KEY_ONLY) { if (key != null) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java new file mode 100644 index 000..2780132 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheRemoveWithTombstonesLoadTest.java @@ -0,0 +1,347 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed; + +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.util.lang.GridAbsPredicate; +import org.apache.ignite.spi.metric.LongMetric; +import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.cache.CacheRebalanceMode.ASYNC; +import static
[ignite] branch ignite-11704 updated: ignite-11704
This is an automated email from the ASF dual-hosted git repository. sboikov pushed a commit to branch ignite-11704 in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/ignite-11704 by this push: new e6a793b ignite-11704 e6a793b is described below commit e6a793bf4caf897453f40ad4df977ea7268dea4c Author: sboikov AuthorDate: Fri Jul 19 21:20:51 2019 +0300 ignite-11704 --- .../processors/cache/CacheGroupContext.java| 3 +- .../processors/cache/GridCacheMapEntry.java| 88 ++- .../cache/IgniteCacheOffheapManager.java | 5 +- .../cache/IgniteCacheOffheapManagerImpl.java | 26 +- .../dht/topology/GridDhtLocalPartition.java| 99 +- .../cache/persistence/GridCacheOffheapManager.java | 8 +- .../distributed/CacheRemoveWithTombstonesTest.java | 39 +++-- 7 files changed, 250 insertions(+), 18 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java index 4af5de5..7963893 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java @@ -47,6 +47,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffini import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopology; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionTopologyImpl; import org.apache.ignite.internal.processors.cache.persistence.DataRegion; @@ -1307,7 +1308,7 @@ public class CacheGroupContext { } public boolean createTombstone(@Nullable GridDhtLocalPartition part) { -return part != null && supportsTombstone(); +return part != null && supportsTombstone() && part.state() == GridDhtPartitionState.MOVING; } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java index adc8699..08986a9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java @@ -1717,8 +1717,12 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme } } -if (cctx.group().createTombstone(localPartition())) -cctx.offheap().removeWithTombstone(cctx, key, newVer, partition(), localPartition()); +if (cctx.group().createTombstone(localPartition())) { +cctx.offheap().removeWithTombstone(cctx, key, newVer, localPartition()); + +if (!cctx.group().createTombstone(localPartition())) +removeTombstone0(newVer); +} else removeValue(); @@ -2818,6 +2822,34 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme } /** + * @param tombstoneVer Tombstone version. + * @throws GridCacheEntryRemovedException If entry was removed. + * @throws IgniteCheckedException If failed. + */ +public void removeTombstone(GridCacheVersion tombstoneVer) throws GridCacheEntryRemovedException, IgniteCheckedException { +lockEntry(); + +try { +checkObsolete(); + +removeTombstone0(tombstoneVer); +} +finally { +unlockEntry(); +} +} + +/** + * @param tombstoneVer Tombstone version. + * @throws IgniteCheckedException If failed. + */ +private void removeTombstone0(GridCacheVersion tombstoneVer) throws IgniteCheckedException { +RemoveClosure closure = new RemoveClosure(this, tombstoneVer); + +cctx.offheap().invoke(cctx, key, localPartition(), closure); +} + +/** * @return {@code True} if this entry should not be evicted from cache. */ protected boolean evictionDisabled() { @@ -5720,6 +5752,58 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme /** * */ +private static class RemoveClosure implements IgniteCacheOffheapManager.OffheapInvokeClosure { +/** */ +private final