This is an automated email from the ASF dual-hosted git repository. sergeychugunov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push: new 29cd623 IGNITE-14469 Additional tests for control utility command indexes_force_rebuild - Fixes #9061. 29cd623 is described below commit 29cd6233e6d4f058e4714d36d5d04d9ed4f5a8c6 Author: Eduard Rakhmankulov <erixon...@gmail.com> AuthorDate: Thu May 13 09:45:35 2021 +0300 IGNITE-14469 Additional tests for control utility command indexes_force_rebuild - Fixes #9061. Signed-off-by: Sergey Chugunov <sergey.chugu...@gmail.com> --- .../cache/CacheIndexesForceRebuild.java | 9 +- .../GridCommandHandlerIndexForceRebuildTest.java | 172 ++++++++++++++++++--- .../processors/cache/GridCacheProcessor.java | 3 +- .../visor/cache/index/IndexForceRebuildTask.java | 135 +++++----------- 4 files changed, 195 insertions(+), 124 deletions(-) diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/cache/CacheIndexesForceRebuild.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/cache/CacheIndexesForceRebuild.java index 8901277..91c2fb5 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/cache/CacheIndexesForceRebuild.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/cache/CacheIndexesForceRebuild.java @@ -84,8 +84,13 @@ public class CacheIndexesForceRebuild extends AbstractCommand<CacheIndexesForceR final UUID nodeId = args.nodeId; try (GridClient client = Command.startClient(clientCfg)) { - taskRes = TaskExecutor.executeTaskByNameOnNode(client, IndexForceRebuildTask.class.getName(), taskArg, - nodeId, clientCfg); + taskRes = TaskExecutor.executeTaskByNameOnNode( + client, + IndexForceRebuildTask.class.getName(), + taskArg, + nodeId, + clientCfg + ); } printResult(taskRes, logger); diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexForceRebuildTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexForceRebuildTest.java index 5cad551..80de9ab 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexForceRebuildTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexForceRebuildTest.java @@ -40,6 +40,8 @@ import org.apache.ignite.internal.processors.query.schema.SchemaIndexOperationCa import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.SB; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.ListeningTestLogger; import org.apache.ignite.testframework.LogListener; @@ -51,6 +53,7 @@ import org.junit.Test; import static java.lang.String.valueOf; import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_INVALID_ARGUMENTS; import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK; +import static org.apache.ignite.internal.commandline.CommandLogger.INDENT; import static org.apache.ignite.testframework.GridTestUtils.assertContains; import static org.apache.ignite.testframework.GridTestUtils.getFieldValue; import static org.apache.ignite.testframework.GridTestUtils.runAsync; @@ -228,7 +231,15 @@ public class GridCommandHandlerIndexForceRebuildTest extends GridCommandHandlerA waitForIndexesRebuild(grid(LAST_NODE_NUM)); - validateTestCacheNamesArgOutput(); + String outputStr = testOut.toString(); + + validateOutputCacheNamesNotFound(outputStr, CACHE_NAME_NON_EXISTING); + + validateOutputIndicesRebuildingInProgress(outputStr, F.asMap(GRP_NAME_2, F.asList(CACHE_NAME_2_1))); + + validateOutputIndicesRebuildWasStarted(outputStr, F.asMap(GRP_NAME_1, F.asList(CACHE_NAME_1_1))); + + assertEquals("Unexpected number of lines in output.", 19, outputStr.split("\n").length); // Index rebuild must be triggered only for cache1_1 and only on node3. assertFalse(cache1Listeners[0].check()); @@ -279,7 +290,21 @@ public class GridCommandHandlerIndexForceRebuildTest extends GridCommandHandlerA waitForIndexesRebuild(grid(LAST_NODE_NUM)); - validateTestCacheGroupArgOutput(); + String outputStr = testOut.toString(); + + validateOutputCacheGroupsNotFound(outputStr, GRP_NAME_NON_EXISTING); + + validateOutputIndicesRebuildingInProgress(outputStr, F.asMap(GRP_NAME_1, F.asList(CACHE_NAME_1_2))); + + validateOutputIndicesRebuildWasStarted( + outputStr, + F.asMap( + GRP_NAME_1, F.asList(CACHE_NAME_1_1), + GRP_NAME_2, F.asList(CACHE_NAME_2_1) + ) + ); + + assertEquals("Unexpected number of lines in outputStr.", 20, outputStr.split("\n").length); assertFalse(cache1Listeners[0].check()); assertFalse(cache1Listeners[1].check()); @@ -476,40 +501,119 @@ public class GridCommandHandlerIndexForceRebuildTest extends GridCommandHandlerA } /** - * Validated control.sh utility output for {@link #testCacheNamesArg()}. + * Checking that a sequence of forced rebuild of indexes is possible + * + * @throws Exception If failed. */ - private void validateTestCacheNamesArgOutput() { - String outputStr = testOut.toString(); + @Test + public void testSequentialForceRebuildIndexes() throws Exception { + IgniteEx grid = grid(0); + + injectTestSystemOut(); + + String outputStr; - assertTrue(outputStr.contains("WARNING: These caches were not found:\n" + - " " + CACHE_NAME_NON_EXISTING)); + forceRebuildIndices(F.asList(CACHE_NAME_1_1), grid); - assertTrue(outputStr.contains("WARNING: These caches have indexes rebuilding in progress:\n" + - " groupName=" + GRP_NAME_2 + ", cacheName=" + CACHE_NAME_2_1)); + outputStr = testOut.toString(); - assertTrue(outputStr.contains("Indexes rebuild was started for these caches:\n" + - " groupName=" + GRP_NAME_1 + ", cacheName=" + CACHE_NAME_1_1)); + validateOutputIndicesRebuildWasStarted(outputStr, F.asMap(GRP_NAME_1, F.asList(CACHE_NAME_1_1))); + + assertFalse(outputStr.contains("WARNING: These caches have indexes rebuilding in progress:")); + + forceRebuildIndices(F.asList(CACHE_NAME_1_1), grid); + + validateOutputIndicesRebuildWasStarted(outputStr, F.asMap(GRP_NAME_1, F.asList(CACHE_NAME_1_1))); + + assertFalse(outputStr.contains("WARNING: These caches have indexes rebuilding in progress:")); + } + + /** + * Validates control.sh output when caches by name not found. + * + * @param outputStr CLI {@code control.sh} utility output. + * @param cacheNames Cache names to print. + */ + private void validateOutputCacheNamesNotFound(String outputStr, String... cacheNames) { + assertContains( + log, + outputStr, + "WARNING: These caches were not found:" + U.nl() + makeStringListWithIndent(cacheNames) + ); + } + + /** + * Validates control.sh output when caches by group not found. + * + * @param outputStr CLI {@code control.sh} utility output. + * @param cacheGrps Cache groups to print. + */ + private void validateOutputCacheGroupsNotFound(String outputStr, String... cacheGrps) { + assertContains( + log, + outputStr, + "WARNING: These cache groups were not found:" + U.nl() + makeStringListWithIndent(cacheGrps) + ); + } - assertEquals("Unexpected number of lines in output.", 19, outputStr.split("\n").length); + /** + * Makes new-line List with indent. + * @param strings List of strings. + * @return Formated text. + */ + private String makeStringListWithIndent(String... strings) { + return INDENT + String.join(U.nl() + INDENT, strings); } /** - * Validated control.sh utility output for {@link #testGroupNamesArg()}. + * Makes formatted text for given caches. + * + * @param cacheGroputToNames Cache groups mapping to non-existing cache names. + * @return Text for CLI print output for given caches. */ - private void validateTestCacheGroupArgOutput() { - String outputStr = testOut.toString(); + private String makeStringListForCacheGroupsAndNames(Map<String, List<String>> cacheGroputToNames) { + SB sb = new SB(); + + for (Map.Entry<String, List<String>> entry : cacheGroputToNames.entrySet()) { + String cacheGrp = entry.getKey(); - assertTrue(outputStr.contains("WARNING: These cache groups were not found:\n" + - " " + GRP_NAME_NON_EXISTING)); + for (String cacheName : entry.getValue()) + sb.a(INDENT).a("groupName=").a(cacheGrp).a(", cacheName=").a(cacheName).a(U.nl()); + } + + return sb.toString(); + } + + /** + * Validates control.sh output when some indices rebuilt in progress. + * + * @param outputStr CLI {@code control.sh} utility output. + * @param cacheGroputToNames Cache groups mapping to non-existing cache names. + */ + private void validateOutputIndicesRebuildingInProgress(String outputStr, Map<String, List<String>> cacheGroputToNames) { + String caches = makeStringListForCacheGroupsAndNames(cacheGroputToNames); - assertTrue(outputStr.contains("WARNING: These caches have indexes rebuilding in progress:\n" + - " groupName=" + GRP_NAME_1 + ", cacheName=" + CACHE_NAME_1_2)); + assertContains( + log, + outputStr, + "WARNING: These caches have indexes rebuilding in progress:" + U.nl() + caches + ); + } - assertTrue(outputStr.contains("Indexes rebuild was started for these caches:\n" + - " groupName=" + GRP_NAME_1 + ", cacheName=" + CACHE_NAME_1_1 + "\n" + - " groupName=" + GRP_NAME_2 + ", cacheName=" + CACHE_NAME_2_1)); + /** + * Validates control.sh output when indices started to rebuild. + * + * @param outputStr CLI {@code control.sh} utility output. + * @param cacheGroputToNames Cache groups mapping to non-existing cache names. + */ + private void validateOutputIndicesRebuildWasStarted(String outputStr, Map<String, List<String>> cacheGroputToNames) { + String caches = makeStringListForCacheGroupsAndNames(cacheGroputToNames); - assertEquals("Unexpected number of lines in output.", 20, outputStr.split("\n").length); + assertContains( + log, + outputStr, + "Indexes rebuild was started for these caches:" + U.nl() + caches + ); } /** @@ -657,4 +761,26 @@ public class GridCommandHandlerIndexForceRebuildTest extends GridCommandHandlerA return idxRebuildFuts.get(cacheId); } + + /** + * Force rebuilds indices for chosen caches, and waits until rebuild process is complete. + * + * @param cacheNames Cache names need indices to rebuild. + * @param grid Ignite node. + * @throws Exception If failed. + */ + private void forceRebuildIndices(Iterable<String> cacheNames, IgniteEx grid) throws Exception { + String cacheNamesArg = String.join(",", cacheNames); + + assertEquals( + EXIT_CODE_OK, + execute( + "--cache", "indexes_force_rebuild", + "--node-id", grid.localNode().id().toString(), + "--cache-names", cacheNamesArg + ) + ); + + waitForIndexesRebuild(grid, getTestTimeout(), Collections.emptyList()); + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 5acc048..8dc4a75 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -4310,7 +4310,7 @@ public class GridCacheProcessor extends GridProcessorAdapter { * @param <V> type of values. * @return Cache instance for given name. */ - public <K, V> IgniteInternalCache<K, V> cache(String name) { + public <K, V> @Nullable IgniteInternalCache<K, V> cache(String name) { assert name != null; if (log.isDebugEnabled()) @@ -4465,6 +4465,7 @@ public class GridCacheProcessor extends GridProcessorAdapter { * @param <K> type of keys. * @param <V> type of values. * @return Cache instance for given name. + * @throws IllegalArgumentException If cache not exists. */ public <K, V> IgniteInternalCache<K, V> publicCache(String name) { assert name != null; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/index/IndexForceRebuildTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/index/IndexForceRebuildTask.java index 23ad2d2..1b55ed3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/index/IndexForceRebuildTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/index/IndexForceRebuildTask.java @@ -17,22 +17,19 @@ package org.apache.ignite.internal.visor.cache.index; +import java.util.Collection; import java.util.HashSet; -import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteException; -import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.GridCacheContext; import org.apache.ignite.internal.processors.cache.GridCacheProcessor; -import org.apache.ignite.internal.processors.cache.IgniteCacheProxy; +import org.apache.ignite.internal.processors.cache.IgniteInternalCache; import org.apache.ignite.internal.processors.task.GridInternal; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.visor.VisorJob; import org.apache.ignite.internal.visor.VisorOneNodeTask; -import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; /** @@ -67,112 +64,54 @@ public class IndexForceRebuildTask extends VisorOneNodeTask<IndexForceRebuildTas @Override protected IndexForceRebuildTaskRes run(@Nullable IndexForceRebuildTaskArg arg) throws IgniteException { - //Either cacheNames or cacheGrps must be specified - assert (arg.cacheNames() == null) != (arg.cacheGrps() == null) : - "Either cacheNames or cacheGroups must be specified"; - - // Collect info about indexes being rebuilt. - Set<IndexRebuildStatusInfoContainer> rebuildIdxCaches = - ignite.context().cache().publicCaches() - .stream() - .filter(c -> !c.indexReadyFuture().isDone()) - .map(this::fromIgniteCache) - .collect(Collectors.toSet()); - - Set<String> rebuildIdxCachesNames = rebuildIdxCaches.stream() - .map(IndexRebuildStatusInfoContainer::cacheName) - .collect(Collectors.toSet()); - - if (arg.cacheNames() != null) - return rebuildByCacheNames(arg.cacheNames(), rebuildIdxCaches, rebuildIdxCachesNames); - else if (arg.cacheGrps() != null) - return rebuildByGroupNames(arg.cacheGrps(), rebuildIdxCaches, rebuildIdxCachesNames); - else { - assert false : "Neither cache names nor cache groups specified"; + assert (arg.cacheNames() == null) ^ (arg.cacheGrps() == null) : + "Either cacheNames or cacheGroups must be specified."; - return null; - } - } + Set<GridCacheContext> cachesToRebuild = new HashSet<>(); + Set<String> notFound = new HashSet<>(); - /** - * Triggers force rebuild of indexes in caches from {@code cacheNames}. - * - * @param cacheNames Set of cache names. - * @param rebuildIdxCaches Set of infos about cached which have indexes being rebuilt at the moment. - * @param rebuildIdxCachesNames Set of names of cached which have indexes being rebuilt at the moment. - * @return {@code IndexForceRebuildTaskRes} object. - */ - @NotNull private IndexForceRebuildTaskRes rebuildByCacheNames( - Set<String> cacheNames, - Set<IndexRebuildStatusInfoContainer> rebuildIdxCaches, - Set<String> rebuildIdxCachesNames) - { final GridCacheProcessor cacheProcessor = ignite.context().cache(); - // Collect info about not found caches. - Set<String> notFoundCaches = new HashSet<>(cacheNames); - notFoundCaches.removeIf(name -> cacheProcessor.cache(name) != null); - - Set<GridCacheContext> cacheContexts = - cacheProcessor.publicCaches() - .stream() - .filter(c -> !rebuildIdxCachesNames.contains(c.getName())) - .filter(c -> cacheNames.contains(c.getName())) - .map(IgniteCacheProxy::context) - .collect(Collectors.toSet()); - - // Collect info about started index rebuild. - Set<IndexRebuildStatusInfoContainer> cachesWithStartedRebuild = - cacheContexts.stream() - .map(c -> new IndexRebuildStatusInfoContainer(c.config())) - .collect(Collectors.toSet()); + if (arg.cacheNames() != null) { + for (String cacheName : arg.cacheNames()) { + IgniteInternalCache cache = cacheProcessor.cache(cacheName); - cacheProcessor.context().database().forceRebuildIndexes(cacheContexts); + if (cache != null) + cachesToRebuild.add(cache.context()); + else + notFound.add(cacheName); + } + } + else { + for (String cacheGrpName : arg.cacheGrps()) { + CacheGroupContext grpCtx = cacheProcessor.cacheGroup(CU.cacheId(cacheGrpName)); + + if (grpCtx != null) + cachesToRebuild.addAll(grpCtx.caches()); + else + notFound.add(cacheGrpName); + } + } - return new IndexForceRebuildTaskRes(cachesWithStartedRebuild, rebuildIdxCaches, notFoundCaches); - } + Collection<GridCacheContext> cachesCtxWithRebuildingInProgress = + ignite.context().cache().context().database().forceRebuildIndexes(cachesToRebuild); - /** - * Triggers force rebuild of indexes in all caches from {@code grpNames}. - * - * @param grpNames Set of cache groups names. - * @param rebuildIdxCaches Set of infos about cached which have indexes being rebuilt at the moment. - * @param rebuildIdxCachesNames Set of names of cached which have indexes being rebuilt at the moment. - * @return {@code IndexForceRebuildTaskRes} object. - */ - @NotNull private IndexForceRebuildTaskRes rebuildByGroupNames( - Set<String> grpNames, - Set<IndexRebuildStatusInfoContainer> rebuildIdxCaches, - Set<String> rebuildIdxCachesNames) - { - final GridCacheProcessor cacheProcessor = ignite.context().cache(); - - // Collect info about not found groups. - Set<String> notFoundGroups = new HashSet<>(grpNames); - notFoundGroups.removeIf(grpName -> cacheProcessor.cacheGroup(CU.cacheId(grpName)) != null); - - Set<GridCacheContext> cacheContexts = - cacheProcessor.cacheGroups() - .stream() - .filter(grpContext -> grpNames.contains(grpContext.name())) - .map(CacheGroupContext::caches) - .flatMap(List::stream) - .filter(c -> !rebuildIdxCachesNames.contains(c.name())) + Set<IndexRebuildStatusInfoContainer> cachesWithRebuildingInProgress = + cachesCtxWithRebuildingInProgress.stream() + .map(c -> new IndexRebuildStatusInfoContainer(c.config())) .collect(Collectors.toSet()); Set<IndexRebuildStatusInfoContainer> cachesWithStartedRebuild = - cacheContexts.stream() + cachesToRebuild.stream() .map(c -> new IndexRebuildStatusInfoContainer(c.config())) + .filter(c -> !cachesWithRebuildingInProgress.contains(c)) .collect(Collectors.toSet()); - cacheProcessor.context().database().forceRebuildIndexes(cacheContexts); - - return new IndexForceRebuildTaskRes(cachesWithStartedRebuild, rebuildIdxCaches, notFoundGroups); - } - - /** */ - private IndexRebuildStatusInfoContainer fromIgniteCache(IgniteCache c) { - return new IndexRebuildStatusInfoContainer((CacheConfiguration)c.getConfiguration(CacheConfiguration.class)); + return new IndexForceRebuildTaskRes( + cachesWithStartedRebuild, + cachesWithRebuildingInProgress, + notFound + ); } } }