This is an automated email from the ASF dual-hosted git repository. nizhikov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push: new b01635b4959 IGNITE-17160 Minor index-reader code improvements (#10091) b01635b4959 is described below commit b01635b4959986399852beb67ea5f0106b1dd2f2 Author: Nikolay <nizhi...@apache.org> AuthorDate: Tue Jun 14 16:52:19 2022 +0300 IGNITE-17160 Minor index-reader code improvements (#10091) --- .../commandline/argument/parser/CLIArgument.java | 10 +- .../argument/parser/CLIArgumentParser.java | 30 +-- .../commandline/indexreader/IgniteIndexReader.java | 223 +++++++++------------ .../commandline/indexreader/PageListsInfo.java | 5 +- .../commandline/indexreader/TreeTraversalInfo.java | 5 +- .../indexreader/TreeTraverseContext.java | 10 +- .../indexreader/IgniteIndexReaderTest.java | 37 ++-- .../internal/cache/query/index/IndexProcessor.java | 7 + 8 files changed, 151 insertions(+), 176 deletions(-) diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgument.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgument.java index 648ccda4261..80b91f36c47 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgument.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgument.java @@ -42,22 +42,22 @@ public class CLIArgument<T> { /** */ public static <T> CLIArgument<T> optionalArg(String name, String usage, Class<T> type) { - return new CLIArgument<T>(name, usage, true, type, null); + return new CLIArgument<>(name, usage, true, type, null); } /** */ public static <T> CLIArgument<T> optionalArg(String name, String usage, Class<T> type, Supplier<T> dfltValSupplier) { - return new CLIArgument<T>(name, usage, true, type, p -> dfltValSupplier.get()); + return new CLIArgument<>(name, usage, true, type, p -> dfltValSupplier.get()); } /** */ public static <T> CLIArgument<T> optionalArg(String name, String usage, Class<T> type, Function<CLIArgumentParser, T> dfltValSupplier) { - return new CLIArgument<T>(name, usage, true, type, dfltValSupplier); + return new CLIArgument<>(name, usage, true, type, dfltValSupplier); } /** */ public static <T> CLIArgument<T> mandatoryArg(String name, String usage, Class<T> type) { - return new CLIArgument<T>(name, usage, false, type, null); + return new CLIArgument<>(name, usage, false, type, null); } /** */ @@ -87,7 +87,7 @@ public class CLIArgument<T> { } /** */ - public Class type() { + public Class<T> type() { return type; } diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java index d1e2e9bbb28..6c616198650 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java @@ -35,14 +35,14 @@ import static java.util.stream.Collectors.toSet; */ public class CLIArgumentParser { /** */ - private final Map<String, CLIArgument> argConfiguration = new LinkedHashMap<>(); + private final Map<String, CLIArgument<?>> argConfiguration = new LinkedHashMap<>(); /** */ private final Map<String, Object> parsedArgs = new HashMap<>(); /** */ - public CLIArgumentParser(List<CLIArgument> argConfiguration) { - for (CLIArgument cliArgument : argConfiguration) + public CLIArgumentParser(List<CLIArgument<?>> argConfiguration) { + for (CLIArgument<?> cliArgument : argConfiguration) this.argConfiguration.put(cliArgument.name(), cliArgument); } @@ -59,7 +59,7 @@ public class CLIArgumentParser { while (argsIter.hasNext()) { String arg = argsIter.next(); - CLIArgument cliArg = argConfiguration.get(arg); + CLIArgument<?> cliArg = argConfiguration.get(arg); if (cliArg == null) throw new IgniteException("Unexpected argument: " + arg); @@ -83,17 +83,17 @@ public class CLIArgumentParser { } /** */ - private Object parseVal(String val, Class type) { + private <T> T parseVal(String val, Class<T> type) { switch (type.getSimpleName()) { - case "String": return val; + case "String": return (T)val; - case "String[]": return val.split(","); + case "String[]": return (T)val.split(","); - case "Integer": return wrapNumberFormatException(() -> Integer.parseInt(val), val, Integer.class); + case "Integer": return (T)wrapNumberFormatException(() -> Integer.parseInt(val), val, Integer.class); - case "Long": return wrapNumberFormatException(() -> Long.parseLong(val), val, Long.class); + case "Long": return (T)wrapNumberFormatException(() -> Long.parseLong(val), val, Long.class); - case "UUID": return UUID.fromString(val); + case "UUID": return (T)UUID.fromString(val); default: throw new IgniteException("Unsupported argument type: " + type.getName()); } @@ -123,7 +123,7 @@ public class CLIArgumentParser { * @param <T> Value type. * @return Value. */ - public <T> T get(CLIArgument arg) { + public <T> T get(CLIArgument<?> arg) { Object val = parsedArgs.get(arg.name()); if (val == null) @@ -140,7 +140,7 @@ public class CLIArgumentParser { * @return Value. */ public <T> T get(String name) { - CLIArgument arg = argConfiguration.get(name); + CLIArgument<?> arg = argConfiguration.get(name); if (arg == null) throw new IgniteException("No such argument: " + name); @@ -156,10 +156,10 @@ public class CLIArgumentParser { public String usage() { GridStringBuilder sb = new GridStringBuilder("Usage: "); - for (CLIArgument arg : argConfiguration.values()) + for (CLIArgument<?> arg : argConfiguration.values()) sb.a(argNameForUsage(arg)).a(" "); - for (CLIArgument arg : argConfiguration.values()) { + for (CLIArgument<?> arg : argConfiguration.values()) { Object dfltVal = null; try { @@ -179,7 +179,7 @@ public class CLIArgumentParser { } /** */ - private String argNameForUsage(CLIArgument arg) { + private String argNameForUsage(CLIArgument<?> arg) { if (arg.optional()) return "[" + arg.name() + "]"; else diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReader.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReader.java index 848b2b09098..4ac13c6357d 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReader.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReader.java @@ -37,8 +37,6 @@ import java.util.Objects; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.regex.Matcher; @@ -47,13 +45,8 @@ import java.util.stream.IntStream; import java.util.stream.LongStream; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; -import org.apache.ignite.internal.cache.query.index.sorted.inline.io.AbstractInlineInnerIO; -import org.apache.ignite.internal.cache.query.index.sorted.inline.io.AbstractInlineLeafIO; +import org.apache.ignite.internal.cache.query.index.IndexProcessor; import org.apache.ignite.internal.cache.query.index.sorted.inline.io.InlineIO; -import org.apache.ignite.internal.cache.query.index.sorted.inline.io.InnerIO; -import org.apache.ignite.internal.cache.query.index.sorted.inline.io.LeafIO; -import org.apache.ignite.internal.cache.query.index.sorted.inline.io.MvccInnerIO; -import org.apache.ignite.internal.cache.query.index.sorted.inline.io.MvccLeafIO; import org.apache.ignite.internal.commandline.ProgressPrinter; import org.apache.ignite.internal.commandline.StringBuilderOutputStream; import org.apache.ignite.internal.commandline.argument.parser.CLIArgument; @@ -141,11 +134,11 @@ public class IgniteIndexReader implements AutoCloseable { public static final String ERROR_PREFIX = "<ERROR> "; /** */ - private static final Pattern CACHE_TYPE_ID_SEACH_PATTERN = + private static final Pattern CACHE_TYPE_ID_SEARCH_PATTERN = Pattern.compile("(?<id>[-0-9]{1,15})_(?<typeId>[-0-9]{1,15})_.*"); /** */ - private static final Pattern CACHE_ID_SEACH_PATTERN = + private static final Pattern CACHE_ID_SEARCH_PATTERN = Pattern.compile("(?<id>[-0-9]{1,15})_.*"); /** */ @@ -156,10 +149,7 @@ public class IgniteIndexReader implements AutoCloseable { /** */ static { - PageIO.registerH2(InnerIO.VERSIONS, LeafIO.VERSIONS, MvccInnerIO.VERSIONS, MvccLeafIO.VERSIONS); - - AbstractInlineInnerIO.register(); - AbstractInlineLeafIO.register(); + IndexProcessor.registerIO(); } /** Page size. */ @@ -285,7 +275,7 @@ public class IgniteIndexReader implements AutoCloseable { } /** */ - private void printPageStat(String prefix, String caption, Map<Class, Long> stat) { + private void printPageStat(String prefix, String caption, Map<Class<? extends PageIO>, Long> stat) { if (caption != null) print(prefix + caption + (stat.isEmpty() ? " empty" : "")); @@ -298,7 +288,7 @@ public class IgniteIndexReader implements AutoCloseable { e.printStackTrace(new PrintStream(os)); - outStream.println(os.toString()); + outStream.println(os); } /** */ @@ -330,35 +320,29 @@ public class IgniteIndexReader implements AutoCloseable { * @throws IgniteCheckedException If failed. */ IgniteBiTuple<Long, Long> partitionRoots(long pageMetaPageId) throws IgniteCheckedException { - AtomicLong pageListMetaPageId = new AtomicLong(); - AtomicLong metaTreeRootId = new AtomicLong(); - - doWithBuffer((buf, addr) -> { + return doWithBuffer((buf, addr) -> { readPage(filePageStore(partId(pageMetaPageId)), pageMetaPageId, buf); PageMetaIO pageMetaIO = PageIO.getPageIO(addr); - pageListMetaPageId.set(normalizePageId(pageMetaIO.getReuseListRoot(addr))); - - metaTreeRootId.set(normalizePageId(pageMetaIO.getTreeRoot(addr))); - - return null; + return new IgniteBiTuple<>( + normalizePageId(pageMetaIO.getTreeRoot(addr)), + normalizePageId(pageMetaIO.getReuseListRoot(addr)) + ); }); - - return new IgniteBiTuple<>(metaTreeRootId.get(), pageListMetaPageId.get()); } /** * Read index file. */ - public void readIdx() { + public void readIdx() throws IgniteCheckedException { long partPageStoresNum = Arrays.stream(partStores) .filter(Objects::nonNull) .count(); print("Partitions files num: " + partPageStoresNum); - Map<Class, Long> pageClasses = new HashMap<>(); + Map<Class<? extends PageIO>, Long> pageClasses = new HashMap<>(); long pagesNum = isNull(idxStore) ? 0 : (idxStore.size() - idxStore.headerSize()) / pageSize; @@ -366,81 +350,63 @@ public class IgniteIndexReader implements AutoCloseable { Set<Long> pageIds = new HashSet<>(); - AtomicReference<Map<String, TreeTraversalInfo>> treeInfo = new AtomicReference<>(); - - AtomicReference<Map<String, TreeTraversalInfo>> horizontalScans = new AtomicReference<>(); - - AtomicReference<PageListsInfo> pageListsInfo = new AtomicReference<>(); - - List<Throwable> errors; + IgniteBiTuple<Long, Long> indexPartitionRoots = partitionRoots(partMetaPageId(INDEX_PARTITION, FLAG_IDX)); - try { - IgniteBiTuple<Long, Long> indexPartitionRoots = partitionRoots(partMetaPageId(INDEX_PARTITION, FLAG_IDX)); - - long metaTreeRootId = indexPartitionRoots.get1(); - long pageListMetaPageId = indexPartitionRoots.get2(); - - // Traversing trees. - treeInfo.set(traverseAllTrees("Index trees traversal", metaTreeRootId, CountOnlyStorage::new, this::traverseTree)); + long metaTreeRootId = indexPartitionRoots.get1(); + long pageListMetaPageId = indexPartitionRoots.get2(); - treeInfo.get().forEach((name, info) -> { - pageIds.addAll(info.innerPageIds); + // Traversing trees. + Map<String, TreeTraversalInfo> treeInfo = + traverseAllTrees("Index trees traversal", metaTreeRootId, CountOnlyStorage::new, this::traverseTree); - pageIds.add(info.rootPageId); - }); + treeInfo.forEach((name, info) -> { + pageIds.addAll(info.innerPageIds); - Supplier<ItemStorage> itemStorageFactory = checkParts ? LinkStorage::new : CountOnlyStorage::new; + pageIds.add(info.rootPageId); + }); - horizontalScans.set( - traverseAllTrees("Scan index trees horizontally", metaTreeRootId, itemStorageFactory, this::horizontalTreeScan) - ); + Supplier<ItemStorage> itemStorageFactory = checkParts ? LinkStorage::new : CountOnlyStorage::new; - // Scanning page reuse lists. - if (pageListMetaPageId != 0) - pageListsInfo.set(getPageListsInfo(pageListMetaPageId)); + Map<String, TreeTraversalInfo> horizontalScans = + traverseAllTrees("Scan index trees horizontally", metaTreeRootId, itemStorageFactory, this::horizontalTreeScan); - ProgressPrinter progressPrinter = new ProgressPrinter(System.out, "Reading pages sequentially", pagesNum); + // Scanning page reuse lists. + PageListsInfo pageListsInfo = pageListMetaPageId == 0 ? null : getPageListsInfo(pageListMetaPageId); - // Scan all pages in file. - errors = scanFileStore(INDEX_PARTITION, FLAG_IDX, idxStore, (pageId, addr, io) -> { - progressPrinter.printProgress(); + ProgressPrinter progressPrinter = new ProgressPrinter(System.out, "Reading pages sequentially", pagesNum); - pageClasses.compute(io.getClass(), (k, v) -> v == null ? 1 : v + 1); + // Scan all pages in file. + List<Throwable> errors = scanFileStore(INDEX_PARTITION, FLAG_IDX, idxStore, (pageId, addr, io) -> { + progressPrinter.printProgress(); - if (!(io instanceof PageMetaIO || io instanceof PagesListMetaIO)) { - if (idxFilter == null) { - if ((io instanceof BPlusMetaIO || io instanceof BPlusInnerIO) - && !pageIds.contains(pageId) - && pageListsInfo.get() != null - && !pageListsInfo.get().allPages.contains(pageId)) { - throw new IgniteException( - "Possibly orphan " + io.getClass().getSimpleName() + " page, pageId=" + pageId - ); - } + pageClasses.compute(io.getClass(), (k, v) -> v == null ? 1 : v + 1); + + if (!(io instanceof PageMetaIO || io instanceof PagesListMetaIO)) { + if (idxFilter == null) { + if ((io instanceof BPlusMetaIO || io instanceof BPlusInnerIO) + && !pageIds.contains(pageId) + && pageListsInfo != null + && !pageListsInfo.allPages.contains(pageId)) { + throw new IgniteException( + "Possibly orphan " + io.getClass().getSimpleName() + " page, pageId=" + pageId + ); } } + } - return true; - }); - } - catch (IgniteCheckedException e) { - throw new IgniteException(INDEX_FILE_NAME + " scan problem", e); - } + return true; + }); - if (treeInfo.get() == null) - printErr("No tree meta info found."); - else { - printTraversalResults(RECURSIVE_TRAVERSE_NAME, treeInfo.get()); + printTraversalResults(RECURSIVE_TRAVERSE_NAME, treeInfo); - printTraversalResults(HORIZONTAL_SCAN_NAME, horizontalScans.get()); - } + printTraversalResults(HORIZONTAL_SCAN_NAME, horizontalScans); - compareTraversals(treeInfo.get(), horizontalScans.get()); + compareTraversals(treeInfo, horizontalScans); - if (pageListsInfo.get() == null) + if (pageListsInfo == null) printErr("No page lists meta info found."); else - printPagesListsInfo(pageListsInfo.get()); + printPagesListsInfo(pageListsInfo); printPageStat("", "\n---These pages types were encountered during sequential scan:", pageClasses); @@ -462,7 +428,7 @@ public class IgniteIndexReader implements AutoCloseable { "from count of pages found in index trees and page lists."); if (checkParts) { - Map<Integer, List<Throwable>> checkPartsErrors = checkParts(horizontalScans.get()); + Map<Integer, List<Throwable>> checkPartsErrors = checkParts(horizontalScans); print(""); @@ -757,7 +723,7 @@ public class IgniteIndexReader implements AutoCloseable { Set<Long> allPages = new HashSet<>(); - Map<Class, Long> pageListStat = new HashMap<>(); + Map<Class<? extends PageIO>, Long> pageListStat = new HashMap<>(); Map<Long, List<Throwable>> errors = new HashMap<>(); @@ -820,7 +786,7 @@ public class IgniteIndexReader implements AutoCloseable { * @param pageStat Page types statistics. * @return List of page ids. */ - private List<Long> getPageList(long pageListStartId, Map<Class, Long> pageStat) { + private List<Long> getPageList(long pageListStartId, Map<Class<? extends PageIO>, Long> pageStat) { List<Long> res = new LinkedList<>(); long nextNodeId = pageListStartId; @@ -916,7 +882,7 @@ public class IgniteIndexReader implements AutoCloseable { private void printTraversalResults(String prefix, Map<String, TreeTraversalInfo> treeInfos) { print("\n" + prefix + "Tree traversal results"); - Map<Class, Long> totalStat = new HashMap<>(); + Map<Class<? extends PageIO>, Long> totalStat = new HashMap<>(); AtomicInteger totalErr = new AtomicInteger(0); @@ -1001,7 +967,7 @@ public class IgniteIndexReader implements AutoCloseable { */ public static IgnitePair<Integer> getCacheAndTypeId(String name) { return CACHE_TYPE_IDS.computeIfAbsent(name, k -> { - Matcher mId = CACHE_TYPE_ID_SEACH_PATTERN.matcher(k); + Matcher mId = CACHE_TYPE_ID_SEARCH_PATTERN.matcher(k); if (mId.find()) { String id = mId.group("id"); @@ -1011,7 +977,7 @@ public class IgniteIndexReader implements AutoCloseable { return new IgnitePair<>(parseInt(id), parseInt(typeId)); } else { - Matcher cId = CACHE_ID_SEACH_PATTERN.matcher(k); + Matcher cId = CACHE_ID_SEARCH_PATTERN.matcher(k); if (cId.find()) { String id = cId.group("id"); @@ -1079,12 +1045,6 @@ public class IgniteIndexReader implements AutoCloseable { @Nullable ItemCallback itemCb, ItemStorage itemStorage ) { - FilePageStore store = filePageStore(partId(rootPageId)); - - Map<Class, Long> ioStat = new HashMap<>(); - - Map<Long, List<Throwable>> errors = new HashMap<>(); - Set<Long> innerPageIds = new HashSet<>(); PageCallback innerCb0 = (content, pageId) -> { @@ -1101,9 +1061,17 @@ public class IgniteIndexReader implements AutoCloseable { itemStorage.add(item); }; - getTreeNode(rootPageId, new TreeTraverseContext(treeName, store, ioStat, errors, innerCb0, leafCb, itemCb0)); + TreeTraverseContext ctx = new TreeTraverseContext( + treeName, + filePageStore(partId(rootPageId)), + innerCb0, + leafCb, + itemCb0 + ); + + getTreeNode(rootPageId, ctx); - return new TreeTraversalInfo(ioStat, errors, innerPageIds, rootPageId, itemStorage); + return new TreeTraversalInfo(ctx.ioStat, ctx.errors, innerPageIds, rootPageId, itemStorage); } /** @@ -1131,20 +1099,20 @@ public class IgniteIndexReader implements AutoCloseable { String treeName, ItemStorage itemStorage ) { - FilePageStore store = filePageStore(partId(rootPageId)); - - Map<Long, List<Throwable>> errors = new HashMap<>(); - - Map<Class, Long> ioStat = new HashMap<>(); - - TreeTraverseContext treeCtx = new TreeTraverseContext(treeName, store, ioStat, errors, null, null, null); + TreeTraverseContext treeCtx = new TreeTraverseContext( + treeName, + filePageStore(partId(rootPageId)), + null, + null, + null + ); ByteBuffer buf = allocateBuffer(pageSize); try { long addr = bufferAddress(buf); - readPage(store, rootPageId, buf); + readPage(treeCtx.store, rootPageId, buf); PageIO pageIO = PageIO.getPageIO(addr); @@ -1153,7 +1121,7 @@ public class IgniteIndexReader implements AutoCloseable { BPlusMetaIO metaIO = (BPlusMetaIO)pageIO; - ioStat.compute(metaIO.getClass(), (k, v) -> v == null ? 1 : v + 1); + treeCtx.ioStat.compute(metaIO.getClass(), (k, v) -> v == null ? 1 : v + 1); int lvlsCnt = metaIO.getLevelsCount(addr); @@ -1166,7 +1134,7 @@ public class IgniteIndexReader implements AutoCloseable { try { buf.rewind(); - readPage(store, pageId, buf); + readPage(treeCtx.store, pageId, buf); pageIO = PageIO.getPageIO(addr); @@ -1176,7 +1144,7 @@ public class IgniteIndexReader implements AutoCloseable { if (!(pageIO instanceof BPlusIO)) throw new IgniteException("Not-BPlus page found, pageId=" + pageId + ", level=" + i); - ioStat.compute(pageIO.getClass(), (k, v) -> v == null ? 1 : v + 1); + treeCtx.ioStat.compute(pageIO.getClass(), (k, v) -> v == null ? 1 : v + 1); if (pageIO instanceof BPlusLeafIO) { PageIOProcessor ioProcessor = getIOProcessor(pageIO); @@ -1186,10 +1154,10 @@ public class IgniteIndexReader implements AutoCloseable { pageContent.items.forEach(itemStorage::add); } - pageId = ((BPlusIO)pageIO).getForward(addr); + pageId = ((BPlusIO<?>)pageIO).getForward(addr); } catch (Throwable e) { - errors.computeIfAbsent(pageId, k -> new LinkedList<>()).add(e); + treeCtx.errors.computeIfAbsent(pageId, k -> new LinkedList<>()).add(e); pageId = 0; } @@ -1197,13 +1165,13 @@ public class IgniteIndexReader implements AutoCloseable { } } catch (Throwable e) { - errors.computeIfAbsent(rootPageId, k -> new LinkedList<>()).add(e); + treeCtx.errors.computeIfAbsent(rootPageId, k -> new LinkedList<>()).add(e); } finally { freeBuffer(buf); } - return new TreeTraversalInfo(ioStat, errors, null, rootPageId, itemStorage); + return new TreeTraversalInfo(treeCtx.ioStat, treeCtx.errors, null, rootPageId, itemStorage); } /** @@ -1315,9 +1283,7 @@ public class IgniteIndexReader implements AutoCloseable { public static void main(String[] args) throws Exception { System.out.println("THIS UTILITY MUST BE LAUNCHED ON PERSISTENT STORE WHICH IS NOT UNDER RUNNING GRID!"); - AtomicReference<CLIArgumentParser> parserRef = new AtomicReference<>(); - - List<CLIArgument> argsConfiguration = asList( + List<CLIArgument<?>> argsConfiguration = asList( mandatoryArg( DIR.arg(), "partition directory, where " + INDEX_FILE_NAME + " and (optionally) partition files are located.", @@ -1336,8 +1302,6 @@ public class IgniteIndexReader implements AutoCloseable { CLIArgumentParser p = new CLIArgumentParser(argsConfiguration); - parserRef.set(p); - if (args.length == 0) { System.out.println(p.usage()); @@ -1372,6 +1336,9 @@ public class IgniteIndexReader implements AutoCloseable { )) { reader.readIdx(); } + catch (IgniteCheckedException e) { + throw new IgniteException(INDEX_FILE_NAME + " scan problem", e); + } } /** @@ -1394,7 +1361,7 @@ public class IgniteIndexReader implements AutoCloseable { CHECK_PARTS("--check-parts"); /** */ - private String arg; + private final String arg; /** */ Args(String arg) { @@ -1473,7 +1440,7 @@ public class IgniteIndexReader implements AutoCloseable { private class InnerPageIOProcessor implements PageIOProcessor { /** {@inheritDoc} */ @Override public PageContent getContent(PageIO io, long addr, long pageId, TreeTraverseContext nodeCtx) { - BPlusInnerIO innerIo = (BPlusInnerIO)io; + BPlusInnerIO<?> innerIo = (BPlusInnerIO<?>)io; int cnt = innerIo.getCount(addr); @@ -1520,16 +1487,16 @@ public class IgniteIndexReader implements AutoCloseable { List<Object> items = new LinkedList<>(); - BPlusLeafIO leafIO = (BPlusLeafIO)io; + BPlusLeafIO<?> leafIO = (BPlusLeafIO<?>)io; for (int j = 0; j < leafIO.getCount(addr); j++) { Object idxItem = null; try { if (io instanceof IndexStorageImpl.MetaStoreLeafIO) { - idxItem = ((IndexStorageImpl.MetaStoreLeafIO)io).getLookupRow(null, addr, j); + idxItem = ((BPlusIO<IndexStorageImpl.IndexItem>)io).getLookupRow(null, addr, j); - sb.a(idxItem.toString() + " "); + sb.a(idxItem + " "); } else idxItem = getLeafItem(leafIO, pageId, addr, j, nodeCtx); @@ -1546,14 +1513,14 @@ public class IgniteIndexReader implements AutoCloseable { } /** */ - private Object getLeafItem(BPlusLeafIO io, long pageId, long addr, int idx, TreeTraverseContext nodeCtx) { + private Object getLeafItem(BPlusLeafIO<?> io, long pageId, long addr, int idx, TreeTraverseContext nodeCtx) { if (isLinkIo(io)) { final long link = getLink(io, addr, idx); final int cacheId; if (io instanceof AbstractDataLeafIO && ((AbstractDataLeafIO)io).storeCacheId()) - cacheId = ((AbstractDataLeafIO)io).getCacheId(addr, idx); + cacheId = ((RowLinkIO)io).getCacheId(addr, idx); else cacheId = nodeCtx.cacheId; @@ -1592,7 +1559,7 @@ public class IgniteIndexReader implements AutoCloseable { PageIO dataIo = PageIO.getPageIO(getType(dataBuf), getVersion(dataBuf)); if (dataIo instanceof AbstractDataPageIO) { - AbstractDataPageIO dataPageIO = (AbstractDataPageIO)dataIo; + AbstractDataPageIO<?> dataPageIO = (AbstractDataPageIO<?>)dataIo; DataPagePayload payload = dataPageIO.readPayload(dataBufAddr, linkedItemId, pageSize); @@ -1639,7 +1606,7 @@ public class IgniteIndexReader implements AutoCloseable { } /** */ - private long getLink(BPlusLeafIO io, long addr, int idx) { + private long getLink(BPlusLeafIO<?> io, long addr, int idx) { if (io instanceof RowLinkIO) return ((RowLinkIO)io).getLink(addr, idx); if (io instanceof InlineIO) diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/PageListsInfo.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/PageListsInfo.java index 4d18a7f21cd..f3b5df48421 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/PageListsInfo.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/PageListsInfo.java @@ -22,6 +22,7 @@ import java.util.Map; import java.util.Set; import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListMetaIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.lang.IgniteBiTuple; /** @@ -38,7 +39,7 @@ class PageListsInfo { final Set<Long> allPages; /** Page type statistics. */ - final Map<Class, Long> pageListStat; + final Map<Class<? extends PageIO>, Long> pageListStat; /** Map of errors, pageId -> list of exceptions. */ final Map<Long, List<Throwable>> errors; @@ -47,7 +48,7 @@ class PageListsInfo { public PageListsInfo( Map<IgniteBiTuple<Long, Integer>, List<Long>> bucketsData, Set<Long> allPages, - Map<Class, Long> pageListStat, + Map<Class<? extends PageIO>, Long> pageListStat, Map<Long, List<Throwable>> errors ) { this.bucketsData = bucketsData; diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeTraversalInfo.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeTraversalInfo.java index a2004b5cee7..6e49c565156 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeTraversalInfo.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeTraversalInfo.java @@ -20,13 +20,14 @@ package org.apache.ignite.internal.commandline.indexreader; import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; /** * This class is used to store result of tree traversal. */ public class TreeTraversalInfo { /** Page type statistics. */ - final Map<Class, Long> ioStat; + final Map<Class<? extends PageIO>, Long> ioStat; /** Map of errors, pageId -> set of exceptions. */ final Map<Long, List<Throwable>> errors; @@ -44,7 +45,7 @@ public class TreeTraversalInfo { /** */ public TreeTraversalInfo( - Map<Class, Long> ioStat, + Map<Class<? extends PageIO>, Long> ioStat, Map<Long, List<Throwable>> errors, Set<Long> innerPageIds, long rootPageId, diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeTraverseContext.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeTraverseContext.java index b254cf2901a..b30aaaee186 100644 --- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeTraverseContext.java +++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeTraverseContext.java @@ -17,10 +17,12 @@ package org.apache.ignite.internal.commandline.indexreader; +import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.jetbrains.annotations.Nullable; import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.getCacheId; @@ -39,7 +41,7 @@ class TreeTraverseContext { final FilePageStore store; /** Page type statistics. */ - final Map<Class, Long> ioStat; + final Map<Class<? extends PageIO>, Long> ioStat; /** Map of errors, pageId -> set of exceptions. */ final Map<Long, List<Throwable>> errors; @@ -57,8 +59,6 @@ class TreeTraverseContext { public TreeTraverseContext( String treeName, FilePageStore store, - Map<Class, Long> ioStat, - Map<Long, List<Throwable>> errors, @Nullable PageCallback innerCb, @Nullable PageCallback leafCb, @Nullable ItemCallback itemCb @@ -66,8 +66,8 @@ class TreeTraverseContext { this.treeName = treeName; this.cacheId = getCacheId(treeName); this.store = store; - this.ioStat = ioStat; - this.errors = errors; + this.ioStat = new HashMap<>(); + this.errors = new HashMap<>(); this.innerCb = innerCb; this.leafCb = leafCb; this.itemCb = itemCb; diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReaderTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReaderTest.java index 8bc8fdd19db..f7e51ae2bf6 100644 --- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReaderTest.java +++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReaderTest.java @@ -27,6 +27,7 @@ import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Collections; import java.util.HashSet; import java.util.LinkedList; import java.util.List; @@ -35,6 +36,7 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cache.QueryEntity; @@ -71,7 +73,6 @@ import static java.util.Objects.nonNull; import static java.util.Objects.requireNonNull; import static java.util.regex.Pattern.compile; import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toList; import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.ERROR_PREFIX; import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.HORIZONTAL_SCAN_NAME; import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.RECURSIVE_TRAVERSE_NAME; @@ -172,12 +173,12 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest { .setMaxSize(64 * 1024L * 1024L) ) ).setCacheConfiguration( - new CacheConfiguration(DEFAULT_CACHE_NAME) + new CacheConfiguration<>(DEFAULT_CACHE_NAME) .setGroupName(CACHE_GROUP_NAME) .setSqlSchema(QueryUtils.DFLT_SCHEMA), - new CacheConfiguration(EMPTY_CACHE_NAME) + new CacheConfiguration<>(EMPTY_CACHE_NAME) .setGroupName(EMPTY_CACHE_GROUP_NAME), - new CacheConfiguration(QUERY_CACHE_NAME) + new CacheConfiguration<>(QUERY_CACHE_NAME) .setGroupName(QUERY_CACHE_GROUP_NAME) .setQueryEntities(asList( new QueryEntity(Integer.class, TestClass1.class) @@ -425,9 +426,9 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest { * @return List of pairs, first is index name, second is list of fields, covered by index, divived by comma. */ private static List<IgnitePair<String>> idxs(String tblName, List<IgnitePair<String>> fields) { - List<IgnitePair<String>> res = new LinkedList<>(); - - res.addAll(fields.stream().map(f -> new IgnitePair<>(tblName + "_" + f.get1() + "_idx", f.get1())).collect(toList())); + List<IgnitePair<String>> res = fields.stream() + .map(f -> new IgnitePair<>(tblName + "_" + f.get1() + "_idx", f.get1())) + .collect(Collectors.toCollection(LinkedList::new)); // Add one multicolumn index. if (fields.size() > 1) { @@ -448,7 +449,7 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest { * {@code false} for inserting, updating and deleting data and deleting indexes, {@code null} for all data processing. * @param info Table info. */ - private static void createAndFillTable(IgniteCache cache, TableInfo info, @Nullable Boolean insert) { + private static void createAndFillTable(IgniteCache<?, ?> cache, TableInfo info, @Nullable Boolean insert) { String idxToDelName = info.tblName + "_idx_to_delete"; List<IgnitePair<String>> fields = fields(info.fieldsCnt); @@ -497,7 +498,7 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest { * @param fields List of fields. * @param cntr Counter which is used to generate data. */ - private static void insertQuery(IgniteCache cache, String tblName, List<IgnitePair<String>> fields, int cntr) { + private static void insertQuery(IgniteCache<?, ?> cache, String tblName, List<IgnitePair<String>> fields, int cntr) { GridStringBuilder q = new GridStringBuilder().a("insert into ").a(tblName).a(" (id, "); q.a(fields.stream().map(IgniteBiTuple::get1).collect(joining(", "))); @@ -520,7 +521,7 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest { * @param fields List of fields. * @param cntr Counter which is used to generate data. */ - private static void updateQuery(IgniteCache cache, String tblName, List<IgnitePair<String>> fields, int cntr) { + private static void updateQuery(IgniteCache<?, ?> cache, String tblName, List<IgnitePair<String>> fields, int cntr) { GridStringBuilder q = new GridStringBuilder().a("update ").a(tblName).a(" set ") .a(fields.stream().map(IgniteBiTuple::get1).collect(joining("=?, ", "", "=?"))) .a(" where id=?"); @@ -545,7 +546,7 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest { * @param qry Query string. * @return Result. */ - private static List<List<?>> query(IgniteCache cache, String qry) { + private static List<List<?>> query(IgniteCache<?, ?> cache, String qry) { return cache.query(new SqlFieldsQuery(qry)).getAll(); } @@ -557,7 +558,7 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest { * @param args Query arguments. * @return Result. */ - private static List<List<?>> query(IgniteCache cache, String qry, Object... args) { + private static List<List<?>> query(IgniteCache<?, ?> cache, String qry, Object... args) { return cache.query(new SqlFieldsQuery(qry).setArgs(args)).getAll(); } @@ -567,7 +568,6 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest { * @param output Output. * @param treesCnt Count of b+ trees. * @param travErrCnt Count of errors that can occur during traversal. - * @param horizScanErrCnt Count of errors that can occur during horizontal scan. * @param pageListsErrCnt Count of errors that can occur during page lists scan. * @param seqErrCnt Count of errors that can occur during sequential scan. * @param partReadingErr partition file reading errors should be present. @@ -717,11 +717,10 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest { * Create new {@link IgniteIndexReaderFilePageStoreFactory}. * * @param dir Data rirectory. - * @throws IgniteCheckedException If failed. */ protected IgniteIndexReaderFilePageStoreFactory createFilePageStoreFactory( File dir - ) throws IgniteCheckedException { + ) { return new IgniteIndexReaderFilePageStoreFactoryImpl(dir, PAGE_SIZE, PART_CNT, PAGE_STORE_VER); } @@ -800,7 +799,7 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest { */ @Test public void testCorruptedIdx() throws Exception { - checkCorruptedIdx(asList(workDir)); + checkCorruptedIdx(Collections.singletonList(workDir)); } /** @@ -816,7 +815,7 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest { */ @Test public void testCorruptedIdxWithCheckParts() throws Exception { - checkCorruptedIdxWithCheckParts(asList(workDir)); + checkCorruptedIdxWithCheckParts(Collections.singletonList(workDir)); } /** @@ -833,7 +832,7 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest { */ @Test public void testCorruptedPart() throws Exception { - checkCorruptedPart(asList(workDir)); + checkCorruptedPart(Collections.singletonList(workDir)); } /** @@ -850,7 +849,7 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest { */ @Test public void testCorruptedIdxAndPart() throws Exception { - checkCorruptedIdxAndPart(asList(workDir)); + checkCorruptedIdxAndPart(Collections.singletonList(workDir)); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cache/query/index/IndexProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/cache/query/index/IndexProcessor.java index aec507602c8..913b409a422 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/cache/query/index/IndexProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/cache/query/index/IndexProcessor.java @@ -79,6 +79,13 @@ public class IndexProcessor extends GridProcessorAdapter { * Register inline IOs for sorted indexes. */ static { + registerIO(); + } + + /** + * Register inline IOs for sorted indexes. + */ + public static void registerIO() { PageIO.registerH2(InnerIO.VERSIONS, LeafIO.VERSIONS, MvccInnerIO.VERSIONS, MvccLeafIO.VERSIONS); AbstractInlineInnerIO.register();