This is an automated email from the ASF dual-hosted git repository. sergeychugunov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push: new 235a779 IGNITE-14246 Parameter to filter certain pages from WAL when reading with WAL converter - Fixes #8834. 235a779 is described below commit 235a779dd11894b0870cff8687ad286748f7969a Author: Kirill Tkalenko <tkalkir...@yandex.ru> AuthorDate: Wed Mar 3 13:58:07 2021 +0300 IGNITE-14246 Parameter to filter certain pages from WAL when reading with WAL converter - Fixes #8834. Signed-off-by: Sergey Chugunov <sergey.chugu...@gmail.com> --- .../development/utils/IgniteWalConverter.java | 61 ++++++- .../utils/IgniteWalConverterArguments.java | 137 ++++++++++++++- .../utils/IgniteWalConverterArgumentsTest.java | 191 +++++++++++++++++---- .../development/utils/IgniteWalConverterTest.java | 125 +++++++++++--- 4 files changed, 442 insertions(+), 72 deletions(-) diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java index e0d55c7..838ba9b 100644 --- a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java +++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java @@ -19,8 +19,15 @@ package org.apache.ignite.development.utils; import java.io.PrintStream; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.wal.WALIterator; import org.apache.ignite.internal.pagemem.wal.record.DataEntry; import org.apache.ignite.internal.pagemem.wal.record.DataRecord; @@ -30,6 +37,7 @@ import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor; import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer; +import org.apache.ignite.internal.processors.cache.persistence.wal.reader.FilteredWalIterator; import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer; import org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasInnerIO; @@ -38,10 +46,15 @@ import org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO; import org.apache.ignite.internal.processors.query.h2.database.io.H2LeafIO; import org.apache.ignite.internal.processors.query.h2.database.io.H2MvccInnerIO; import org.apache.ignite.internal.processors.query.h2.database.io.H2MvccLeafIO; -import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.T2; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteBiTuple; +import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.WalFilters.checkpoint; +import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.WalFilters.pageOwner; +import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.WalFilters.partitionMetaStateUpdate; + /** * Print WAL log data in human-readable form. */ @@ -94,7 +107,7 @@ public class IgniteWalConverter { boolean printAlways = F.isEmpty(params.getRecordTypes()); - try (WALIterator stIt = factory.iterator(iteratorParametersBuilder)) { + try (WALIterator stIt = walIterator(factory.iterator(iteratorParametersBuilder), params.getPages())) { String currentWalPath = null; while (stIt.hasNextX()) { @@ -155,27 +168,29 @@ public class IgniteWalConverter { } /** - * Get current wal file path, used in {@code WALIterator} + * Get current wal file path, used in {@code WALIterator}. * * @param it WALIterator. * @return Current wal file path. */ private static String getCurrentWalFilePath(WALIterator it) { - String result = null; + String res = null; try { - final Integer curIdx = IgniteUtils.field(it, "curIdx"); + WALIterator walIter = it instanceof FilteredWalIterator ? U.field(it, "delegateWalIter") : it; + + Integer curIdx = U.field(walIter, "curIdx"); - final List<FileDescriptor> walFileDescriptors = IgniteUtils.field(it, "walFileDescriptors"); + List<FileDescriptor> walFileDescriptors = U.field(walIter, "walFileDescriptors"); - if (curIdx != null && walFileDescriptors != null && !walFileDescriptors.isEmpty()) - result = walFileDescriptors.get(curIdx).getAbsolutePath(); + if (curIdx != null && walFileDescriptors != null && curIdx < walFileDescriptors.size()) + res = walFileDescriptors.get(curIdx).getAbsolutePath(); } catch (Exception e) { e.printStackTrace(); } - return result; + return res; } /** @@ -201,4 +216,32 @@ public class IgniteWalConverter { return walRecord.toString(); } + + /** + * Getting WAL iterator. + * + * @param walIter WAL iterator. + * @param pageIds Pages for searching in format grpId:pageId. + * @return WAL iterator. + */ + private static WALIterator walIterator( + WALIterator walIter, + Collection<T2<Integer, Long>> pageIds + ) throws IgniteCheckedException { + Predicate<IgniteBiTuple<WALPointer, WALRecord>> filter = null; + + if (!pageIds.isEmpty()) { + Set<T2<Integer, Long>> grpAndPageIds0 = new HashSet<>(pageIds); + + // Collect all (group, partition) partition pairs. + Set<T2<Integer, Integer>> grpAndParts = grpAndPageIds0.stream() + .map((tup) -> new T2<>(tup.get1(), PageIdUtils.partId(tup.get2()))) + .collect(Collectors.toSet()); + + // Build WAL filter. (Checkoint, Page, Partition meta) + filter = checkpoint().or(pageOwner(grpAndPageIds0)).or(partitionMetaStateUpdate(grpAndParts)); + } + + return filter != null ? new FilteredWalIterator(walIter, filter) : walIter; + } } diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverterArguments.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverterArguments.java index af3bfb8..3e368cc 100644 --- a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverterArguments.java +++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverterArguments.java @@ -17,16 +17,27 @@ package org.apache.ignite.development.utils; +import java.io.BufferedReader; import java.io.File; +import java.io.FileReader; +import java.io.IOException; import java.io.PrintStream; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Date; import java.util.HashSet; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.T2; +import org.jetbrains.annotations.Nullable; + +import static java.util.Collections.emptyList; /** * Parameters for IgniteWalConverter with parsed and validated. @@ -71,6 +82,12 @@ public class IgniteWalConverterArguments { /** */ private static final String SKIP_CRC = "skipCrc"; + /** Argument "pages". */ + private static final String PAGES = "pages"; + + /** Record pattern for {@link #PAGES}. */ + private static final Pattern PAGE_ID_PATTERN = Pattern.compile("(\\d+):(\\d+)"); + /** Path to dir with wal files. */ private final File walDir; @@ -110,7 +127,12 @@ public class IgniteWalConverterArguments { /** Skip CRC calculation/check flag */ private final boolean skipCrc; + /** Pages for searching in format grpId:pageId. */ + private final Collection<T2<Integer, Long>> pages; + /** + * Constructor. + * * @param walDir Path to dir with wal files. * @param walArchiveDir Path to dir with archive wal files. * @param pageSize Size of pages, which was selected for file store (1024, 2048, 4096, etc). @@ -124,12 +146,13 @@ public class IgniteWalConverterArguments { * @param processSensitiveData Strategy for the processing of sensitive data (SHOW, HIDE, HASH, MD5). * @param printStat Write summary statistics for WAL. * @param skipCrc Skip CRC calculation/check flag. + * @param pages Pages for searching in format grpId:pageId. */ public IgniteWalConverterArguments(File walDir, File walArchiveDir, int pageSize, File binaryMetadataFileStoreDir, File marshallerMappingFileStoreDir, boolean keepBinary, Set<WALRecord.RecordType> recordTypes, Long fromTime, Long toTime, String recordContainsText, ProcessSensitiveData processSensitiveData, - boolean printStat, boolean skipCrc) { + boolean printStat, boolean skipCrc, Collection<T2<Integer, Long>> pages) { this.walDir = walDir; this.walArchiveDir = walArchiveDir; this.pageSize = pageSize; @@ -143,6 +166,7 @@ public class IgniteWalConverterArguments { this.processSensitiveData = processSensitiveData; this.printStat = printStat; this.skipCrc = skipCrc; + this.pages = pages; } /** @@ -263,12 +287,21 @@ public class IgniteWalConverterArguments { } /** + * Return pages for searching in format grpId:pageId. + * + * @return Pages. + */ + public Collection<T2<Integer, Long>> getPages() { + return pages; + } + + /** * Parse command line arguments and return filled IgniteWalConverterArguments * * @param args Command line arguments. * @return IgniteWalConverterArguments. */ - public static IgniteWalConverterArguments parse(final PrintStream out, String args[]) { + public static IgniteWalConverterArguments parse(final PrintStream out, String... args) { if (args == null || args.length < 1) { out.println("Print WAL log data in human-readable form."); out.println("You need to provide:"); @@ -285,6 +318,8 @@ public class IgniteWalConverterArguments { out.println(" processSensitiveData (Optional) Strategy for the processing of sensitive data (SHOW, HIDE, HASH, MD5). Default SHOW."); out.println(" printStat Write summary statistics for WAL. Default false."); out.println(" skipCrc Skip CRC calculation/check flag. Default false."); + out.println(" pages (Optional) Comma-separated pages or path to file with " + + "pages on each line in grpId:pageId format."); out.println("For example:"); out.println(" walDir=/work/db/wal"); out.println(" walArchiveDir=/work/db/wal_archive"); @@ -298,6 +333,7 @@ public class IgniteWalConverterArguments { out.println(" recordContainsText=search string"); out.println(" processSensitiveData=SHOW"); out.println(" skipCrc=true"); + out.println(" pages=123456:789456123,123456:789456124"); return null; } @@ -314,6 +350,7 @@ public class IgniteWalConverterArguments { ProcessSensitiveData processSensitiveData = ProcessSensitiveData.SHOW; boolean printStat = false; boolean skipCrc = false; + Collection<T2<Integer, Long>> pages = emptyList(); for (String arg : args) { if (arg.startsWith(WAL_DIR + "=")) { @@ -420,6 +457,13 @@ public class IgniteWalConverterArguments { else if (arg.startsWith(SKIP_CRC + "=")) { skipCrc = parseBoolean(SKIP_CRC, arg.substring(SKIP_CRC.length() + 1)); } + else if (arg.startsWith(PAGES + "=")) { + String pagesStr = arg.replace(PAGES + "=", ""); + + File pagesFile = new File(pagesStr); + + pages = pagesFile.exists() ? parsePageIds(pagesFile) : parsePageIds(pagesStr.split(",")); + } } if (walDir == null && walArchiveDir == null) @@ -459,9 +503,13 @@ public class IgniteWalConverterArguments { out.printf("\t%s = %b\n", SKIP_CRC, skipCrc); + if (!pages.isEmpty()) + out.printf("\t%s = %s\n", PAGES, pages); + return new IgniteWalConverterArguments(walDir, walArchiveDir, pageSize, binaryMetadataFileStoreDir, marshallerMappingFileStoreDir, - keepBinary, recordTypes, fromTime, toTime, recordContainsText, processSensitiveData, printStat, skipCrc); + keepBinary, recordTypes, fromTime, toTime, recordContainsText, processSensitiveData, printStat, skipCrc, + pages); } /** @@ -488,4 +536,87 @@ public class IgniteWalConverterArguments { else throw new IllegalArgumentException("Incorrect flag " + name + ", valid value: true or false. Error parse: " + value); } + + /** + * Parsing and checking the string representation of the page in grpId:pageId format. + * Example: 123:456. + * + * @param s String value. + * @return Parsed value. + * @throws IllegalArgumentException If the string value is invalid. + */ + static T2<Integer, Long> parsePageId(@Nullable String s) throws IllegalArgumentException { + if (s == null) + throw new IllegalArgumentException("Null value."); + else if (s.isEmpty()) + throw new IllegalArgumentException("Empty value."); + + Matcher m = PAGE_ID_PATTERN.matcher(s); + + if (!m.matches()) { + throw new IllegalArgumentException("Incorrect value " + s + ", valid format: grpId:pageId. " + + "Example: 123:456"); + } + + return new T2<>(Integer.parseInt(m.group(1)), Long.parseLong(m.group(2))); + } + + /** + * Parsing a file in which each line is expected to be grpId:pageId format. + * + * @param f File. + * @return Parsed pages. + * @throws IllegalArgumentException If there is an error when working with a file or parsing lines. + * @see #parsePageId + */ + static Collection<T2<Integer, Long>> parsePageIds(File f) throws IllegalArgumentException { + try (BufferedReader reader = new BufferedReader(new FileReader(f))) { + int i = 0; + String s; + + Collection<T2<Integer, Long>> res = new ArrayList<>(); + + while ((s = reader.readLine()) != null) { + try { + res.add(parsePageId(s)); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Error parsing value \"" + s + "\" on " + i + " line of the file: " + f.getAbsolutePath(), + e + ); + } + + i++; + } + + return res.isEmpty() ? emptyList() : res; + } + catch (IOException e) { + throw new IllegalArgumentException("Error when working with the file: " + f.getAbsolutePath(), e); + } + } + + /** + * Parsing strings in which each element is expected to be in grpId:pageId format. + * + * @param strs String values. + * @return Parsed pages. + * @throws IllegalArgumentException If there is an error parsing the strs. + * @see #parsePageId + */ + static Collection<T2<Integer, Long>> parsePageIds(String... strs) throws IllegalArgumentException { + Collection<T2<Integer, Long>> res = new ArrayList<>(); + + for (int i = 0; i < strs.length; i++) { + try { + res.add(parsePageId(strs[i])); + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Error parsing value \"" + strs[i] + "\" of " + i + " element", e); + } + } + + return res.isEmpty() ? emptyList() : res; + } } diff --git a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterArgumentsTest.java b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterArgumentsTest.java index 84098fb..b5c6976 100644 --- a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterArgumentsTest.java +++ b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterArgumentsTest.java @@ -23,14 +23,21 @@ import java.io.IOException; import java.io.PrintStream; import java.lang.reflect.Field; import java.lang.reflect.Modifier; - import org.apache.ignite.IgniteException; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; -import org.apache.ignite.testframework.GridTestUtils; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.T2; +import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Assert; import org.junit.Test; +import static java.nio.charset.Charset.defaultCharset; +import static org.apache.ignite.development.utils.IgniteWalConverterArguments.parse; +import static org.apache.ignite.development.utils.IgniteWalConverterArguments.parsePageId; +import static org.apache.ignite.development.utils.IgniteWalConverterArguments.parsePageIds; +import static org.apache.ignite.testframework.GridTestUtils.assertThrows; + /** * Test for IgniteWalConverterArguments */ @@ -54,7 +61,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { public void testViewHelp() throws Exception { final ByteArrayOutputStream out = new ByteArrayOutputStream(); - final IgniteWalConverterArguments parseArgs = IgniteWalConverterArguments.parse(new PrintStream(out), null); + final IgniteWalConverterArguments parseArgs = parse(new PrintStream(out), null); Assert.assertNull(parseArgs); @@ -82,8 +89,8 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { */ @Test public void testRequiredWalDir() throws Exception { - GridTestUtils.assertThrows(log, () -> { - IgniteWalConverterArguments.parse(System.out, new String[] {"pageSize=4096"}); + assertThrows(log, () -> { + parse(System.out, new String[] {"pageSize=4096"}); }, IgniteException.class, "The paths to the WAL files are not specified."); } @@ -94,8 +101,8 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { */ @Test public void testIncorrectWalDir() throws Exception { - GridTestUtils.assertThrows(log, () -> { - IgniteWalConverterArguments.parse(System.out, new String[] {"walDir=non_existing_path"}); + assertThrows(log, () -> { + parse(System.out, new String[] {"walDir=non_existing_path"}); }, IgniteException.class, "Incorrect path to dir with wal files: non_existing_path"); } @@ -106,8 +113,8 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { */ @Test public void testIncorrectWalArchiveDir() throws Exception { - GridTestUtils.assertThrows(log, () -> { - IgniteWalConverterArguments.parse(System.out, new String[] {"walArchiveDir=non_existing_path"}); + assertThrows(log, () -> { + parse(System.out, new String[] {"walArchiveDir=non_existing_path"}); }, IgniteException.class, "Incorrect path to dir with archive wal files: non_existing_path"); } @@ -118,7 +125,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { */ @Test public void testIncorrectPageSize() throws Exception { - GridTestUtils.assertThrows(log, () -> { + assertThrows(log, () -> { final File wal = File.createTempFile("wal", ""); wal.deleteOnExit(); @@ -127,7 +134,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { "pageSize=not_integer" }; - IgniteWalConverterArguments.parse(System.out, args); + parse(System.out, args); }, IgniteException.class, "Incorrect page size. Error parse: not_integer"); } @@ -138,7 +145,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { */ @Test public void testIncorrectBinaryMetadataFileStoreDir() throws Exception { - GridTestUtils.assertThrows(log, () -> { + assertThrows(log, () -> { final File wal = File.createTempFile("wal", ""); wal.deleteOnExit(); @@ -147,7 +154,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { "binaryMetadataFileStoreDir=non_existing_path" }; - IgniteWalConverterArguments.parse(System.out, args); + parse(System.out, args); }, IgniteException.class, "Incorrect path to dir with binary meta files: non_existing_path"); } @@ -158,7 +165,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { */ @Test public void testIncorrectMarshallerMappingFileStoreDir() throws Exception { - GridTestUtils.assertThrows(log, () -> { + assertThrows(log, () -> { final File wal = File.createTempFile("wal", ""); wal.deleteOnExit(); @@ -167,7 +174,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { "marshallerMappingFileStoreDir=non_existing_path" }; - IgniteWalConverterArguments.parse(System.out, args); + parse(System.out, args); }, IgniteException.class, "Incorrect path to dir with marshaller files: non_existing_path"); } @@ -178,7 +185,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { */ @Test public void testIncorrectKeepBinary() throws Exception { - GridTestUtils.assertThrows(log, () -> { + assertThrows(log, () -> { final File wal = File.createTempFile("wal", ""); wal.deleteOnExit(); @@ -187,7 +194,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { "keepBinary=not_boolean" }; - IgniteWalConverterArguments.parse(System.out, args); + parse(System.out, args); }, IgniteException.class, "Incorrect flag keepBinary, valid value: true or false. Error parse: not_boolean"); } @@ -198,7 +205,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { */ @Test public void testIncorrectRecordTypes() throws Exception { - GridTestUtils.assertThrows(log, () -> { + assertThrows(log, () -> { final File wal = File.createTempFile("wal", ""); wal.deleteOnExit(); @@ -207,7 +214,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { "recordTypes=not_exist" }; - IgniteWalConverterArguments.parse(System.out, args); + parse(System.out, args); }, IgniteException.class, "Unknown record types: [not_exist]."); } @@ -218,7 +225,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { */ @Test public void testIncorrectSeveralRecordTypes() throws Exception { - GridTestUtils.assertThrows(log, () -> { + assertThrows(log, () -> { final File wal = File.createTempFile("wal", ""); wal.deleteOnExit(); @@ -227,7 +234,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { "recordTypes=not_exist1,not_exist2" }; - IgniteWalConverterArguments.parse(System.out, args); + parse(System.out, args); }, IgniteException.class, "Unknown record types: [not_exist1, not_exist2]."); } @@ -238,7 +245,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { */ @Test public void testIncorrectWalTimeFromMillis() throws Exception { - GridTestUtils.assertThrows(log, () -> { + assertThrows(log, () -> { final File wal = File.createTempFile("wal", ""); wal.deleteOnExit(); @@ -247,7 +254,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { "walTimeFromMillis=not_long" }; - IgniteWalConverterArguments.parse(System.out, args); + parse(System.out, args); }, IgniteException.class, "Incorrect walTimeFromMillis. Error parse: not_long"); } @@ -258,7 +265,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { */ @Test public void testIncorrectWalTimeToMillis() throws Exception { - GridTestUtils.assertThrows(log, () -> { + assertThrows(log, () -> { final File wal = File.createTempFile("wal", ""); wal.deleteOnExit(); @@ -267,7 +274,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { "walTimeToMillis=not_long" }; - IgniteWalConverterArguments.parse(System.out, args); + parse(System.out, args); }, IgniteException.class, "Incorrect walTimeToMillis. Error parse: not_long"); } @@ -278,7 +285,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { */ @Test public void testIncorrectProcessSensitiveData() throws Exception { - GridTestUtils.assertThrows(log, () -> { + assertThrows(log, () -> { final File wal = File.createTempFile("wal", ""); wal.deleteOnExit(); @@ -287,7 +294,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { "processSensitiveData=unknown" }; - IgniteWalConverterArguments.parse(System.out, args); + parse(System.out, args); }, IgniteException.class, "Unknown processSensitiveData: unknown. Supported: "); } @@ -298,7 +305,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { */ @Test public void testIncorrectPrintStat() throws Exception { - GridTestUtils.assertThrows(log, () -> { + assertThrows(log, () -> { final File wal = File.createTempFile("wal", ""); wal.deleteOnExit(); @@ -307,7 +314,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { "printStat=not_boolean" }; - IgniteWalConverterArguments.parse(System.out, args); + parse(System.out, args); }, IgniteException.class, "Incorrect flag printStat, valid value: true or false. Error parse: not_boolean"); } @@ -318,7 +325,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { */ @Test public void testIncorrectSkipCrc() throws Exception { - GridTestUtils.assertThrows(log, () -> { + assertThrows(log, () -> { final File wal = File.createTempFile("wal", ""); wal.deleteOnExit(); @@ -327,7 +334,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { "skipCrc=not_boolean" }; - IgniteWalConverterArguments.parse(System.out, args); + parse(System.out, args); }, IgniteException.class, "Incorrect flag skipCrc, valid value: true or false. Error parse: not_boolean"); } @@ -345,7 +352,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { "walDir=" + wal.getAbsolutePath() }; - final IgniteWalConverterArguments parseArgs = IgniteWalConverterArguments.parse(System.out, args); + final IgniteWalConverterArguments parseArgs = parse(System.out, args); Assert.assertEquals(4096, parseArgs.getPageSize()); Assert.assertNull(parseArgs.getBinaryMetadataFileStoreDir()); @@ -392,7 +399,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { "printStat=true", "skipCrc=true"}; - final IgniteWalConverterArguments parseArgs = IgniteWalConverterArguments.parse(System.out, args); + final IgniteWalConverterArguments parseArgs = parse(System.out, args); Assert.assertEquals(wal, parseArgs.getWalDir()); Assert.assertEquals(walArchive, parseArgs.getWalArchiveDir()); Assert.assertEquals(2048, parseArgs.getPageSize()); @@ -408,4 +415,122 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest { Assert.assertTrue(parseArgs.isPrintStat()); Assert.assertTrue(parseArgs.isSkipCrc()); } + + /** + * Checking the correctness of the method {@link IgniteWalConverterArguments#parsePageId}. + */ + @Test + public void testParsePageId() { + String[] invalidValues = { + null, + "", + " ", + "a", + "a:", + "a:b", + "a:b", + "a:1", + "1:b", + "1;1", + "1a:1", + "1:1b", + "1:1:1", + }; + + for (String v : invalidValues) + assertThrows(log, () -> parsePageId(v), IllegalArgumentException.class, null); + + assertEquals(new T2<>(1, 1L), parsePageId("1:1")); + } + + /** + * Checking the correctness of the method {@link IgniteWalConverterArguments#parsePageIds(File)}. + * + * @throws Exception If failed. + */ + @Test + public void testParsePageIdsFile() throws Exception { + File f = new File(System.getProperty("java.io.tmpdir"), "test"); + + try { + assertThrows(log, () -> parsePageIds(f), IllegalArgumentException.class, null); + + assertTrue(f.createNewFile()); + + assertTrue(parsePageIds(f).isEmpty()); + + U.writeStringToFile(f, "a:b", defaultCharset().toString(), false); + assertThrows(log, () -> parsePageIds(f), IllegalArgumentException.class, null); + + U.writeStringToFile(f, "1:1,1:1", defaultCharset().toString(), false); + assertThrows(log, () -> parsePageIds(f), IllegalArgumentException.class, null); + + U.writeStringToFile(f, "1:1", defaultCharset().toString(), false); + assertEqualsCollections(F.asList(new T2<>(1, 1L)), parsePageIds(f)); + + U.writeStringToFile(f, U.nl() + "2:2", defaultCharset().toString(), true); + assertEqualsCollections(F.asList(new T2<>(1, 1L), new T2<>(2, 2L)), parsePageIds(f)); + } + finally { + assertTrue(U.delete(f)); + } + } + + /** + * Checking the correctness of the method {@link IgniteWalConverterArguments#parsePageIds(String...)}. + */ + @Test + public void testParsePageIdsStrings() { + assertTrue(parsePageIds().isEmpty()); + + assertThrows(log, () -> parsePageIds("a:b"), IllegalArgumentException.class, null); + assertThrows(log, () -> parsePageIds("1:1", "a:b"), IllegalArgumentException.class, null); + + assertEqualsCollections(F.asList(new T2<>(1, 1L)), parsePageIds("1:1")); + assertEqualsCollections(F.asList(new T2<>(1, 1L), new T2<>(2, 2L)), parsePageIds("1:1", "2:2")); + } + + /** + * Checking the correctness of parsing the argument "pages". + * + * @throws Exception If failed. + */ + @Test + public void testParsePagesArgument() throws IOException { + File walDir = new File(System.getProperty("java.io.tmpdir"), "walDir"); + + try { + assertTrue(walDir.mkdir()); + + String walDirStr = "walDir=" + walDir.getAbsolutePath(); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(baos); + + assertThrows(log, () -> parse(ps, walDirStr, "pages=1"), IllegalArgumentException.class, null); + assertThrows(log, () -> parse(ps, walDirStr, "pages="), IllegalArgumentException.class, null); + + assertEqualsCollections(F.asList(new T2<>(1, 1L)), parse(ps, walDirStr, "pages=1:1").getPages()); + + File f = new File(System.getProperty("java.io.tmpdir"), "test"); + + try { + String pagesFileStr = "pages=" + f.getAbsolutePath(); + + assertThrows(log, () -> parse(ps, walDirStr, pagesFileStr), IllegalArgumentException.class, null); + + assertTrue(f.createNewFile()); + assertTrue(parse(ps, walDirStr, pagesFileStr).getPages().isEmpty()); + + U.writeStringToFile(f, "1:1", defaultCharset().toString(), false); + assertEqualsCollections(F.asList(new T2<>(1, 1L)), parse(ps, walDirStr, pagesFileStr).getPages()); + } + finally { + assertTrue(U.delete(f)); + } + } + finally { + assertTrue(U.delete(walDir)); + } + } } diff --git a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterTest.java b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterTest.java index 1e38dbd..f3627b7 100644 --- a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterTest.java +++ b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterTest.java @@ -21,6 +21,7 @@ import java.io.ByteArrayOutputStream; import java.io.File; import java.io.PrintStream; import java.io.RandomAccessFile; +import java.util.ArrayList; import java.util.Base64; import java.util.LinkedList; import java.util.List; @@ -34,12 +35,27 @@ import org.apache.ignite.configuration.DataStorageConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.configuration.WALMode; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.pagemem.wal.WALIterator; +import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; +import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer; +import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer; +import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.jetbrains.annotations.Nullable; import org.junit.Test; +import static java.util.Collections.emptyList; +import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_BINARY_METADATA_PATH; +import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_MARSHALLER_PATH; +import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH; +import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_PATH; +import static org.apache.ignite.development.utils.IgniteWalConverter.convert; +import static org.apache.ignite.development.utils.IgniteWalConverterArguments.parse; +import static org.apache.ignite.testframework.GridTestUtils.assertContains; + /** * Test for IgniteWalConverter */ @@ -65,6 +81,7 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest { @Override protected void beforeTest() throws Exception { super.beforeTest(); + stopAllGrids(); cleanPersistenceDir(); } @@ -145,24 +162,24 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest { public void testIgniteWalConverter() throws Exception { final List<Person> list = new LinkedList<>(); - final String nodeFolder = createWal(list); + final String nodeFolder = createWal(list, null); final ByteArrayOutputStream outByte = new ByteArrayOutputStream(); final PrintStream out = new PrintStream(outByte); final IgniteWalConverterArguments arg = new IgniteWalConverterArguments( - U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_WAL_PATH, false), - U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, false), + U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_PATH, false), + U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_ARCHIVE_PATH, false), DataStorageConfiguration.DFLT_PAGE_SIZE, - new File(U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, false), nodeFolder), - U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_MARSHALLER_PATH, false), + new File(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_BINARY_METADATA_PATH, false), nodeFolder), + U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_MARSHALLER_PATH, false), false, null, - null, null, null, null, true,true + null, null, null, null, true,true, emptyList() ); - IgniteWalConverter.convert(out, arg); + convert(out, arg); final String result = outByte.toString(); @@ -208,24 +225,24 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest { public void testIgniteWalConverterWithOutBinaryMeta() throws Exception { final List<Person> list = new LinkedList<>(); - createWal(list); + createWal(list, null); final ByteArrayOutputStream outByte = new ByteArrayOutputStream(); final PrintStream out = new PrintStream(outByte); final IgniteWalConverterArguments arg = new IgniteWalConverterArguments( - U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_WAL_PATH, false), - U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, false), + U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_PATH, false), + U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_ARCHIVE_PATH, false), DataStorageConfiguration.DFLT_PAGE_SIZE, null, null, false, null, - null, null, null, null, true,true + null, null, null, null, true,true, emptyList() ); - IgniteWalConverter.convert(out, arg); + convert(out, arg); final String result = outByte.toString(); @@ -274,9 +291,9 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest { public void testIgniteWalConverterWithBrokenWal() throws Exception { final List<Person> list = new LinkedList<>(); - final String nodeFolder = createWal(list); + final String nodeFolder = createWal(list, null); - final File walDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_WAL_PATH, false); + final File walDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_PATH, false); final File wal = new File(walDir, nodeFolder + File.separator + "0000000000000000.wal"); @@ -332,16 +349,16 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest { final IgniteWalConverterArguments arg = new IgniteWalConverterArguments( walDir, - U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, false), + U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_ARCHIVE_PATH, false), DataStorageConfiguration.DFLT_PAGE_SIZE, - new File(U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, false), nodeFolder), - U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_MARSHALLER_PATH, false), + new File(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_BINARY_METADATA_PATH, false), nodeFolder), + U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_MARSHALLER_PATH, false), false, null, - null, null, null, null, true,true + null, null, null, null, true,true, emptyList() ); - IgniteWalConverter.convert(out, arg); + convert(out, arg); final String result = outByte.toString(); @@ -392,9 +409,9 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest { public void testIgniteWalConverterWithUnreadableWal() throws Exception { final List<Person> list = new LinkedList<>(); - final String nodeFolder = createWal(list); + final String nodeFolder = createWal(list, null); - final File walDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_WAL_PATH, false); + final File walDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_PATH, false); final File wal = new File(walDir, nodeFolder + File.separator + "0000000000000000.wal"); @@ -436,16 +453,16 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest { final IgniteWalConverterArguments arg = new IgniteWalConverterArguments( walDir, - U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, false), + U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_ARCHIVE_PATH, false), DataStorageConfiguration.DFLT_PAGE_SIZE, - new File(U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, false), nodeFolder), - U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_MARSHALLER_PATH, false), + new File(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_BINARY_METADATA_PATH, false), nodeFolder), + U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_MARSHALLER_PATH, false), false, null, - null, null, null, null, true,true + null, null, null, null, true,true, emptyList() ); - IgniteWalConverter.convert(out, arg); + convert(out, arg); final String result = outByte.toString(); @@ -488,6 +505,53 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest { } /** + * Check that when using the "pages" argument we will see WalRecord with this pages in the utility output. + * + * @throws Exception If failed. + */ + @Test + public void testPages() throws Exception { + List<T2<PageSnapshot, String>> walRecords = new ArrayList<>(); + + String nodeDir = createWal(new ArrayList<>(), n -> { + try (WALIterator walIter = n.context().cache().context().wal().replay(new WALPointer(0, 0, 0))) { + while (walIter.hasNextX()) { + WALRecord walRecord = walIter.nextX().get2(); + + if (walRecord instanceof PageSnapshot) + walRecords.add(new T2<>((PageSnapshot)walRecord, walRecord.toString())); + } + } + }); + + assertFalse(walRecords.isEmpty()); + + File walDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_PATH, false); + assertTrue(U.fileCount(walDir.toPath()) > 0); + + File walNodeDir = new File(walDir, nodeDir); + assertTrue(U.fileCount(walNodeDir.toPath()) > 0); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PrintStream ps = new PrintStream(baos); + + T2<PageSnapshot, String> expRec = walRecords.get(0); + + IgniteWalConverterArguments args = parse( + ps, + "walDir=" + walDir.getAbsolutePath(), + "pages=" + expRec.get1().fullPageId().groupId() + ':' + expRec.get1().fullPageId().pageId(), + "skipCrc=" + true + ); + + baos.reset(); + + convert(ps, args); + + assertContains(log, baos.toString(), expRec.get2()); + } + + /** * Common part * <ul> * <li>Start node</li> @@ -496,10 +560,14 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest { * </ul> * * @param list Returns entities that have been added. + * @param afterPopulateConsumer * @return Node folder name. * @throws Exception */ - private String createWal(List<Person> list) throws Exception { + private String createWal( + List<Person> list, + @Nullable IgniteThrowableConsumer<IgniteEx> afterPopulateConsumer + ) throws Exception { String nodeFolder; try (final IgniteEx node = startGrid(0)) { @@ -523,6 +591,9 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest { list.add(value); } + + if (afterPopulateConsumer != null) + afterPopulateConsumer.accept(node); } return nodeFolder;