This is an automated email from the ASF dual-hosted git repository.
sergeychugunov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 782449d IGNITE-13687 Improvement of human-readable format of WAL
records (StandaloneWalRecordsIterator). Fix code style - Fixes #8441.
782449d is described below
commit 782449d118aeb8e1d51b6cb70237b70cbd421f88
Author: a-polyakov <polyakov.alexandr.alexandrov...@gmail.com>
AuthorDate: Tue Jan 26 15:58:11 2021 +0300
IGNITE-13687 Improvement of human-readable format of WAL records
(StandaloneWalRecordsIterator). Fix code style - Fixes #8441.
Signed-off-by: Sergey Chugunov <sergey.chugu...@gmail.com>
---
.../org/apache/ignite/binary/BinaryIdMapper.java | 2 +-
.../apache/ignite/binary/BinaryObjectBuilder.java | 2 +-
.../configuration/IgniteReflectionFactory.java | 4 +-
.../db/wal/reader/IgniteWalReaderTest.java | 7 +-
.../ignite/development/utils/DataEntryWrapper.java | 120 ++++-
.../development/utils/IgniteWalConverter.java | 183 ++++---
.../utils/IgniteWalConverterArguments.java | 491 +++++++++++++++++++
.../utils/MetastoreDataRecordWrapper.java | 13 +-
.../development/utils/DevUtilsTestSuite.java | 2 +
.../utils/IgniteWalConverterArgumentsTest.java | 411 ++++++++++++++++
.../utils/IgniteWalConverterSensitiveDataTest.java | 69 +--
.../development/utils/IgniteWalConverterTest.java | 530 +++++++++++++++++++++
.../apache/ignite/development/utils/Person.java | 65 +++
.../apache/ignite/development/utils/PersonEx.java | 79 +++
.../{DevUtilsTestSuite.java => PersonKey.java} | 46 +-
15 files changed, 1859 insertions(+), 165 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/binary/BinaryIdMapper.java
b/modules/core/src/main/java/org/apache/ignite/binary/BinaryIdMapper.java
index dac5c44..03c0886 100644
--- a/modules/core/src/main/java/org/apache/ignite/binary/BinaryIdMapper.java
+++ b/modules/core/src/main/java/org/apache/ignite/binary/BinaryIdMapper.java
@@ -50,7 +50,7 @@ public interface BinaryIdMapper {
*
* @param typeId Type ID.
* @param fieldName Field name. Filed anme is a result of {@link
BinaryNameMapper#fieldName(String)} call for an
- * initial filed name.
+ * initial field name.
* @return Field ID.
* @see BinaryNameMapper#fieldName(String)
*/
diff --git
a/modules/core/src/main/java/org/apache/ignite/binary/BinaryObjectBuilder.java
b/modules/core/src/main/java/org/apache/ignite/binary/BinaryObjectBuilder.java
index 15bd799..78740da 100644
---
a/modules/core/src/main/java/org/apache/ignite/binary/BinaryObjectBuilder.java
+++
b/modules/core/src/main/java/org/apache/ignite/binary/BinaryObjectBuilder.java
@@ -83,7 +83,7 @@ public interface BinaryObjectBuilder {
* Collections and maps returned from this method are modifiable.
*
* @param name Field name.
- * @return Filed value.
+ * @return Field value.
*/
public <T> T getField(String name);
diff --git
a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteReflectionFactory.java
b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteReflectionFactory.java
index 85b86c3..0e64f91 100644
---
a/modules/core/src/main/java/org/apache/ignite/configuration/IgniteReflectionFactory.java
+++
b/modules/core/src/main/java/org/apache/ignite/configuration/IgniteReflectionFactory.java
@@ -164,7 +164,7 @@ public class IgniteReflectionFactory<T> implements
Factory<T> {
/**
* Gets a map of properties. Map contains entries of component class field
name
- * to value of the filed which will be used as initial value.
+ * to value of the field which will be used as initial value.
*
* @return Properties.
*/
@@ -174,7 +174,7 @@ public class IgniteReflectionFactory<T> implements
Factory<T> {
/**
* Sets a map of properties. Map contains entries of component class field
name
- * to a value of the filed which will be used as initial value.
+ * to a value of the field which will be used as initial value.
*
* @param props Properties.
*/
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
index 4a7f2d0..76188f5 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
@@ -618,7 +618,7 @@ public class IgniteWalReaderTest extends
GridCommonAbstractTest {
TestStringContainerToBePrinted val = new
TestStringContainerToBePrinted(search2);
- ctrlStringsToSearch.add(val.toString()); //will validate original
toString() was called
+ ctrlStringsToSearch.add("v = [ " + val.getClass().getSimpleName() +
"{data='" + search2 + "'}]"); //will validate original toString() was called
ctrlStringsForBinaryObjSearch.add(search2);
addlCache.put("SearchValue", val);
@@ -626,7 +626,8 @@ public class IgniteWalReaderTest extends
GridCommonAbstractTest {
String search3 = "SomeTestStringContainerToBePrintedLongLine2";
TestStringContainerToBePrinted key = new
TestStringContainerToBePrinted(search3);
- ctrlStringsToSearch.add(key.toString()); //will validate original
toString() was called
+
+ ctrlStringsToSearch.add("k = " + key.getClass().getSimpleName() +
"{data='" + search3 + "'}"); //will validate original toString() was called
ctrlStringsForBinaryObjSearch.add(search3); //validate only string
itself
addlCache.put(key, "SearchKey");
@@ -676,8 +677,10 @@ public class IgniteWalReaderTest extends
GridCommonAbstractTest {
for (Iterator<String> iter = ctrlStringsToSearch.iterator();
iter.hasNext(); ) {
final String next = iter.next();
+
if (strRepresentation.contains(next)) {
iter.remove();
+
break;
}
}
diff --git
a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/DataEntryWrapper.java
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/DataEntryWrapper.java
index f96620a..49edc6c 100644
---
a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/DataEntryWrapper.java
+++
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/DataEntryWrapper.java
@@ -17,23 +17,31 @@
package org.apache.ignite.development.utils;
+import java.util.Base64;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
-import org.apache.ignite.internal.pagemem.wal.record.UnwrappedDataEntry;
+import org.apache.ignite.internal.pagemem.wal.record.UnwrapDataEntry;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.CacheObjectValueContext;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.SB;
import org.jetbrains.annotations.Nullable;
-import static java.lang.String.valueOf;
-import static java.util.Objects.isNull;
import static org.apache.ignite.development.utils.ProcessSensitiveData.HASH;
+import static org.apache.ignite.development.utils.ProcessSensitiveData.HIDE;
import static org.apache.ignite.development.utils.ProcessSensitiveData.MD5;
-import static
org.apache.ignite.development.utils.ProcessSensitiveDataUtils.md5;
/**
* Wrapper {@link DataEntry} for sensitive data output.
*/
class DataEntryWrapper extends DataEntry {
- /** Unwrapped DataEntry. */
- @Nullable private final UnwrappedDataEntry unwrappedDataEntry;
+ /**
+ * Source DataEntry.
+ */
+ @Nullable private final DataEntry source;
/** Strategy for the processing of sensitive data. */
private final ProcessSensitiveData sensitiveData;
@@ -60,30 +68,100 @@ class DataEntryWrapper extends DataEntry {
dataEntry.partitionCounter()
);
- this.sensitiveData = sensitiveData;
+ this.source = dataEntry;
- this.unwrappedDataEntry =
UnwrappedDataEntry.class.isInstance(dataEntry) ?
- (UnwrappedDataEntry) dataEntry : null;
+ this.sensitiveData = sensitiveData;
}
/** {@inheritDoc} */
@Override public String toString() {
- if (isNull(unwrappedDataEntry))
- return super.toString();
+ final String keyStr;
+ final String valueStr;
+ if (source instanceof UnwrapDataEntry) {
+ final UnwrapDataEntry unwrappedDataEntry =
(UnwrapDataEntry)this.source;
+
+ keyStr = toString(unwrappedDataEntry.unwrappedKey(),
this.source.key());
- Object key = unwrappedDataEntry.unwrappedKey();
- Object value = unwrappedDataEntry.unwrappedValue();
+ valueStr = toString(unwrappedDataEntry.unwrappedValue(),
this.source.value());
+ }
+ else {
+ keyStr = toString(null, this.source.key());
- if (HASH == sensitiveData) {
- key = valueOf(key).hashCode();
- value = valueOf(value).hashCode();
+ valueStr = toString(null, this.source.value());
}
- else if (MD5 == sensitiveData) {
- key = md5(valueOf(key));
- value = md5(valueOf(value));
+
+ return new SB(this.source.getClass().getSimpleName())
+ .a("[k = ").a(keyStr)
+ .a(", v = [").a(valueStr).a("]")
+ .a(", super = [").a(S.toString(DataEntry.class, source)).a("]]")
+ .toString();
+ }
+
+ /**
+ * Returns a string representation of the entry key or entry value.
+ *
+ * @param value unwrappedKey or unwrappedValue
+ * @param co key or value
+ * @return String presentation of the entry key or entry value depends on
{@code isValue}.
+ */
+ public String toString(Object value, CacheObject co) {
+ String str;
+ if (sensitiveData == HIDE)
+ return "";
+
+ if (sensitiveData == HASH)
+ if (value != null)
+ return Integer.toString(value.hashCode());
+ else
+ return Integer.toString(co.hashCode());
+
+ if (value instanceof String)
+ str = (String)value;
+ else if (value instanceof BinaryObject)
+ str = value.toString();
+ else if (value != null)
+ str = toStringRecursive(value.getClass(), value);
+ else if (co instanceof BinaryObject)
+ str = co.toString();
+ else
+ str = null;
+
+ if (str == null || str.isEmpty()) {
+
+ try {
+ CacheObjectValueContext ctx = null;
+ try {
+ ctx = IgniteUtils.field(source, "cacheObjValCtx");
+ }
+ catch (Exception e) {
+ throw new IgniteException(e);
+ }
+ str = Base64.getEncoder().encodeToString(co.valueBytes(ctx));
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
}
- return new SB().a(unwrappedDataEntry.getClass().getSimpleName())
- .a("[k = ").a(key).a(", v = [ ").a(value).a("], super =
[").a(super.toString()).a("]]").toString();
+ if (sensitiveData == MD5)
+ str = ProcessSensitiveDataUtils.md5(str);
+
+ return str;
+ }
+
+ /**
+ * Produces auto-generated output of string presentation for given object
(given the whole hierarchy).
+ *
+ * @param cls Declaration class of the object.
+ * @param obj Object to get a string presentation for.
+ * @return String presentation of the given object.
+ */
+ public static String toStringRecursive(Class cls, Object obj) {
+ String result = null;
+
+ if (cls != Object.class)
+ result = S.toString(cls, obj,
toStringRecursive(cls.getSuperclass(), obj));
+
+ return result;
}
}
diff --git
a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
index c4f8b5b..e0d55c7 100644
---
a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
+++
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
@@ -17,152 +17,187 @@
package org.apache.ignite.development.utils;
-import java.io.File;
+import java.io.PrintStream;
+import java.util.ArrayList;
import java.util.List;
+import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.internal.pagemem.wal.WALIterator;
import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.TimeStampRecord;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
-import
org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import
org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
import
org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
+import
org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer;
import
org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasInnerIO;
import
org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasLeafIO;
import org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO;
import org.apache.ignite.internal.processors.query.h2.database.io.H2LeafIO;
import
org.apache.ignite.internal.processors.query.h2.database.io.H2MvccInnerIO;
import org.apache.ignite.internal.processors.query.h2.database.io.H2MvccLeafIO;
+import org.apache.ignite.internal.util.IgniteUtils;
+import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.logger.NullLogger;
-import org.jetbrains.annotations.Nullable;
-
-import static java.util.stream.Collectors.toList;
-import static
org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_INCLUDE_SENSITIVE;
-import static org.apache.ignite.IgniteSystemProperties.getBoolean;
-import static org.apache.ignite.IgniteSystemProperties.getEnum;
-import static org.apache.ignite.development.utils.ProcessSensitiveData.HIDE;
-import static org.apache.ignite.development.utils.ProcessSensitiveData.SHOW;
/**
* Print WAL log data in human-readable form.
*/
public class IgniteWalConverter {
/**
- * System property for printing {@link WALRecord}. By default, {@code
false}.
- */
- static final String PRINT_RECORDS = "PRINT_RECORDS";
-
- /**
- * System property for printing {@link WalStat}. By default, {@code true}.
+ * @param args Args.
+ * @throws Exception If failed.
*/
- static final String PRINT_STAT = "PRINT_STAT";
+ public static void main(String[] args) {
+ final IgniteWalConverterArguments parameters =
IgniteWalConverterArguments.parse(System.out, args);
- /**
- * System property for setting {@link ProcessSensitiveData strategy} of
output sensitive data.
- * By default, {@link ProcessSensitiveData#SHOW}.
- */
- static final String SENSITIVE_DATA = "SENSITIVE_DATA";
+ if (parameters != null)
+ convert(System.out, parameters);
+ }
/**
- * @param args Args.
- * @throws Exception If failed.
+ * Write to out WAL log data in human-readable form.
+ *
+ * @param out Receiver of result.
+ * @param params Parameters.
*/
- public static void main(String[] args) throws Exception {
- if (args.length < 2)
- throw new IllegalArgumentException("\nYou need to provide:\n" +
- "\t1. Size of pages, which was selected for file store
(1024, 2048, 4096, etc).\n" +
- "\t2. Path to dir with wal files.\n" +
- "\t3. (Optional) Path to dir with archive wal files.");
-
+ public static void convert(final PrintStream out, final
IgniteWalConverterArguments params) {
PageIO.registerH2(H2InnerIO.VERSIONS, H2LeafIO.VERSIONS,
H2MvccInnerIO.VERSIONS, H2MvccLeafIO.VERSIONS);
H2ExtrasInnerIO.register();
H2ExtrasLeafIO.register();
- boolean printRecords = getBoolean(PRINT_RECORDS, false); //TODO read
them from argumetns
- boolean printStat = getBoolean(PRINT_STAT, true); //TODO read them
from argumetns
- ProcessSensitiveData sensitiveData = getEnum(SENSITIVE_DATA, SHOW);
//TODO read them from argumetns
+
System.setProperty(IgniteSystemProperties.IGNITE_TO_STRING_INCLUDE_SENSITIVE,
+ Boolean.toString(params.getProcessSensitiveData() ==
ProcessSensitiveData.HIDE));
+
+ System.setProperty(IgniteSystemProperties.IGNITE_PDS_SKIP_CRC,
Boolean.toString(params.isSkipCrc()));
+ RecordV1Serializer.skipCrc = params.isSkipCrc();
- if (printRecords && HIDE == sensitiveData)
- System.setProperty(IGNITE_TO_STRING_INCLUDE_SENSITIVE,
Boolean.FALSE.toString());
+ System.setProperty(IgniteSystemProperties.IGNITE_TO_STRING_MAX_LENGTH,
String.valueOf(Integer.MAX_VALUE));
- final IgniteWalIteratorFactory factory = new
IgniteWalIteratorFactory(new NullLogger());
+ final WalStat stat = params.isPrintStat() ? new WalStat() : null;
- final File walWorkDirWithConsistentId = new File(args[1]);
+ IgniteWalIteratorFactory.IteratorParametersBuilder
iteratorParametersBuilder = new
IgniteWalIteratorFactory.IteratorParametersBuilder()
+ .pageSize(params.getPageSize())
+ .binaryMetadataFileStoreDir(params.getBinaryMetadataFileStoreDir())
+
.marshallerMappingFileStoreDir(params.getMarshallerMappingFileStoreDir())
+ .keepBinary(params.isKeepBinary());
- final File[] workFiles =
walWorkDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER);
+ if (params.getWalDir() != null)
+ iteratorParametersBuilder.filesOrDirs(params.getWalDir());
- if (workFiles == null)
- throw new IllegalArgumentException("No .wal files in dir: " +
args[1]);
+ if (params.getWalArchiveDir() != null)
+ iteratorParametersBuilder.filesOrDirs(params.getWalArchiveDir());
- @Nullable final WalStat stat = printStat ? new WalStat() : null;
+ final IgniteWalIteratorFactory factory = new
IgniteWalIteratorFactory();
- IgniteWalIteratorFactory.IteratorParametersBuilder
iteratorParametersBuilder =
- new
IgniteWalIteratorFactory.IteratorParametersBuilder().filesOrDirs(workFiles)
- .pageSize(Integer.parseInt(args[0]));
+ boolean printAlways = F.isEmpty(params.getRecordTypes());
try (WALIterator stIt = factory.iterator(iteratorParametersBuilder)) {
+ String currentWalPath = null;
+
while (stIt.hasNextX()) {
+ final String currentRecordWalPath =
getCurrentWalFilePath(stIt);
+
+ if (currentWalPath == null ||
!currentWalPath.equals(currentRecordWalPath)) {
+ out.println("File: " + currentRecordWalPath);
+
+ currentWalPath = currentRecordWalPath;
+ }
+
IgniteBiTuple<WALPointer, WALRecord> next = stIt.nextX();
final WALPointer pointer = next.get1();
+
final WALRecord record = next.get2();
if (stat != null)
stat.registerRecord(record, pointer, true);
- if (printRecords)
- System.out.println("[W] " + toString(record,
sensitiveData));
+ if (printAlways ||
params.getRecordTypes().contains(record.type())) {
+ boolean print = true;
+
+ if (record instanceof TimeStampRecord)
+ print = withinTimeRange((TimeStampRecord) record,
params.getFromTime(), params.getToTime());
+
+ final String recordStr = toString(record,
params.getProcessSensitiveData());
+
+ if (print && (F.isEmpty(params.getRecordContainsText()) ||
recordStr.contains(params.getRecordContainsText())))
+ out.println(recordStr);
+ }
}
}
+ catch (Exception e) {
+ e.printStackTrace(out);
+ }
- if (args.length >= 3) {
- final File walArchiveDirWithConsistentId = new File(args[2]);
+ if (stat != null)
+ out.println("Statistic collected:\n" + stat.toString());
+ }
- try (WALIterator stIt =
factory.iterator(walArchiveDirWithConsistentId)) {
- while (stIt.hasNextX()) {
- IgniteBiTuple<WALPointer, WALRecord> next = stIt.nextX();
+ /**
+ * Checks if provided TimeStampRecord is within time range.
+ *
+ * @param rec Record.
+ * @param fromTime Lower bound for timestamp.
+ * @param toTime Upper bound for timestamp;
+ * @return {@code True} if timestamp is within range.
+ */
+ private static boolean withinTimeRange(TimeStampRecord rec, Long fromTime,
Long toTime) {
+ if (fromTime != null && rec.timestamp() < fromTime)
+ return false;
- final WALPointer pointer = next.get1();
- final WALRecord record = next.get2();
+ if (toTime != null && rec.timestamp() > toTime)
+ return false;
- if (stat != null)
- stat.registerRecord(record, pointer, false);
+ return true;
+ }
- if (printRecords)
- System.out.println("[A] " + toString(record,
sensitiveData));
- }
- }
- }
+ /**
+ * Get current wal file path, used in {@code WALIterator}
+ *
+ * @param it WALIterator.
+ * @return Current wal file path.
+ */
+ private static String getCurrentWalFilePath(WALIterator it) {
+ String result = null;
- System.err.flush();
+ try {
+ final Integer curIdx = IgniteUtils.field(it, "curIdx");
- if (stat != null)
- System.out.println("Statistic collected:\n" + stat.toString());
+ final List<FileDescriptor> walFileDescriptors =
IgniteUtils.field(it, "walFileDescriptors");
+
+ if (curIdx != null && walFileDescriptors != null &&
!walFileDescriptors.isEmpty())
+ result = walFileDescriptors.get(curIdx).getAbsolutePath();
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return result;
}
/**
* Converting {@link WALRecord} to a string with sensitive data.
*
- * @param walRecord Instance of {@link WALRecord}.
+ * @param walRecord Instance of {@link WALRecord}.
* @param sensitiveData Strategy for processing of sensitive data.
* @return String representation of {@link WALRecord}.
*/
private static String toString(WALRecord walRecord, ProcessSensitiveData
sensitiveData) {
- if (SHOW == sensitiveData || HIDE == sensitiveData)
- return walRecord.toString();
+ if (walRecord instanceof DataRecord) {
+ final DataRecord dataRecord = (DataRecord)walRecord;
- if (MetastoreDataRecord.class.isInstance(walRecord))
- walRecord = new
MetastoreDataRecordWrapper((MetastoreDataRecord)walRecord, sensitiveData);
- else if (DataRecord.class.isInstance(walRecord)) {
- DataRecord dataRecord = (DataRecord)walRecord;
+ final List<DataEntry> entryWrappers = new
ArrayList<>(dataRecord.writeEntries().size());
- List<DataEntry> entryWrappers = dataRecord.writeEntries().stream()
- .map(dataEntry -> new DataEntryWrapper(dataEntry,
sensitiveData)).collect(toList());
+ for (DataEntry dataEntry : dataRecord.writeEntries())
+ entryWrappers.add(new DataEntryWrapper(dataEntry,
sensitiveData));
dataRecord.setWriteEntries(entryWrappers);
}
+ else if (walRecord instanceof MetastoreDataRecord)
+ walRecord = new
MetastoreDataRecordWrapper((MetastoreDataRecord)walRecord, sensitiveData);
return walRecord.toString();
}
diff --git
a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverterArguments.java
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverterArguments.java
new file mode 100644
index 0000000..af3bfb8
--- /dev/null
+++
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverterArguments.java
@@ -0,0 +1,491 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.development.utils;
+
+import java.io.File;
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.util.typedef.F;
+
+/**
+ * Parameters for IgniteWalConverter with parsed and validated.
+ */
+public class IgniteWalConverterArguments {
+ /** */
+ private static final String WAL_DIR = "walDir";
+
+ /** */
+ private static final String WAL_ARCHIVE_DIR = "walArchiveDir";
+
+ /** */
+ private static final String PAGE_SIZE = "pageSize";
+
+ /** */
+ private static final String BINARY_METADATA_FILE_STORE_DIR =
"binaryMetadataFileStoreDir";
+
+ /** */
+ private static final String MARSHALLER_MAPPING_FILE_STORE_DIR =
"marshallerMappingFileStoreDir";
+
+ /** */
+ private static final String KEEP_BINARY = "keepBinary";
+
+ /** */
+ private static final String RECORD_TYPES = "recordTypes";
+
+ /** */
+ private static final String WAL_TIME_FROM_MILLIS = "walTimeFromMillis";
+
+ /** */
+ private static final String WAL_TIME_TO_MILLIS = "walTimeToMillis";
+
+ /** */
+ private static final String RECORD_CONTAINS_TEXT = "recordContainsText";
+
+ /** */
+ private static final String PROCESS_SENSITIVE_DATA =
"processSensitiveData";
+
+ /** */
+ private static final String PRINT_STAT = "printStat";
+
+ /** */
+ private static final String SKIP_CRC = "skipCrc";
+
+ /** Path to dir with wal files. */
+ private final File walDir;
+
+ /** Path to dir with archive wal files. */
+ private final File walArchiveDir;
+
+ /** Size of pages, which was selected for file store (1024, 2048, 4096,
etc). */
+ private final int pageSize;
+
+ /** Path to binary metadata dir. */
+ private final File binaryMetadataFileStoreDir;
+
+ /** Path to marshaller dir. */
+ private final File marshallerMappingFileStoreDir;
+
+ /** Keep binary flag. */
+ private final boolean keepBinary;
+
+ /** WAL record types (TX_RECORD, DATA_RECORD, etc). */
+ private final Set<WALRecord.RecordType> recordTypes;
+
+ /** The start time interval for the record time in milliseconds. */
+ private final Long fromTime;
+
+ /** The end time interval for the record time in milliseconds. */
+ private final Long toTime;
+
+ /** Filter by substring in the WAL record. */
+ private final String recordContainsText;
+
+ /** Strategy for the processing of sensitive data (SHOW, HIDE, HASH, MD5).
*/
+ private final ProcessSensitiveData processSensitiveData;
+
+ /** Write summary statistics for WAL */
+ private final boolean printStat;
+
+ /** Skip CRC calculation/check flag */
+ private final boolean skipCrc;
+
+ /**
+ * @param walDir Path to dir with wal files.
+ * @param walArchiveDir Path to dir with archive wal files.
+ * @param pageSize Size of pages, which was selected
for file store (1024, 2048, 4096, etc).
+ * @param binaryMetadataFileStoreDir Path to binary metadata dir.
+ * @param marshallerMappingFileStoreDir Path to marshaller dir.
+ * @param keepBinary Keep binary flag.
+ * @param recordTypes WAL record types (TX_RECORD,
DATA_RECORD, etc).
+ * @param fromTime The start time interval for the
record time in milliseconds.
+ * @param toTime The end time interval for the
record time in milliseconds.
+ * @param recordContainsText Filter by substring in the WAL
record.
+ * @param processSensitiveData Strategy for the processing of
sensitive data (SHOW, HIDE, HASH, MD5).
+ * @param printStat Write summary statistics for WAL.
+ * @param skipCrc Skip CRC calculation/check flag.
+ */
+ public IgniteWalConverterArguments(File walDir, File walArchiveDir, int
pageSize,
+ File binaryMetadataFileStoreDir, File marshallerMappingFileStoreDir,
boolean keepBinary,
+ Set<WALRecord.RecordType> recordTypes, Long fromTime, Long toTime,
String recordContainsText,
+ ProcessSensitiveData processSensitiveData,
+ boolean printStat, boolean skipCrc) {
+ this.walDir = walDir;
+ this.walArchiveDir = walArchiveDir;
+ this.pageSize = pageSize;
+ this.binaryMetadataFileStoreDir = binaryMetadataFileStoreDir;
+ this.marshallerMappingFileStoreDir = marshallerMappingFileStoreDir;
+ this.keepBinary = keepBinary;
+ this.recordTypes = recordTypes;
+ this.fromTime = fromTime;
+ this.toTime = toTime;
+ this.recordContainsText = recordContainsText;
+ this.processSensitiveData = processSensitiveData;
+ this.printStat = printStat;
+ this.skipCrc = skipCrc;
+ }
+
+ /**
+ * Path to dir with wal files.
+ *
+ * @return walDir
+ */
+ public File getWalDir() {
+ return walDir;
+ }
+
+ /**
+ * Path to dir with archive wal files.
+ *
+ * @return walArchiveDir
+ */
+ public File getWalArchiveDir() {
+ return walArchiveDir;
+ }
+
+ /**
+ * Size of pages, which was selected for file store (1024, 2048, 4096,
etc).
+ *
+ * @return pageSize
+ */
+ public int getPageSize() {
+ return pageSize;
+ }
+
+ /**
+ * Path to binary metadata dir.
+ *
+ * @return binaryMetadataFileStoreD
+ */
+ public File getBinaryMetadataFileStoreDir() {
+ return binaryMetadataFileStoreDir;
+ }
+
+ /**
+ * Path to marshaller dir.
+ *
+ * @return marshallerMappingFileStoreD
+ */
+ public File getMarshallerMappingFileStoreDir() {
+ return marshallerMappingFileStoreDir;
+ }
+
+ /**
+ * Keep binary flag.
+ *
+ * @return keepBina
+ */
+ public boolean isKeepBinary() {
+ return keepBinary;
+ }
+
+ /**
+ * WAL record types (TX_RECORD, DATA_RECORD, etc).
+ *
+ * @return recordTypes
+ */
+ public Set<WALRecord.RecordType> getRecordTypes() {
+ return recordTypes;
+ }
+
+ /**
+ * The start time interval for the record time in milliseconds.
+ *
+ * @return fromTime
+ */
+ public Long getFromTime() {
+ return fromTime;
+ }
+
+ /**
+ * The end time interval for the record time in milliseconds.
+ *
+ * @return toTime
+ */
+ public Long getToTime() {
+ return toTime;
+ }
+
+ /**
+ * Filter by substring in the WAL record.
+ *
+ * @return recordContainsText
+ */
+ public String getRecordContainsText() {
+ return recordContainsText;
+ }
+
+ /**
+ * Strategy for the processing of sensitive data (SHOW, HIDE, HASH, MD5).
+ *
+ * @return processSensitiveData
+ */
+ public ProcessSensitiveData getProcessSensitiveData() {
+ return processSensitiveData;
+ }
+
+ /**
+ * Write summary statistics for WAL.
+ *
+ * @return printStat
+ */
+ public boolean isPrintStat() {
+ return printStat;
+ }
+
+ /**
+ * Skip CRC calculation/check flag.
+ *
+ * @return skipCrc
+ */
+ public boolean isSkipCrc() {
+ return skipCrc;
+ }
+
+ /**
+ * Parse command line arguments and return filled
IgniteWalConverterArguments
+ *
+ * @param args Command line arguments.
+ * @return IgniteWalConverterArguments.
+ */
+ public static IgniteWalConverterArguments parse(final PrintStream out,
String args[]) {
+ if (args == null || args.length < 1) {
+ out.println("Print WAL log data in human-readable form.");
+ out.println("You need to provide:");
+ out.println(" walDir Path to dir with
wal files.");
+ out.println(" walArchiveDir Path to dir with
archive wal files. walDir or walArchiveDir must be specified.");
+ out.println(" pageSize Size of pages,
which was selected for file store (1024, 2048, 4096, etc). Default 4096.");
+ out.println(" binaryMetadataFileStoreDir (Optional) Path
to binary meta.");
+ out.println(" marshallerMappingFileStoreDir (Optional) Path
to marshaller dir.");
+ out.println(" keepBinary Keep binary
flag. Default true.");
+ out.println(" recordTypes (Optional)
Comma-separated WAL record types (TX_RECORD, DATA_RECORD, etc). Default all.");
+ out.println(" walTimeFromMillis (Optional) The
start time interval for the record time in milliseconds.");
+ out.println(" walTimeToMillis (Optional) The
end time interval for the record time in milliseconds.");
+ out.println(" recordContainsText (Optional)
Filter by substring in the WAL record.");
+ out.println(" processSensitiveData (Optional)
Strategy for the processing of sensitive data (SHOW, HIDE, HASH, MD5). Default
SHOW.");
+ out.println(" printStat Write summary
statistics for WAL. Default false.");
+ out.println(" skipCrc Skip CRC
calculation/check flag. Default false.");
+ out.println("For example:");
+ out.println(" walDir=/work/db/wal");
+ out.println(" walArchiveDir=/work/db/wal_archive");
+ out.println(" pageSize=4096");
+ out.println("
binaryMetadataFileStoreDir=/work/db/nodeId-consistentId");
+ out.println("
marshallerMappingFileStoreDir=/work/db/marshaller");
+ out.println(" keepBinary=true");
+ out.println(" recordTypes=DataRecord,TxRecord");
+ out.println(" walTimeFromMillis=1575158400000");
+ out.println(" walTimeToMillis=1577836740999");
+ out.println(" recordContainsText=search string");
+ out.println(" processSensitiveData=SHOW");
+ out.println(" skipCrc=true");
+ return null;
+ }
+
+ File walDir = null;
+ File walArchiveDir = null;
+ int pageSize = 4096;
+ File binaryMetadataFileStoreDir = null;
+ File marshallerMappingFileStoreDir = null;
+ boolean keepBinary = true;
+ final Set<WALRecord.RecordType> recordTypes = new HashSet<>();
+ Long fromTime = null;
+ Long toTime = null;
+ String recordContainsText = null;
+ ProcessSensitiveData processSensitiveData = ProcessSensitiveData.SHOW;
+ boolean printStat = false;
+ boolean skipCrc = false;
+
+ for (String arg : args) {
+ if (arg.startsWith(WAL_DIR + "=")) {
+ final String walPath = arg.substring(WAL_DIR.length() + 1);
+
+ walDir = new File(walPath);
+
+ if (!walDir.exists())
+ throw new IllegalArgumentException("Incorrect path to dir
with wal files: " + walPath);
+ }
+ else if (arg.startsWith(WAL_ARCHIVE_DIR + "=")) {
+ final String walArchivePath =
arg.substring(WAL_ARCHIVE_DIR.length() + 1);
+
+ walArchiveDir = new File(walArchivePath);
+
+ if (!walArchiveDir.exists())
+ throw new IllegalArgumentException("Incorrect path to dir
with archive wal files: " + walArchivePath);
+ }
+ else if (arg.startsWith(PAGE_SIZE + "=")) {
+ final String pageSizeStr = arg.substring(PAGE_SIZE.length() +
1);
+
+ try {
+ pageSize = Integer.parseInt(pageSizeStr);
+ }
+ catch (Exception e) {
+ throw new IllegalArgumentException("Incorrect page size.
Error parse: " + pageSizeStr);
+ }
+ }
+ else if (arg.startsWith(BINARY_METADATA_FILE_STORE_DIR + "=")) {
+ final String binaryMetadataFileStorePath =
arg.substring(BINARY_METADATA_FILE_STORE_DIR.length() + 1);
+
+ binaryMetadataFileStoreDir = new
File(binaryMetadataFileStorePath);
+
+ if (!binaryMetadataFileStoreDir.isDirectory())
+ throw new IllegalArgumentException("Incorrect path to dir
with binary meta files: " + binaryMetadataFileStorePath);
+ }
+ else if (arg.startsWith(MARSHALLER_MAPPING_FILE_STORE_DIR + "=")) {
+ final String marshallerMappingFileStorePath =
arg.substring(MARSHALLER_MAPPING_FILE_STORE_DIR.length() + 1);
+
+ marshallerMappingFileStoreDir = new
File(marshallerMappingFileStorePath);
+
+ if (!marshallerMappingFileStoreDir.isDirectory())
+ throw new IllegalArgumentException("Incorrect path to dir
with marshaller files: " + marshallerMappingFileStorePath);
+ }
+ else if (arg.startsWith(KEEP_BINARY + "=")) {
+ keepBinary = parseBoolean(KEEP_BINARY,
arg.substring(KEEP_BINARY.length() + 1));
+ }
+ else if (arg.startsWith(RECORD_TYPES + "=")) {
+ final String recordTypesStr =
arg.substring(RECORD_TYPES.length() + 1);
+
+ final String[] recordTypesStrArray = recordTypesStr.split(",");
+
+ final SortedSet<String> unknownRecordTypes = new TreeSet<>();
+
+ for (String recordTypeStr : recordTypesStrArray) {
+ try {
+
recordTypes.add(WALRecord.RecordType.valueOf(recordTypeStr));
+ }
+ catch (Exception e) {
+ unknownRecordTypes.add(recordTypeStr);
+ }
+ }
+
+ if (!unknownRecordTypes.isEmpty())
+ throw new IllegalArgumentException("Unknown record types:
" + unknownRecordTypes +
+ ". Supported record types: " +
Arrays.toString(WALRecord.RecordType.values()));
+ }
+ else if (arg.startsWith(WAL_TIME_FROM_MILLIS + "=")) {
+ final String fromTimeStr =
arg.substring(WAL_TIME_FROM_MILLIS.length() + 1);
+
+ try {
+ fromTime = Long.parseLong(fromTimeStr);
+ }
+ catch (Exception e) {
+ throw new IllegalArgumentException("Incorrect
walTimeFromMillis. Error parse: " + fromTimeStr);
+ }
+ }
+ else if (arg.startsWith(WAL_TIME_TO_MILLIS + "=")) {
+ final String toTimeStr =
arg.substring(WAL_TIME_TO_MILLIS.length() + 1);
+
+ try {
+ toTime = Long.parseLong(toTimeStr);
+ }
+ catch (Exception e) {
+ throw new IllegalArgumentException("Incorrect
walTimeToMillis. Error parse: " + toTimeStr);
+ }
+ }
+ else if (arg.startsWith(RECORD_CONTAINS_TEXT + "=")) {
+ recordContainsText =
arg.substring(RECORD_CONTAINS_TEXT.length() + 1);
+ }
+ else if (arg.startsWith(PROCESS_SENSITIVE_DATA + "=")) {
+ final String processSensitiveDataStr =
arg.substring(PROCESS_SENSITIVE_DATA.length() + 1);
+ try {
+ processSensitiveData =
ProcessSensitiveData.valueOf(processSensitiveDataStr);
+ }
+ catch (Exception e) {
+ throw new IllegalArgumentException("Unknown
processSensitiveData: " + processSensitiveDataStr +
+ ". Supported: " +
Arrays.toString(ProcessSensitiveData.values()));
+ }
+ }
+ else if (arg.startsWith(PRINT_STAT + "=")) {
+ printStat = parseBoolean(PRINT_STAT,
arg.substring(PRINT_STAT.length() + 1));
+ }
+ else if (arg.startsWith(SKIP_CRC + "=")) {
+ skipCrc = parseBoolean(SKIP_CRC,
arg.substring(SKIP_CRC.length() + 1));
+ }
+ }
+
+ if (walDir == null && walArchiveDir == null)
+ throw new IllegalArgumentException("The paths to the WAL files are
not specified.");
+
+ out.println("Program arguments:");
+
+ if (walDir != null)
+ out.printf("\t%s = %s\n", WAL_DIR, walDir.getAbsolutePath());
+
+ if (walArchiveDir != null)
+ out.printf("\t%s = %s\n", WAL_ARCHIVE_DIR,
walArchiveDir.getAbsolutePath());
+
+ out.printf("\t%s = %d\n", PAGE_SIZE, pageSize);
+
+ if (binaryMetadataFileStoreDir != null)
+ out.printf("\t%s = %s\n", BINARY_METADATA_FILE_STORE_DIR,
binaryMetadataFileStoreDir);
+
+ if (marshallerMappingFileStoreDir != null)
+ out.printf("\t%s = %s\n", MARSHALLER_MAPPING_FILE_STORE_DIR,
marshallerMappingFileStoreDir);
+
+ out.printf("\t%s = %s\n", KEEP_BINARY, keepBinary);
+
+ if (!F.isEmpty(recordTypes))
+ out.printf("\t%s = %s\n", RECORD_TYPES, recordTypes);
+
+ if (fromTime != null)
+ out.printf("\t%s = %s\n", WAL_TIME_FROM_MILLIS, new
Date(fromTime));
+
+ if (toTime != null)
+ out.printf("\t%s = %s\n", WAL_TIME_TO_MILLIS, new Date(toTime));
+
+ if (recordContainsText != null)
+ out.printf("\t%s = %s\n", RECORD_CONTAINS_TEXT,
recordContainsText);
+
+ out.printf("\t%s = %b\n", PRINT_STAT, printStat);
+
+ out.printf("\t%s = %b\n", SKIP_CRC, skipCrc);
+
+ return new IgniteWalConverterArguments(walDir, walArchiveDir, pageSize,
+ binaryMetadataFileStoreDir, marshallerMappingFileStoreDir,
+ keepBinary, recordTypes, fromTime, toTime, recordContainsText,
processSensitiveData, printStat, skipCrc);
+ }
+
+ /**
+ * Parses the string argument as a boolean. The {@code boolean}
+ * returned represents the value {@code true} if the string argument
+ * is not {@code null} and is equal, ignoring case, to the string
+ * {@code "true"}, returned value {@code false} if the string argument
+ * is not {@code null} and is equal, ignoring case, to the string
+ * {@code "false"}, else throw IllegalArgumentException<p>
+ *
+ * @param name parameter name of boolean type.
+ * @param value the {@code String} containing the boolean representation
to be parsed.
+ * @return the boolean represented by the string argument
+ *
+ */
+ private static boolean parseBoolean(String name, String value) {
+ if (value == null)
+ throw new IllegalArgumentException("Null value passed for flag " +
name);
+
+ if (value.equalsIgnoreCase(Boolean.TRUE.toString()))
+ return true;
+ else if (value.equalsIgnoreCase(Boolean.FALSE.toString()))
+ return false;
+ else
+ throw new IllegalArgumentException("Incorrect flag " + name + ",
valid value: true or false. Error parse: " + value);
+ }
+}
diff --git
a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/MetastoreDataRecordWrapper.java
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/MetastoreDataRecordWrapper.java
index f32c8f4..f76a99a 100644
---
a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/MetastoreDataRecordWrapper.java
+++
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/MetastoreDataRecordWrapper.java
@@ -23,6 +23,7 @@ import
org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
import static java.lang.String.valueOf;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.ignite.development.utils.ProcessSensitiveData.HASH;
+import static org.apache.ignite.development.utils.ProcessSensitiveData.HIDE;
import static org.apache.ignite.development.utils.ProcessSensitiveData.MD5;
import static
org.apache.ignite.development.utils.ProcessSensitiveDataUtils.md5;
@@ -38,11 +39,13 @@ class MetastoreDataRecordWrapper extends
MetastoreDataRecord {
*/
public MetastoreDataRecordWrapper(MetastoreDataRecord metastoreRecord,
ProcessSensitiveData sensitiveData) {
super(
- HASH == sensitiveData ? valueOf(metastoreRecord.key().hashCode()) :
- MD5 == sensitiveData ? md5(metastoreRecord.key()) :
metastoreRecord.key(),
- HASH == sensitiveData ?
valueOf(Arrays.hashCode(metastoreRecord.value())).getBytes(UTF_8) :
- MD5 == sensitiveData ?
md5(Arrays.toString(metastoreRecord.value())).getBytes(UTF_8) :
- metastoreRecord.value()
+ HIDE == sensitiveData ? "" :
+ HASH == sensitiveData ?
valueOf(metastoreRecord.key().hashCode()) :
+ MD5 == sensitiveData ? md5(metastoreRecord.key()) :
metastoreRecord.key(),
+ HIDE == sensitiveData ? new byte[0] :
+ HASH == sensitiveData ?
valueOf(Arrays.hashCode(metastoreRecord.value())).getBytes(UTF_8) :
+ MD5 == sensitiveData ?
md5(Arrays.toString(metastoreRecord.value())).getBytes(UTF_8) :
+ metastoreRecord.value()
);
size(metastoreRecord.size());
diff --git
a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/DevUtilsTestSuite.java
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/DevUtilsTestSuite.java
index 5f424d9..919fb9b 100644
---
a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/DevUtilsTestSuite.java
+++
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/DevUtilsTestSuite.java
@@ -25,6 +25,8 @@ import org.junit.runners.Suite;
*/
@RunWith(Suite.class)
@Suite.SuiteClasses({
+ IgniteWalConverterTest.class,
+ IgniteWalConverterArgumentsTest.class,
IgniteWalConverterSensitiveDataTest.class
})
public class DevUtilsTestSuite {
diff --git
a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterArgumentsTest.java
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterArgumentsTest.java
new file mode 100644
index 0000000..84098fb
--- /dev/null
+++
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterArgumentsTest.java
@@ -0,0 +1,411 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.development.utils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test for IgniteWalConverterArguments
+ */
+public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
+ /**
+ *
+ */
+ public IgniteWalConverterArgumentsTest() {
+ super(false);
+ }
+
+ /**
+ * View help
+ * <ul>
+ * <li>Read wal with out params</li>
+ * </ul>
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testViewHelp() throws Exception {
+ final ByteArrayOutputStream out = new ByteArrayOutputStream();
+
+ final IgniteWalConverterArguments parseArgs =
IgniteWalConverterArguments.parse(new PrintStream(out), null);
+
+ Assert.assertNull(parseArgs);
+
+ final String help = out.toString();
+
+ Assert.assertTrue(help.startsWith("Print WAL log data in
human-readable form."));
+
+ for (final Field field :
IgniteWalConverterArguments.class.getDeclaredFields()) {
+ if (Modifier.isStatic(field.getModifiers())
+ && Modifier.isStatic(field.getModifiers())
+ && field.getType() == String.class) {
+ field.setAccessible(true);
+
+ final String arg = (String)field.get(null);
+
+ Assert.assertTrue(help.contains(" " + arg + " "));
+ }
+ }
+ }
+
+ /**
+ * Checking whether fields "walDir" or "walArchiveDir" are mandatory.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testRequiredWalDir() throws Exception {
+ GridTestUtils.assertThrows(log, () -> {
+ IgniteWalConverterArguments.parse(System.out, new String[]
{"pageSize=4096"});
+ }, IgniteException.class, "The paths to the WAL files are not
specified.");
+ }
+
+ /**
+ * Checking whether field "walDir" are incorrect.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIncorrectWalDir() throws Exception {
+ GridTestUtils.assertThrows(log, () -> {
+ IgniteWalConverterArguments.parse(System.out, new String[]
{"walDir=non_existing_path"});
+ }, IgniteException.class, "Incorrect path to dir with wal files:
non_existing_path");
+ }
+
+ /**
+ * Checking whether field "walArchiveDir" are incorrect.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIncorrectWalArchiveDir() throws Exception {
+ GridTestUtils.assertThrows(log, () -> {
+ IgniteWalConverterArguments.parse(System.out, new String[]
{"walArchiveDir=non_existing_path"});
+ }, IgniteException.class, "Incorrect path to dir with archive wal
files: non_existing_path");
+ }
+
+ /**
+ * Checking whether field "pageSize" are incorrect.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIncorrectPageSize() throws Exception {
+ GridTestUtils.assertThrows(log, () -> {
+ final File wal = File.createTempFile("wal", "");
+ wal.deleteOnExit();
+
+ final String[] args = {
+ "walDir=" + wal.getAbsolutePath(),
+ "pageSize=not_integer"
+ };
+
+ IgniteWalConverterArguments.parse(System.out, args);
+ }, IgniteException.class, "Incorrect page size. Error parse:
not_integer");
+ }
+
+ /**
+ * Checking whether field "binaryMetadataFileStoreDir" are incorrect.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIncorrectBinaryMetadataFileStoreDir() throws Exception {
+ GridTestUtils.assertThrows(log, () -> {
+ final File wal = File.createTempFile("wal", "");
+ wal.deleteOnExit();
+
+ final String[] args = {
+ "walDir=" + wal.getAbsolutePath(),
+ "binaryMetadataFileStoreDir=non_existing_path"
+ };
+
+ IgniteWalConverterArguments.parse(System.out, args);
+ }, IgniteException.class, "Incorrect path to dir with binary meta
files: non_existing_path");
+ }
+
+ /**
+ * Checking whether field "marshallerMappingFileStoreDir" are incorrect.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIncorrectMarshallerMappingFileStoreDir() throws Exception {
+ GridTestUtils.assertThrows(log, () -> {
+ final File wal = File.createTempFile("wal", "");
+ wal.deleteOnExit();
+
+ final String[] args = {
+ "walDir=" + wal.getAbsolutePath(),
+ "marshallerMappingFileStoreDir=non_existing_path"
+ };
+
+ IgniteWalConverterArguments.parse(System.out, args);
+ }, IgniteException.class, "Incorrect path to dir with marshaller
files: non_existing_path");
+ }
+
+ /**
+ * Checking whether field "keepBinary" are incorrect.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIncorrectKeepBinary() throws Exception {
+ GridTestUtils.assertThrows(log, () -> {
+ final File wal = File.createTempFile("wal", "");
+ wal.deleteOnExit();
+
+ final String[] args = {
+ "walDir=" + wal.getAbsolutePath(),
+ "keepBinary=not_boolean"
+ };
+
+ IgniteWalConverterArguments.parse(System.out, args);
+ }, IgniteException.class, "Incorrect flag keepBinary, valid value:
true or false. Error parse: not_boolean");
+ }
+
+ /**
+ * Checking whether field "recordTypes" are incorrect.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIncorrectRecordTypes() throws Exception {
+ GridTestUtils.assertThrows(log, () -> {
+ final File wal = File.createTempFile("wal", "");
+ wal.deleteOnExit();
+
+ final String[] args = {
+ "walDir=" + wal.getAbsolutePath(),
+ "recordTypes=not_exist"
+ };
+
+ IgniteWalConverterArguments.parse(System.out, args);
+ }, IgniteException.class, "Unknown record types: [not_exist].");
+ }
+
+ /**
+ * Checking whether field "recordTypes" are incorrect several value.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIncorrectSeveralRecordTypes() throws Exception {
+ GridTestUtils.assertThrows(log, () -> {
+ final File wal = File.createTempFile("wal", "");
+ wal.deleteOnExit();
+
+ final String[] args = {
+ "walDir=" + wal.getAbsolutePath(),
+ "recordTypes=not_exist1,not_exist2"
+ };
+
+ IgniteWalConverterArguments.parse(System.out, args);
+ }, IgniteException.class, "Unknown record types: [not_exist1,
not_exist2].");
+ }
+
+ /**
+ * Checking whether field "walTimeFromMillis" are incorrect.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIncorrectWalTimeFromMillis() throws Exception {
+ GridTestUtils.assertThrows(log, () -> {
+ final File wal = File.createTempFile("wal", "");
+ wal.deleteOnExit();
+
+ final String[] args = {
+ "walDir=" + wal.getAbsolutePath(),
+ "walTimeFromMillis=not_long"
+ };
+
+ IgniteWalConverterArguments.parse(System.out, args);
+ }, IgniteException.class, "Incorrect walTimeFromMillis. Error parse:
not_long");
+ }
+
+ /**
+ * Checking whether field "walTimeToMillis" are incorrect.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIncorrectWalTimeToMillis() throws Exception {
+ GridTestUtils.assertThrows(log, () -> {
+ final File wal = File.createTempFile("wal", "");
+ wal.deleteOnExit();
+
+ final String[] args = {
+ "walDir=" + wal.getAbsolutePath(),
+ "walTimeToMillis=not_long"
+ };
+
+ IgniteWalConverterArguments.parse(System.out, args);
+ }, IgniteException.class, "Incorrect walTimeToMillis. Error parse:
not_long");
+ }
+
+ /**
+ * Checking whether field "processSensitiveData" are incorrect.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIncorrectProcessSensitiveData() throws Exception {
+ GridTestUtils.assertThrows(log, () -> {
+ final File wal = File.createTempFile("wal", "");
+ wal.deleteOnExit();
+
+ final String[] args = {
+ "walDir=" + wal.getAbsolutePath(),
+ "processSensitiveData=unknown"
+ };
+
+ IgniteWalConverterArguments.parse(System.out, args);
+ }, IgniteException.class, "Unknown processSensitiveData: unknown.
Supported: ");
+ }
+
+ /**
+ * Checking whether field "printStat" are incorrect.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIncorrectPrintStat() throws Exception {
+ GridTestUtils.assertThrows(log, () -> {
+ final File wal = File.createTempFile("wal", "");
+ wal.deleteOnExit();
+
+ final String[] args = {
+ "walDir=" + wal.getAbsolutePath(),
+ "printStat=not_boolean"
+ };
+
+ IgniteWalConverterArguments.parse(System.out, args);
+ }, IgniteException.class, "Incorrect flag printStat, valid value: true
or false. Error parse: not_boolean");
+ }
+
+ /**
+ * Checking whether field "skipCrc" are incorrect.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIncorrectSkipCrc() throws Exception {
+ GridTestUtils.assertThrows(log, () -> {
+ final File wal = File.createTempFile("wal", "");
+ wal.deleteOnExit();
+
+ final String[] args = {
+ "walDir=" + wal.getAbsolutePath(),
+ "skipCrc=not_boolean"
+ };
+
+ IgniteWalConverterArguments.parse(System.out, args);
+ }, IgniteException.class, "Incorrect flag skipCrc, valid value: true
or false. Error parse: not_boolean");
+ }
+
+ /**
+ * Checking default value.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testDefault() throws IOException {
+ final File wal = File.createTempFile("wal", "");
+ wal.deleteOnExit();
+
+ final String[] args = {
+ "walDir=" + wal.getAbsolutePath()
+ };
+
+ final IgniteWalConverterArguments parseArgs =
IgniteWalConverterArguments.parse(System.out, args);
+
+ Assert.assertEquals(4096, parseArgs.getPageSize());
+ Assert.assertNull(parseArgs.getBinaryMetadataFileStoreDir());
+ Assert.assertNull(parseArgs.getMarshallerMappingFileStoreDir());
+ Assert.assertTrue(parseArgs.isKeepBinary());
+ Assert.assertTrue(parseArgs.getRecordTypes().isEmpty());
+ Assert.assertNull(parseArgs.getFromTime());
+ Assert.assertNull(parseArgs.getToTime());
+ Assert.assertNull(parseArgs.getRecordContainsText());
+ Assert.assertEquals(ProcessSensitiveData.SHOW,
parseArgs.getProcessSensitiveData());
+ Assert.assertFalse(parseArgs.isPrintStat());
+ Assert.assertFalse(parseArgs.isSkipCrc());
+ }
+
+ /**
+ * Checking all value set.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testParse() throws IOException {
+ final File wal = File.createTempFile("wal", "");
+ wal.deleteOnExit();
+
+ final File walArchive = File.createTempFile("wal_archive", "");
+ walArchive.deleteOnExit();
+
+ final File binaryMetadataDir = new
File(System.getProperty("java.io.tmpdir"));
+
+ final File marshallerDir = binaryMetadataDir;
+
+ final String[] args = {
+ "walDir=" + wal.getAbsolutePath(),
+ "walArchiveDir=" + walArchive.getAbsolutePath(),
+ "pageSize=2048",
+ "binaryMetadataFileStoreDir=" +
binaryMetadataDir.getAbsolutePath(),
+ "marshallerMappingFileStoreDir=" + marshallerDir.getAbsolutePath(),
+ "keepBinary=false",
+ "recordTypes=DATA_RECORD,TX_RECORD",
+ "walTimeFromMillis=1575158400000",
+ "walTimeToMillis=1577836740999",
+ "recordContainsText=search string",
+ "processSensitiveData=MD5",
+ "printStat=true",
+ "skipCrc=true"};
+
+ final IgniteWalConverterArguments parseArgs =
IgniteWalConverterArguments.parse(System.out, args);
+ Assert.assertEquals(wal, parseArgs.getWalDir());
+ Assert.assertEquals(walArchive, parseArgs.getWalArchiveDir());
+ Assert.assertEquals(2048, parseArgs.getPageSize());
+ Assert.assertEquals(binaryMetadataDir,
parseArgs.getBinaryMetadataFileStoreDir());
+ Assert.assertEquals(marshallerDir,
parseArgs.getMarshallerMappingFileStoreDir());
+ Assert.assertFalse(parseArgs.isKeepBinary());
+
Assert.assertTrue(parseArgs.getRecordTypes().contains(WALRecord.RecordType.DATA_RECORD));
+
Assert.assertTrue(parseArgs.getRecordTypes().contains(WALRecord.RecordType.TX_RECORD));
+ Assert.assertEquals(1575158400000L, (long)parseArgs.getFromTime());
+ Assert.assertEquals(1577836740999L, (long)parseArgs.getToTime());
+ Assert.assertEquals("search string",
parseArgs.getRecordContainsText());
+ Assert.assertEquals(ProcessSensitiveData.MD5,
parseArgs.getProcessSensitiveData());
+ Assert.assertTrue(parseArgs.isPrintStat());
+ Assert.assertTrue(parseArgs.isSkipCrc());
+ }
+}
diff --git
a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterSensitiveDataTest.java
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterSensitiveDataTest.java
index e2b2dd4..bb4cab9 100644
---
a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterSensitiveDataTest.java
+++
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterSensitiveDataTest.java
@@ -45,21 +45,16 @@ import
org.apache.ignite.internal.processors.cache.GridCacheOperation;
import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.util.typedef.internal.CU;
-import org.apache.ignite.testframework.junits.WithSystemProperty;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.transactions.Transaction;
import org.junit.Test;
import static java.lang.String.valueOf;
import static java.lang.System.setOut;
-import static java.lang.System.setProperty;
import static java.util.Arrays.asList;
import static java.util.Objects.requireNonNull;
import static java.util.function.Function.identity;
-import static
org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_INCLUDE_SENSITIVE;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
-import static
org.apache.ignite.development.utils.IgniteWalConverter.PRINT_RECORDS;
-import static
org.apache.ignite.development.utils.IgniteWalConverter.SENSITIVE_DATA;
import static org.apache.ignite.testframework.GridTestUtils.assertContains;
import static org.apache.ignite.testframework.GridTestUtils.assertNotContains;
import static
org.apache.ignite.testframework.wal.record.RecordUtils.isIncludeIntoLog;
@@ -155,7 +150,7 @@ public class IgniteWalConverterSensitiveDataTest extends
GridCommonAbstractTest
@Override protected void afterTest() throws Exception {
super.afterTest();
- log.info("Test output for " + currentTestMethod());
+ log.info("Test output for " + getName());
log.info("----------------------------------------");
setOut(sysOut);
@@ -189,42 +184,13 @@ public class IgniteWalConverterSensitiveDataTest extends
GridCommonAbstractTest
}
/**
- * Test checks that by default {@link WALRecord} will not be output without
- * system option {@link IgniteWalConverter#PRINT_RECORDS}.
- *
- * @throws Exception If failed.
- */
- @Test
- public void testNotPrintRecordsByDefault() throws Exception {
- exeWithCheck(false, false, identity());
- }
-
- /**
* Test checks that by default sensitive data is displayed.
*
* @throws Exception If failed.
*/
@Test
- @WithSystemProperty(key = PRINT_RECORDS, value = "true")
public void testShowSensitiveDataByDefault() throws Exception {
- exeWithCheck(true, true, identity());
- }
-
- /**
- * Test checks that sensitive data is displayed.
- *
- * @throws Exception If failed.
- */
- @Test
- @WithSystemProperty(key = PRINT_RECORDS, value = "true")
- @WithSystemProperty(key = SENSITIVE_DATA, value = "SHOW")
- public void testShowSensitiveData() throws Exception {
- exeWithCheck(true, true, identity());
-
- setProperty(SENSITIVE_DATA, currentTestMethod().getName());
- resetTestOut();
-
- exeWithCheck(true, true, identity());
+ exeWithCheck(null, true, true, identity());
}
/**
@@ -233,11 +199,8 @@ public class IgniteWalConverterSensitiveDataTest extends
GridCommonAbstractTest
* @throws Exception If failed.
*/
@Test
- @WithSystemProperty(key = PRINT_RECORDS, value = "true")
- @WithSystemProperty(key = SENSITIVE_DATA, value = "HIDE")
- @WithSystemProperty(key = IGNITE_TO_STRING_INCLUDE_SENSITIVE, value =
"true")
public void testHideSensitiveData() throws Exception {
- exeWithCheck(false, false, identity());
+ exeWithCheck(ProcessSensitiveData.HIDE, false, false, identity());
}
/**
@@ -246,10 +209,8 @@ public class IgniteWalConverterSensitiveDataTest extends
GridCommonAbstractTest
* @throws Exception If failed.
*/
@Test
- @WithSystemProperty(key = PRINT_RECORDS, value = "true")
- @WithSystemProperty(key = SENSITIVE_DATA, value = "HASH")
public void testHashSensitiveData() throws Exception {
- exeWithCheck(true, false, s -> valueOf(s.hashCode()));
+ exeWithCheck(ProcessSensitiveData.HASH, true, false, s ->
valueOf(s.hashCode()));
}
/**
@@ -258,21 +219,21 @@ public class IgniteWalConverterSensitiveDataTest extends
GridCommonAbstractTest
* @throws Exception If failed.
*/
@Test
- @WithSystemProperty(key = PRINT_RECORDS, value = "true")
- @WithSystemProperty(key = SENSITIVE_DATA, value = "MD5")
public void testMd5HashSensitiveData() throws Exception {
- exeWithCheck(true, false, ProcessSensitiveDataUtils::md5);
+ exeWithCheck(ProcessSensitiveData.MD5, true, false,
ProcessSensitiveDataUtils::md5);
}
/**
* Executing {@link IgniteWalConverter} with checking the content of its
output.
*
- * @param containsData Contains or not elements {@link #sensitiveValues}
in utility output.
- * @param containsPrefix Contains or not {@link
#SENSITIVE_DATA_VALUE_PREFIX} in utility output.
- * @param converter Converting elements {@link #sensitiveValues} for
checking in utility output.
+ * @param processSensitiveData Strategy for the processing of sensitive
data.
+ * @param containsData Contains or not elements {@link
#sensitiveValues} in utility output.
+ * @param containsPrefix Contains or not {@link
#SENSITIVE_DATA_VALUE_PREFIX} in utility output.
+ * @param converter Converting elements {@link
#sensitiveValues} for checking in utility output.
* @throws Exception If failed.
*/
private void exeWithCheck(
+ ProcessSensitiveData processSensitiveData,
boolean containsData,
boolean containsPrefix,
Function<String, String> converter
@@ -281,7 +242,13 @@ public class IgniteWalConverterSensitiveDataTest extends
GridCommonAbstractTest
injectTestSystemOut();
- IgniteWalConverter.main(new String[] {valueOf(pageSize), walDirPath});
+ List<String> args = new ArrayList<>();
+ args.add("pageSize=" + pageSize);
+ args.add("walDir=" + walDirPath);
+ if (processSensitiveData != null)
+ args.add("processSensitiveData=" + processSensitiveData.name());
+
+ IgniteWalConverter.main(args.toArray(new String[args.size()]));
String testOutStr = testOut.toString();
@@ -373,7 +340,7 @@ public class IgniteWalConverterSensitiveDataTest extends
GridCommonAbstractTest
* Constructor.
*
* @param orgId Organization id.
- * @param name Organization name.
+ * @param name Organization name.
*/
Person(int orgId, String name) {
this.orgId = orgId;
diff --git
a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterTest.java
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterTest.java
new file mode 100644
index 0000000..1e38dbd
--- /dev/null
+++
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterTest.java
@@ -0,0 +1,530 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.development.utils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.PrintStream;
+import java.io.RandomAccessFile;
+import java.util.Base64;
+import java.util.LinkedList;
+import java.util.List;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import
org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+/**
+ * Test for IgniteWalConverter
+ */
+public class IgniteWalConverterTest extends GridCommonAbstractTest {
+ /** */
+ public static final String PERSON_NAME_PREFIX = "Name ";
+
+ /** Flag "skip CRC calculation" in system property save before test and
restore after. */
+ private String beforeIgnitePdsSkipCrc;
+
+ /** Flag "skip CRC calculation" in RecordV1Serializer save before test and
restore after. */
+ private boolean beforeSkipCrc;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ beforeIgnitePdsSkipCrc =
System.getProperty(IgniteSystemProperties.IGNITE_PDS_SKIP_CRC);
+ beforeSkipCrc = RecordV1Serializer.skipCrc;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids(true);
+
+ cleanPersistenceDir();
+
+ super.afterTest();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ if (beforeIgnitePdsSkipCrc != null)
+ System.setProperty(IgniteSystemProperties.IGNITE_PDS_SKIP_CRC,
beforeIgnitePdsSkipCrc);
+ else
+ System.clearProperty(IgniteSystemProperties.IGNITE_PDS_SKIP_CRC);
+
+ RecordV1Serializer.skipCrc = beforeSkipCrc;
+
+ super.afterTestsStopped();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String
igniteInstanceName) throws Exception {
+ final IgniteConfiguration igniteConfiguration =
super.getConfiguration(igniteInstanceName);
+
+
igniteConfiguration.setDataStorageConfiguration(getDataStorageConfiguration());
+
+ final CacheConfiguration cacheConfiguration = new
CacheConfiguration<>()
+ .setName(DEFAULT_CACHE_NAME)
+ .setCacheMode(CacheMode.PARTITIONED)
+ .setBackups(0)
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+ .setIndexedTypes(PersonKey.class, Person.class);
+
+ igniteConfiguration.setCacheConfiguration(cacheConfiguration);
+
+ return igniteConfiguration;
+ }
+
+ /** @return DataStorageConfiguration. */
+ private DataStorageConfiguration getDataStorageConfiguration() {
+ final DataStorageConfiguration dataStorageConfiguration = new
DataStorageConfiguration()
+ .setWalSegmentSize(4 * 1024 * 1024)
+ .setWalMode(WALMode.LOG_ONLY)
+ .setCheckpointFrequency(1000)
+ .setWalCompactionEnabled(true)
+ .setDefaultDataRegionConfiguration(getDataRegionConfiguration());
+
+ return dataStorageConfiguration;
+ }
+
+ /** @return DataRegionConfiguration. */
+ private DataRegionConfiguration getDataRegionConfiguration() {
+ final DataRegionConfiguration dataRegionConfiguration = new
DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setMaxSize(100L * 1024 * 1024);
+
+ return dataRegionConfiguration;
+ }
+
+ /**
+ * Checking utility IgniteWalConverter
+ * <ul>
+ * <li>Start node</li>
+ * <li>Create cache with <a
href="https://apacheignite.readme.io/docs/indexes#section-registering-indexed-types";>Registering
Indexed Types</a></li>
+ * <li>Put several entity</li>
+ * <li>Stop node</li>
+ * <li>Read wal with specifying binaryMetadata</li>
+ * <li>Check that the output contains all DataRecord with previously
added entities</li>
+ * </ul>
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIgniteWalConverter() throws Exception {
+ final List<Person> list = new LinkedList<>();
+
+ final String nodeFolder = createWal(list);
+
+ final ByteArrayOutputStream outByte = new ByteArrayOutputStream();
+
+ final PrintStream out = new PrintStream(outByte);
+
+ final IgniteWalConverterArguments arg = new
IgniteWalConverterArguments(
+ U.resolveWorkDirectory(U.defaultWorkDirectory(),
DataStorageConfiguration.DFLT_WAL_PATH, false),
+ U.resolveWorkDirectory(U.defaultWorkDirectory(),
DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, false),
+ DataStorageConfiguration.DFLT_PAGE_SIZE,
+ new File(U.resolveWorkDirectory(U.defaultWorkDirectory(),
DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, false), nodeFolder),
+ U.resolveWorkDirectory(U.defaultWorkDirectory(),
DataStorageConfiguration.DFLT_MARSHALLER_PATH, false),
+ false,
+ null,
+ null, null, null, null, true,true
+ );
+
+ IgniteWalConverter.convert(out, arg);
+
+ final String result = outByte.toString();
+
+ int index = 0;
+
+ for (Person person : list) {
+ boolean find = false;
+
+ index = result.indexOf("DataRecord", index);
+
+ if (index > 0) {
+ index = result.indexOf("PersonKey", index + 10);
+
+ if (index > 0) {
+ index = result.indexOf("id=" + person.getId(), index + 9);
+
+ if (index > 0) {
+ index = result.indexOf("name=" + person.getName(),
index + 4);
+
+ find = index > 0;
+ }
+ }
+ }
+
+ assertTrue("DataRecord for Person(id=" + person.getId() + ") not
found", find);
+ }
+ }
+
+ /**
+ * Checking utility IgniteWalConverter with out binary_meta
+ * <ul>
+ * <li>Start node</li>
+ * <li>Create cache with <a
href="https://apacheignite.readme.io/docs/indexes#section-registering-indexed-types";>Registering
Indexed Types</a></li>
+ * <li>Put several entity</li>
+ * <li>Stop node</li>
+ * <li>Read wal with <b>out</b> specifying binaryMetadata</li>
+ * <li>Check that the output contains all DataRecord and in DataRecord
not empty key and value</li>
+ * </ul>
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIgniteWalConverterWithOutBinaryMeta() throws Exception {
+ final List<Person> list = new LinkedList<>();
+
+ createWal(list);
+
+ final ByteArrayOutputStream outByte = new ByteArrayOutputStream();
+
+ final PrintStream out = new PrintStream(outByte);
+
+ final IgniteWalConverterArguments arg = new
IgniteWalConverterArguments(
+ U.resolveWorkDirectory(U.defaultWorkDirectory(),
DataStorageConfiguration.DFLT_WAL_PATH, false),
+ U.resolveWorkDirectory(U.defaultWorkDirectory(),
DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, false),
+ DataStorageConfiguration.DFLT_PAGE_SIZE,
+ null,
+ null,
+ false,
+ null,
+ null, null, null, null, true,true
+ );
+
+ IgniteWalConverter.convert(out, arg);
+
+ final String result = outByte.toString();
+
+ int index = 0;
+
+ for (Person person : list) {
+ boolean find = false;
+
+ index = result.indexOf("DataRecord", index);
+
+ if (index > 0) {
+ index = result.indexOf(" v = [", index + 10);
+
+ if (index > 0) {
+ int start = index + 6;
+
+ index = result.indexOf("]", start);
+
+ if (index > 0) {
+ final String value = result.substring(start, index);
+
+ find = new
String(Base64.getDecoder().decode(value)).contains(person.getName());
+ }
+ }
+ }
+
+ assertTrue("DataRecord for Person(id=" + person.getId() + ") not
found", find);
+ }
+ }
+
+ /**
+ * Checking utility IgniteWalConverter on broken WAL
+ * <ul>
+ * <li>Start node</li>
+ * <li>Create cache with <a
href="https://apacheignite.readme.io/docs/indexes#section-registering-indexed-types";>Registering
Indexed Types</a></li>
+ * <li>Put several entity</li>
+ * <li>Stop node</li>
+ * <li>Change byte in DataRecord value</li>
+ * <li>Read wal</li>
+ * <li>Check one error when reading WAL</li>
+ * </ul>
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIgniteWalConverterWithBrokenWal() throws Exception {
+ final List<Person> list = new LinkedList<>();
+
+ final String nodeFolder = createWal(list);
+
+ final File walDir = U.resolveWorkDirectory(U.defaultWorkDirectory(),
DataStorageConfiguration.DFLT_WAL_PATH, false);
+
+ final File wal = new File(walDir, nodeFolder + File.separator +
"0000000000000000.wal");
+
+ try (RandomAccessFile raf = new RandomAccessFile(wal, "rw")) {
+ raf.seek(RecordV1Serializer.HEADER_RECORD_SIZE); // HeaderRecord
+
+ byte findByte[] = (PERSON_NAME_PREFIX + 0).getBytes();
+
+ boolean find = false;
+
+ while (!find) {
+ int recordTypeIndex = raf.read();
+
+ if (recordTypeIndex > 0) {
+ recordTypeIndex--;
+
+ final long idx = raf.readLong();
+
+ final int fileOff = Integer.reverseBytes(raf.readInt());
+
+ final int len = Integer.reverseBytes(raf.readInt());
+
+ if (recordTypeIndex ==
WALRecord.RecordType.DATA_RECORD.index()) {
+ int i = 0;
+
+ int b;
+
+ while (!find && (b = raf.read()) >= 0) {
+ if (findByte[i] == b) {
+ i++;
+
+ if (i == findByte.length)
+ find = true;
+ }
+ else
+ i = 0;
+ }
+ if (find) {
+ raf.seek(raf.getFilePointer() - 1);
+
+ raf.write(' ');
+ }
+ }
+
+ raf.seek(fileOff + len);
+ }
+ }
+ }
+
+ final ByteArrayOutputStream outByte = new ByteArrayOutputStream();
+
+ final PrintStream out = new PrintStream(outByte);
+
+ final IgniteWalConverterArguments arg = new
IgniteWalConverterArguments(
+ walDir,
+ U.resolveWorkDirectory(U.defaultWorkDirectory(),
DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, false),
+ DataStorageConfiguration.DFLT_PAGE_SIZE,
+ new File(U.resolveWorkDirectory(U.defaultWorkDirectory(),
DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, false), nodeFolder),
+ U.resolveWorkDirectory(U.defaultWorkDirectory(),
DataStorageConfiguration.DFLT_MARSHALLER_PATH, false),
+ false,
+ null,
+ null, null, null, null, true,true
+ );
+
+ IgniteWalConverter.convert(out, arg);
+
+ final String result = outByte.toString();
+
+ int index = 0;
+
+ int countErrorRead = 0;
+
+ for (Person person : list) {
+ boolean find = false;
+
+ index = result.indexOf("DataRecord", index);
+
+ if (index > 0) {
+ index = result.indexOf("PersonKey", index + 10);
+
+ if (index > 0) {
+ index = result.indexOf("id=" + person.getId(), index + 9);
+
+ if (index > 0) {
+ index = result.indexOf("name=" + person.getName(),
index + 4);
+
+ find = index > 0;
+ }
+ }
+ }
+
+ if (!find)
+ countErrorRead++;
+ }
+ assertEquals(1, countErrorRead);
+ }
+
+ /**
+ * Checking utility IgniteWalConverter on unreadable WAL
+ * <ul>
+ * <li>Start node</li>
+ * <li>Create cache with <a
href="https://apacheignite.readme.io/docs/indexes#section-registering-indexed-types";>Registering
Indexed Types</a></li>
+ * <li>Put several entity</li>
+ * <li>Stop node</li>
+ * <li>Change byte RecordType in second DataRecord</li>
+ * <li>Read wal</li>
+ * <li>Check contains one DataRecord in output before error when
reading WAL</li>
+ * </ul>
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testIgniteWalConverterWithUnreadableWal() throws Exception {
+ final List<Person> list = new LinkedList<>();
+
+ final String nodeFolder = createWal(list);
+
+ final File walDir = U.resolveWorkDirectory(U.defaultWorkDirectory(),
DataStorageConfiguration.DFLT_WAL_PATH, false);
+
+ final File wal = new File(walDir, nodeFolder + File.separator +
"0000000000000000.wal");
+
+ try (RandomAccessFile raf = new RandomAccessFile(wal, "rw")) {
+ raf.seek(RecordV1Serializer.HEADER_RECORD_SIZE); // HeaderRecord
+
+ int find = 0;
+
+ while (find < 2) {
+ int recordTypeIndex = raf.read();
+
+ if (recordTypeIndex > 0) {
+ recordTypeIndex--;
+
+ if (recordTypeIndex ==
WALRecord.RecordType.DATA_RECORD.index()) {
+ find++;
+
+ if (find == 2) {
+ raf.seek(raf.getFilePointer() - 1);
+
+ raf.write(Byte.MAX_VALUE);
+ }
+ }
+
+ final long idx = raf.readLong();
+
+ final int fileOff = Integer.reverseBytes(raf.readInt());
+
+ final int len = Integer.reverseBytes(raf.readInt());
+
+ raf.seek(fileOff + len);
+ }
+ }
+ }
+
+ final ByteArrayOutputStream outByte = new ByteArrayOutputStream();
+
+ final PrintStream out = new PrintStream(outByte);
+
+ final IgniteWalConverterArguments arg = new
IgniteWalConverterArguments(
+ walDir,
+ U.resolveWorkDirectory(U.defaultWorkDirectory(),
DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, false),
+ DataStorageConfiguration.DFLT_PAGE_SIZE,
+ new File(U.resolveWorkDirectory(U.defaultWorkDirectory(),
DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, false), nodeFolder),
+ U.resolveWorkDirectory(U.defaultWorkDirectory(),
DataStorageConfiguration.DFLT_MARSHALLER_PATH, false),
+ false,
+ null,
+ null, null, null, null, true,true
+ );
+
+ IgniteWalConverter.convert(out, arg);
+
+ final String result = outByte.toString();
+
+ int index = 0;
+
+ int countErrorRead = 0;
+
+ for (Person person : list) {
+ boolean find = false;
+
+ index = result.indexOf("DataRecord", index);
+
+ if (index > 0) {
+ index = result.indexOf("PersonKey", index + 10);
+
+ if (index > 0) {
+ index = result.indexOf("id=" + person.getId(), index + 9);
+
+ if (index > 0) {
+ index =
result.indexOf(person.getClass().getSimpleName(), index + 4);
+
+ if (index > 0) {
+ index = result.indexOf("id=" + person.getId(),
index + person.getClass().getSimpleName().length());
+
+ if (index > 0) {
+ index = result.indexOf("name=" +
person.getName(), index + 4);
+
+ find = index > 0;
+ }
+ }
+ }
+ }
+ }
+
+ if (!find)
+ countErrorRead++;
+ }
+
+ assertEquals(9, countErrorRead);
+ }
+
+ /**
+ * Common part
+ * <ul>
+ * <li>Start node</li>
+ * <li>Create cache with <a
href="https://apacheignite.readme.io/docs/indexes#section-registering-indexed-types";>Registering
Indexed Types</a></li>
+ * <li>Put several entity</li>
+ * </ul>
+ *
+ * @param list Returns entities that have been added.
+ * @return Node folder name.
+ * @throws Exception
+ */
+ private String createWal(List<Person> list) throws Exception {
+ String nodeFolder;
+
+ try (final IgniteEx node = startGrid(0)) {
+ node.cluster().active(true);
+
+ nodeFolder =
node.context().pdsFolderResolver().resolveFolders().folderName();
+
+ final IgniteCache<PersonKey, Person> cache =
node.cache(DEFAULT_CACHE_NAME);
+
+ for (int i = 0; i < 10; i++) {
+ final PersonKey key = new PersonKey(i);
+
+ final Person value;
+
+ if (i % 2 == 0)
+ value = new Person(i, PERSON_NAME_PREFIX + i);
+ else
+ value = new PersonEx(i, PERSON_NAME_PREFIX + i,
"Additional information " + i, "Description " + i);
+
+ cache.put(key, value);
+
+ list.add(value);
+ }
+ }
+
+ return nodeFolder;
+ }
+}
diff --git
a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/Person.java
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/Person.java
new file mode 100644
index 0000000..67c104b
--- /dev/null
+++
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/Person.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.development.utils;
+
+import java.util.Objects;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+
+/**
+ * A person entity used for the tests.
+ */
+public class Person {
+ /** Id. */
+ private final Integer id;
+
+ /** Name. */
+ @QuerySqlField
+ private final String name;
+
+ /** Constructor. */
+ public Person(Integer id, String name) {
+ this.id = id;
+ this.name = name;
+ }
+
+ /** @return id. */
+ public Integer getId() {
+ return id;
+ }
+
+ /** @return name. */
+ public String getName() {
+ return name;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Objects.hash(id, name);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ if (!(obj instanceof Person))
+ return false;
+
+ Person other = (Person)obj;
+
+ return Objects.equals(id, other.id) &&
+ Objects.equals(name, other.name);
+ }
+}
diff --git
a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/PersonEx.java
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/PersonEx.java
new file mode 100644
index 0000000..f03dc30
--- /dev/null
+++
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/PersonEx.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.development.utils;
+
+import java.util.Objects;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+
+/**
+ * A person entity used for the tests.
+ */
+public class PersonEx extends Person {
+ /**
+ * Additional information.
+ */
+ @QuerySqlField
+ private final String info;
+
+ /**
+ * Description - not declared as SQL field.
+ */
+ private final String description;
+
+ /**
+ * Constructor.
+ */
+ public PersonEx(Integer id, String name, String info, String description) {
+ super(id, name);
+ this.info = info;
+ this.description = description;
+ }
+
+ /**
+ * @return Additional information.
+ */
+ public String getInfo() {
+ return info;
+ }
+
+ /**
+ * @return Description.
+ */
+ public String getDescription() {
+ return description;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public int hashCode() {
+ return Objects.hash(super.hashCode(), info, description);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public boolean equals(Object obj) {
+ if (!(obj instanceof PersonEx))
+ return false;
+
+ PersonEx other = (PersonEx)obj;
+
+ return super.equals(other) && Objects.equals(info, other.info) &&
Objects.equals(description, other.description);
+ }
+}
diff --git
a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/DevUtilsTestSuite.java
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/PersonKey.java
similarity index 52%
copy from
modules/dev-utils/src/test/java/org/apache/ignite/development/utils/DevUtilsTestSuite.java
copy to
modules/dev-utils/src/test/java/org/apache/ignite/development/utils/PersonKey.java
index 5f424d9..a5f82b7 100644
---
a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/DevUtilsTestSuite.java
+++
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/PersonKey.java
@@ -17,15 +17,45 @@
package org.apache.ignite.development.utils;
-import org.junit.runner.RunWith;
-import org.junit.runners.Suite;
+import java.util.Objects;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
/**
- * Test suite for dev utils.
+ * A person entity used for the tests.
*/
-@RunWith(Suite.class)
-@Suite.SuiteClasses({
- IgniteWalConverterSensitiveDataTest.class
-})
-public class DevUtilsTestSuite {
+public class PersonKey {
+ /**
+ * Id.
+ */
+ @QuerySqlField(index = true)
+ private final Integer id;
+
+ /**
+ * Constructor.
+ */
+ public PersonKey(Integer id) {
+ this.id = id;
+ }
+
+ /**
+ * @return id.
+ */
+ public Integer getId() {
+ return id;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object obj) {
+ if (!(obj instanceof PersonKey))
+ return false;
+
+ PersonKey other = (PersonKey)obj;
+
+ return Objects.equals(other.id, id);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Objects.hash(id);
+ }
}
