This is an automated email from the ASF dual-hosted git repository.
irakov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new e7e34f0 IGNITE-12906 Add to IgniteWalConverter possibility output
only hashes instead real data - Fixes #7678.
e7e34f0 is described below
commit e7e34f0c3bcf080bc3d0da9b3e72ee72af9d0a32
Author: Kirill Tkalenko <tkalkir...@yandex.ru>
AuthorDate: Wed Apr 22 18:28:14 2020 +0300
IGNITE-12906 Add to IgniteWalConverter possibility output only hashes
instead real data - Fixes #7678.
Signed-off-by: Ivan Rakov <ivan.glu...@gmail.com>
---
.../org/apache/ignite/IgniteSystemProperties.java | 7 +-
.../pagemem/wal/record/MetastoreDataRecord.java | 2 +
.../pagemem/wal/record/UnwrapDataEntry.java | 14 +-
.../pagemem/wal/record/UnwrapMvccDataEntry.java | 14 +-
.../wal/record/WALRecordSerializationTest.java | 8 +-
.../testframework/wal/record/RecordUtils.java | 10 +
modules/dev-utils/pom.xml | 28 ++
.../ignite/development/utils/DataEntryWrapper.java | 89 +++++
.../development/utils/IgniteWalConverter.java | 65 +++-
.../utils/MetastoreDataRecordWrapper.java | 53 +++
.../development/utils/ProcessSensitiveData.java | 32 ++
.../utils/ProcessSensitiveDataUtils.java | 56 +++
.../development/utils/DevUtilsTestSuite.java | 31 ++
.../utils/IgniteWalConverterSensitiveDataTest.java | 383 +++++++++++++++++++++
14 files changed, 772 insertions(+), 20 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index bcec9ed..66c3811 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -1347,7 +1347,12 @@ public final class IgniteSystemProperties {
if (val == null)
return dflt;
- return Enum.valueOf(enumCls, val);
+ try {
+ return Enum.valueOf(enumCls, val);
+ }
+ catch (IllegalArgumentException ignore) {
+ return dflt;
+ }
}
/**
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java
index 9e73424..5cff9ba 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java
@@ -19,6 +19,7 @@
package org.apache.ignite.internal.pagemem.wal.record;
import
org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.jetbrains.annotations.Nullable;
@@ -27,6 +28,7 @@ import org.jetbrains.annotations.Nullable;
*/
public class MetastoreDataRecord extends WALRecord implements
WalRecordCacheGroupAware {
/** */
+ @GridToStringInclude(sensitive = true)
private final String key;
/** */
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapDataEntry.java
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapDataEntry.java
index 5dd268b..ad5e1d0 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapDataEntry.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapDataEntry.java
@@ -23,6 +23,8 @@ import
org.apache.ignite.internal.processors.cache.CacheObjectValueContext;
import org.apache.ignite.internal.processors.cache.GridCacheOperation;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.SB;
/**
* Data Entry for automatic unwrapping key and value from Data Entry
@@ -108,9 +110,13 @@ public class UnwrapDataEntry extends DataEntry implements
UnwrappedDataEntry {
/** {@inheritDoc} */
@Override public String toString() {
- return getClass().getSimpleName() + "[k = " + unwrappedKey() + ", v =
[ "
- + unwrappedValue()
- + "], super = ["
- + super.toString() + "]]";
+ SB sb = new SB();
+
+ sb.a(getClass().getSimpleName()).a('[');
+
+ if (S.includeSensitive())
+ sb.a("k = ").a(unwrappedKey()).a(", v = [
").a(unwrappedValue()).a("], ");
+
+ return sb.a("super = [").a(super.toString()).a("]]").toString();
}
}
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapMvccDataEntry.java
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapMvccDataEntry.java
index c3c12a3..47a9d7c 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapMvccDataEntry.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapMvccDataEntry.java
@@ -24,6 +24,8 @@ import
org.apache.ignite.internal.processors.cache.GridCacheOperation;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.mvcc.MvccVersion;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.SB;
/**
* Data Entry for automatic unwrapping key and value from Mvcc Data Entry
@@ -112,9 +114,13 @@ public class UnwrapMvccDataEntry extends MvccDataEntry
implements UnwrappedDataE
/** {@inheritDoc} */
@Override public String toString() {
- return getClass().getSimpleName() + "[k = " + unwrappedKey() + ", v =
[ "
- + unwrappedValue()
- + "], super = ["
- + super.toString() + "]]";
+ SB sb = new SB();
+
+ sb.a(getClass().getSimpleName()).a('[');
+
+ if (S.includeSensitive())
+ sb.a("k = ").a(unwrappedKey()).a(", v = [
").a(unwrappedValue()).a("], ");
+
+ return sb.a("super = [").a(super.toString()).a("]]").toString();
}
}
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/wal/record/WALRecordSerializationTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/wal/record/WALRecordSerializationTest.java
index edd348d..bc3470f 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/wal/record/WALRecordSerializationTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/wal/record/WALRecordSerializationTest.java
@@ -36,7 +36,6 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.testframework.wal.record.RecordUtils;
-import org.apache.ignite.testframework.wal.record.UnsupportedWalRecord;
import org.junit.Test;
import org.mockito.internal.matchers.apachecommons.ReflectionEquals;
@@ -108,7 +107,7 @@ public class WALRecordSerializationTest extends
GridCommonAbstractTest {
for (WALRecord.RecordType recordType : recordTypes) {
WALRecord record = RecordUtils.buildWalRecord(recordType);
- if (!(record instanceof UnsupportedWalRecord) && !(record
instanceof SwitchSegmentRecord)) {
+ if (RecordUtils.isIncludeIntoLog(record)) {
serializedRecords.add(new ReflectionEquals(record, "prev",
"pos",
"updateCounter" //updateCounter for
PartitionMetaStateRecord isn't serialized.
));
@@ -165,11 +164,8 @@ public class WALRecordSerializationTest extends
GridCommonAbstractTest {
for (WALRecord.RecordType recordType : recordTypes) {
WALRecord record = RecordUtils.buildWalRecord(recordType);
- if (!(record instanceof UnsupportedWalRecord)
- && !(record instanceof SwitchSegmentRecord)
- && (recordType.purpose() ==
WALRecord.RecordPurpose.LOGICAL ||
+ if (RecordUtils.isIncludeIntoLog(record) &&
(recordType.purpose() == WALRecord.RecordPurpose.LOGICAL ||
recordType == WALRecord.RecordType.CHECKPOINT_RECORD)) {
-
serializedRecords.add(new ReflectionEquals(record, "prev",
"pos",
"updateCounter" //updateCounter for
PartitionMetaStateRecord isn't serialized.
));
diff --git
a/modules/core/src/test/java/org/apache/ignite/testframework/wal/record/RecordUtils.java
b/modules/core/src/test/java/org/apache/ignite/testframework/wal/record/RecordUtils.java
index 37dbeb8..793825e 100644
---
a/modules/core/src/test/java/org/apache/ignite/testframework/wal/record/RecordUtils.java
+++
b/modules/core/src/test/java/org/apache/ignite/testframework/wal/record/RecordUtils.java
@@ -571,4 +571,14 @@ public class RecordUtils {
public static UnsupportedWalRecord buildBtreeMetaPageInitRootV3() {
return new UnsupportedWalRecord(BTREE_META_PAGE_INIT_ROOT_V3);
}
+
+ /**
+ * Return {@code true} if include to write-ahead log.
+ *
+ * @param walRecord Instance of {@link WALRecord}.
+ * @return {@code True} if include to write-ahead log.
+ */
+ public static boolean isIncludeIntoLog(WALRecord walRecord) {
+ return !UnsupportedWalRecord.class.isInstance(walRecord) &&
!SwitchSegmentRecord.class.isInstance(walRecord);
+ }
}
diff --git a/modules/dev-utils/pom.xml b/modules/dev-utils/pom.xml
index e5f830f..d6cf4e0 100644
--- a/modules/dev-utils/pom.xml
+++ b/modules/dev-utils/pom.xml
@@ -46,6 +46,34 @@
<artifactId>ignite-indexing</artifactId>
<version>${project.version}</version>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-context</artifactId>
+ <version>${spring.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.springframework</groupId>
+ <artifactId>spring-beans</artifactId>
+ <version>${spring.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
diff --git
a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/DataEntryWrapper.java
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/DataEntryWrapper.java
new file mode 100644
index 0000000..f96620a
--- /dev/null
+++
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/DataEntryWrapper.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.development.utils;
+
+import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
+import org.apache.ignite.internal.pagemem.wal.record.UnwrappedDataEntry;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.jetbrains.annotations.Nullable;
+
+import static java.lang.String.valueOf;
+import static java.util.Objects.isNull;
+import static org.apache.ignite.development.utils.ProcessSensitiveData.HASH;
+import static org.apache.ignite.development.utils.ProcessSensitiveData.MD5;
+import static
org.apache.ignite.development.utils.ProcessSensitiveDataUtils.md5;
+
+/**
+ * Wrapper {@link DataEntry} for sensitive data output.
+ */
+class DataEntryWrapper extends DataEntry {
+ /** Unwrapped DataEntry. */
+ @Nullable private final UnwrappedDataEntry unwrappedDataEntry;
+
+ /** Strategy for the processing of sensitive data. */
+ private final ProcessSensitiveData sensitiveData;
+
+ /**
+ * Constructor.
+ *
+ * @param dataEntry Instance of {@link DataEntry}.
+ * @param sensitiveData Strategy for the processing of sensitive data.
+ */
+ public DataEntryWrapper(
+ DataEntry dataEntry,
+ ProcessSensitiveData sensitiveData
+ ) {
+ super(
+ dataEntry.cacheId(),
+ dataEntry.key(),
+ dataEntry.value(),
+ dataEntry.op(),
+ dataEntry.nearXidVersion(),
+ dataEntry.writeVersion(),
+ dataEntry.expireTime(),
+ dataEntry.partitionId(),
+ dataEntry.partitionCounter()
+ );
+
+ this.sensitiveData = sensitiveData;
+
+ this.unwrappedDataEntry =
UnwrappedDataEntry.class.isInstance(dataEntry) ?
+ (UnwrappedDataEntry) dataEntry : null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ if (isNull(unwrappedDataEntry))
+ return super.toString();
+
+ Object key = unwrappedDataEntry.unwrappedKey();
+ Object value = unwrappedDataEntry.unwrappedValue();
+
+ if (HASH == sensitiveData) {
+ key = valueOf(key).hashCode();
+ value = valueOf(value).hashCode();
+ }
+ else if (MD5 == sensitiveData) {
+ key = md5(valueOf(key));
+ value = md5(valueOf(value));
+ }
+
+ return new SB().a(unwrappedDataEntry.getClass().getSimpleName())
+ .a("[k = ").a(key).a(", v = [ ").a(value).a("], super =
[").a(super.toString()).a("]]").toString();
+ }
+}
diff --git
a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
index ca144ab..d9e9423 100644
---
a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
+++
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
@@ -18,9 +18,12 @@
package org.apache.ignite.development.utils;
import java.io.File;
-import org.apache.ignite.IgniteSystemProperties;
+import java.util.List;
import org.apache.ignite.internal.pagemem.wal.WALIterator;
import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
+import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import
org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
@@ -35,11 +38,34 @@ import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.logger.NullLogger;
import org.jetbrains.annotations.Nullable;
+import static java.util.stream.Collectors.toList;
+import static
org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_INCLUDE_SENSITIVE;
+import static org.apache.ignite.IgniteSystemProperties.getBoolean;
+import static org.apache.ignite.IgniteSystemProperties.getEnum;
+import static org.apache.ignite.development.utils.ProcessSensitiveData.HIDE;
+import static org.apache.ignite.development.utils.ProcessSensitiveData.SHOW;
+
/**
* Print WAL log data in human-readable form.
*/
public class IgniteWalConverter {
/**
+ * System property for printing {@link WALRecord}. By default, {@code
false}.
+ */
+ static final String PRINT_RECORDS = "PRINT_RECORDS";
+
+ /**
+ * System property for printing {@link WalStat}. By default, {@code true}.
+ */
+ static final String PRINT_STAT = "PRINT_STAT";
+
+ /**
+ * System property for setting {@link ProcessSensitiveData strategy} of
output sensitive data.
+ * By default, {@link ProcessSensitiveData#SHOW}.
+ */
+ static final String SENSITIVE_DATA = "SENSITIVE_DATA";
+
+ /**
* @param args Args.
* @throws Exception If failed.
*/
@@ -54,8 +80,12 @@ public class IgniteWalConverter {
H2ExtrasInnerIO.register();
H2ExtrasLeafIO.register();
- boolean printRecords =
IgniteSystemProperties.getBoolean("PRINT_RECORDS", false); //TODO read them
from argumetns
- boolean printStat = IgniteSystemProperties.getBoolean("PRINT_STAT",
true); //TODO read them from argumetns
+ boolean printRecords = getBoolean(PRINT_RECORDS, false); //TODO read
them from argumetns
+ boolean printStat = getBoolean(PRINT_STAT, true); //TODO read them
from argumetns
+ ProcessSensitiveData sensitiveData = getEnum(SENSITIVE_DATA, SHOW);
//TODO read them from argumetns
+
+ if (printRecords && HIDE == sensitiveData)
+ System.setProperty(IGNITE_TO_STRING_INCLUDE_SENSITIVE,
Boolean.FALSE.toString());
final IgniteWalIteratorFactory factory = new
IgniteWalIteratorFactory(new NullLogger());
@@ -83,7 +113,7 @@ public class IgniteWalConverter {
stat.registerRecord(record, pointer, true);
if (printRecords)
- System.out.println("[W] " + record);
+ System.out.println("[W] " + toString(record,
sensitiveData));
}
}
@@ -101,7 +131,7 @@ public class IgniteWalConverter {
stat.registerRecord(record, pointer, false);
if (printRecords)
- System.out.println("[A] " + record);
+ System.out.println("[A] " + toString(record,
sensitiveData));
}
}
}
@@ -111,4 +141,29 @@ public class IgniteWalConverter {
if (stat != null)
System.out.println("Statistic collected:\n" + stat.toString());
}
+
+ /**
+ * Converting {@link WALRecord} to a string with sensitive data.
+ *
+ * @param walRecord Instance of {@link WALRecord}.
+ * @param sensitiveData Strategy for processing of sensitive data.
+ * @return String representation of {@link WALRecord}.
+ */
+ private static String toString(WALRecord walRecord, ProcessSensitiveData
sensitiveData) {
+ if (SHOW == sensitiveData || HIDE == sensitiveData)
+ return walRecord.toString();
+
+ if (MetastoreDataRecord.class.isInstance(walRecord))
+ walRecord = new
MetastoreDataRecordWrapper((MetastoreDataRecord)walRecord, sensitiveData);
+ else if (DataRecord.class.isInstance(walRecord)) {
+ DataRecord dataRecord = (DataRecord)walRecord;
+
+ List<DataEntry> entryWrappers = dataRecord.writeEntries().stream()
+ .map(dataEntry -> new DataEntryWrapper(dataEntry,
sensitiveData)).collect(toList());
+
+ dataRecord.setWriteEntries(entryWrappers);
+ }
+
+ return walRecord.toString();
+ }
}
diff --git
a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/MetastoreDataRecordWrapper.java
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/MetastoreDataRecordWrapper.java
new file mode 100644
index 0000000..f32c8f4
--- /dev/null
+++
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/MetastoreDataRecordWrapper.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.development.utils;
+
+import java.util.Arrays;
+import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
+
+import static java.lang.String.valueOf;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.ignite.development.utils.ProcessSensitiveData.HASH;
+import static org.apache.ignite.development.utils.ProcessSensitiveData.MD5;
+import static
org.apache.ignite.development.utils.ProcessSensitiveDataUtils.md5;
+
+/**
+ * Wrapper {@link MetastoreDataRecord} for sensitive data output.
+ */
+class MetastoreDataRecordWrapper extends MetastoreDataRecord {
+ /**
+ * Constructor.
+ *
+ * @param metastoreRecord Instance of {@link MetastoreDataRecord}.
+ * @param sensitiveData Strategy for the processing of sensitive data.
+ */
+ public MetastoreDataRecordWrapper(MetastoreDataRecord metastoreRecord,
ProcessSensitiveData sensitiveData) {
+ super(
+ HASH == sensitiveData ? valueOf(metastoreRecord.key().hashCode()) :
+ MD5 == sensitiveData ? md5(metastoreRecord.key()) :
metastoreRecord.key(),
+ HASH == sensitiveData ?
valueOf(Arrays.hashCode(metastoreRecord.value())).getBytes(UTF_8) :
+ MD5 == sensitiveData ?
md5(Arrays.toString(metastoreRecord.value())).getBytes(UTF_8) :
+ metastoreRecord.value()
+ );
+
+ size(metastoreRecord.size());
+ chainSize(metastoreRecord.chainSize());
+ previous(metastoreRecord.previous());
+ position(metastoreRecord.position());
+ }
+}
diff --git
a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/ProcessSensitiveData.java
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/ProcessSensitiveData.java
new file mode 100644
index 0000000..8b30055
--- /dev/null
+++
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/ProcessSensitiveData.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.development.utils;
+
+/**
+ * Strategy for the processing of sensitive data.
+ */
+enum ProcessSensitiveData {
+ /** Show sensitive data. */
+ SHOW,
+ /** Hide sensitive data. */
+ HIDE,
+ /** Replace sensitive data with {@link Object#hashCode}. */
+ HASH,
+ /** Replace sensitive data with MD5 hash. */
+ MD5;
+}
diff --git
a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/ProcessSensitiveDataUtils.java
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/ProcessSensitiveDataUtils.java
new file mode 100644
index 0000000..4355568
--- /dev/null
+++
b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/ProcessSensitiveDataUtils.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.development.utils;
+
+import java.security.MessageDigest;
+import java.util.Base64;
+import org.apache.ignite.IgniteException;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * Utility class for processing sensitive data.
+ */
+class ProcessSensitiveDataUtils {
+ /**
+ * Conversion to md5 hash string.
+ *
+ * @param val String value.
+ * @return MD5 hash string.
+ * */
+ public static String md5(String val) {
+ try {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ md.update(val.getBytes(UTF_8));
+
+ byte[] digest = md.digest();
+
+ return Base64.getEncoder().encodeToString(digest);
+ }
+ catch (Exception e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /**
+ * Private constructor.
+ */
+ private ProcessSensitiveDataUtils(){
+ throw new RuntimeException("Don't create.");
+ }
+}
diff --git
a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/DevUtilsTestSuite.java
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/DevUtilsTestSuite.java
new file mode 100644
index 0000000..5f424d9
--- /dev/null
+++
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/DevUtilsTestSuite.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.development.utils;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+
+/**
+ * Test suite for dev utils.
+ */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ IgniteWalConverterSensitiveDataTest.class
+})
+public class DevUtilsTestSuite {
+}
diff --git
a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterSensitiveDataTest.java
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterSensitiveDataTest.java
new file mode 100644
index 0000000..e2b2dd4
--- /dev/null
+++
b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterSensitiveDataTest.java
@@ -0,0 +1,383 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.development.utils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.PrintStream;
+import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.function.Function;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryIndex;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
+import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.CacheObjectImpl;
+import org.apache.ignite.internal.processors.cache.GridCacheOperation;
+import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.testframework.junits.WithSystemProperty;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.junit.Test;
+
+import static java.lang.String.valueOf;
+import static java.lang.System.setOut;
+import static java.lang.System.setProperty;
+import static java.util.Arrays.asList;
+import static java.util.Objects.requireNonNull;
+import static java.util.function.Function.identity;
+import static
org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_INCLUDE_SENSITIVE;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static
org.apache.ignite.development.utils.IgniteWalConverter.PRINT_RECORDS;
+import static
org.apache.ignite.development.utils.IgniteWalConverter.SENSITIVE_DATA;
+import static org.apache.ignite.testframework.GridTestUtils.assertContains;
+import static org.apache.ignite.testframework.GridTestUtils.assertNotContains;
+import static
org.apache.ignite.testframework.wal.record.RecordUtils.isIncludeIntoLog;
+
+/**
+ * Class for testing sensitive data when reading {@link WALRecord} using
+ * {@link IgniteWalConverter}.
+ */
+public class IgniteWalConverterSensitiveDataTest extends
GridCommonAbstractTest {
+ /** Sensitive data prefix. */
+ private static final String SENSITIVE_DATA_VALUE_PREFIX = "must_hide_it_";
+
+ /** Path to directory where WAL is stored. */
+ private static String walDirPath;
+
+ /** Page size. */
+ private static int pageSize;
+
+ /** System out. */
+ private static PrintStream sysOut;
+
+ /** Sensitive data values. */
+ private static List<String> sensitiveValues = new ArrayList<>();
+
+ /**
+ * Test out - can be injected via {@link #injectTestSystemOut()} instead
+ * of System.out and analyzed in test.
+ */
+ private static ByteArrayOutputStream testOut;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ sysOut = System.out;
+ testOut = new ByteArrayOutputStream(16 * 1024);
+
+ int nodeId = 0;
+
+ IgniteEx crd = startGrid(nodeId);
+ crd.cluster().active(true);
+
+ try (Transaction tx = crd.transactions().txStart()) {
+ IgniteCache<Object, Object> cache = crd.cache(DEFAULT_CACHE_NAME);
+
+ sensitiveValues.add(SENSITIVE_DATA_VALUE_PREFIX + 0);
+ sensitiveValues.add(SENSITIVE_DATA_VALUE_PREFIX + 1);
+ sensitiveValues.add(SENSITIVE_DATA_VALUE_PREFIX + 2);
+
+ String val0 = sensitiveValues.get(0);
+ String val1 = sensitiveValues.get(1);
+ String val2 = sensitiveValues.get(2);
+
+ cache.put(val0, val0);
+ cache.withKeepBinary().put(val1, val1);
+ cache.put(val2, new Person(1, val2));
+
+ tx.commit();
+ }
+
+ GridKernalContext kernalCtx = crd.context();
+ IgniteWriteAheadLogManager wal = kernalCtx.cache().context().wal();
+
+ for (WALRecord walRecord : withSensitiveData()) {
+ if (isIncludeIntoLog(walRecord))
+ wal.log(walRecord);
+ }
+
+ sensitiveValues.add(SENSITIVE_DATA_VALUE_PREFIX);
+
+ wal.flush(null, true);
+
+ IgniteConfiguration cfg = crd.configuration();
+
+ String wd = cfg.getWorkDirectory();
+ String wp = cfg.getDataStorageConfiguration().getWalPath();
+ String fn =
kernalCtx.pdsFolderResolver().resolveFolders().folderName();
+
+ walDirPath = wd + File.separator + wp + File.separator + fn;
+ pageSize = cfg.getDataStorageConfiguration().getPageSize();
+
+ stopGrid(nodeId);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ clearGridToStringClassCache();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ log.info("Test output for " + currentTestMethod());
+ log.info("----------------------------------------");
+
+ setOut(sysOut);
+
+ log.info(testOut.toString());
+ resetTestOut();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String
igniteInstanceName) throws Exception {
+ return super.getConfiguration(igniteInstanceName)
+ .setCacheConfiguration(
+ new CacheConfiguration<>(DEFAULT_CACHE_NAME)
+ .setAtomicityMode(TRANSACTIONAL)
+ .setQueryEntities(asList(personQueryEntity()))
+ )
+ .setDataStorageConfiguration(
+ new
DataStorageConfiguration().setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration().setPersistenceEnabled(true)
+ )
+ );
+ }
+
+ /**
+ * Test checks that by default {@link WALRecord} will not be output without
+ * system option {@link IgniteWalConverter#PRINT_RECORDS}.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testNotPrintRecordsByDefault() throws Exception {
+ exeWithCheck(false, false, identity());
+ }
+
+ /**
+ * Test checks that by default sensitive data is displayed.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ @WithSystemProperty(key = PRINT_RECORDS, value = "true")
+ public void testShowSensitiveDataByDefault() throws Exception {
+ exeWithCheck(true, true, identity());
+ }
+
+ /**
+ * Test checks that sensitive data is displayed.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ @WithSystemProperty(key = PRINT_RECORDS, value = "true")
+ @WithSystemProperty(key = SENSITIVE_DATA, value = "SHOW")
+ public void testShowSensitiveData() throws Exception {
+ exeWithCheck(true, true, identity());
+
+ setProperty(SENSITIVE_DATA, currentTestMethod().getName());
+ resetTestOut();
+
+ exeWithCheck(true, true, identity());
+ }
+
+ /**
+ * Test verifies that sensitive data will be hidden.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ @WithSystemProperty(key = PRINT_RECORDS, value = "true")
+ @WithSystemProperty(key = SENSITIVE_DATA, value = "HIDE")
+ @WithSystemProperty(key = IGNITE_TO_STRING_INCLUDE_SENSITIVE, value =
"true")
+ public void testHideSensitiveData() throws Exception {
+ exeWithCheck(false, false, identity());
+ }
+
+ /**
+ * Test verifies that sensitive data should be replaced with hash.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ @WithSystemProperty(key = PRINT_RECORDS, value = "true")
+ @WithSystemProperty(key = SENSITIVE_DATA, value = "HASH")
+ public void testHashSensitiveData() throws Exception {
+ exeWithCheck(true, false, s -> valueOf(s.hashCode()));
+ }
+
+ /**
+ * Test verifies that sensitive data should be replaced with MD5 hash.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ @WithSystemProperty(key = PRINT_RECORDS, value = "true")
+ @WithSystemProperty(key = SENSITIVE_DATA, value = "MD5")
+ public void testMd5HashSensitiveData() throws Exception {
+ exeWithCheck(true, false, ProcessSensitiveDataUtils::md5);
+ }
+
+ /**
+ * Executing {@link IgniteWalConverter} with checking the content of its
output.
+ *
+ * @param containsData Contains or not elements {@link #sensitiveValues}
in utility output.
+ * @param containsPrefix Contains or not {@link
#SENSITIVE_DATA_VALUE_PREFIX} in utility output.
+ * @param converter Converting elements {@link #sensitiveValues} for
checking in utility output.
+ * @throws Exception If failed.
+ */
+ private void exeWithCheck(
+ boolean containsData,
+ boolean containsPrefix,
+ Function<String, String> converter
+ ) throws Exception {
+ requireNonNull(converter);
+
+ injectTestSystemOut();
+
+ IgniteWalConverter.main(new String[] {valueOf(pageSize), walDirPath});
+
+ String testOutStr = testOut.toString();
+
+ if (containsPrefix)
+ assertContains(log, testOutStr, SENSITIVE_DATA_VALUE_PREFIX);
+ else
+ assertNotContains(log, testOutStr, SENSITIVE_DATA_VALUE_PREFIX);
+
+ for (String sensitiveDataValue : sensitiveValues) {
+ if (containsData)
+ assertContains(log, testOutStr,
converter.apply(sensitiveDataValue));
+ else
+ assertNotContains(log, testOutStr,
converter.apply(sensitiveDataValue));
+ }
+ }
+
+ /**
+ * Inject {@link #testOut} to System.out for analyze in test.
+ */
+ private void injectTestSystemOut() {
+ setOut(new PrintStream(testOut));
+ }
+
+ /**
+ * Reset {@link #testOut}.
+ */
+ private void resetTestOut() {
+ testOut.reset();
+ }
+
+ /**
+ * Creating {@link WALRecord} instances with sensitive data.
+ *
+ * @return {@link WALRecord} instances with sensitive data.
+ */
+ private Collection<WALRecord> withSensitiveData() {
+ List<WALRecord> walRecords = new ArrayList<>();
+
+ int cacheId = CU.cacheId(DEFAULT_CACHE_NAME);
+
+ DataEntry dataEntry = new DataEntry(
+ cacheId,
+ new KeyCacheObjectImpl(SENSITIVE_DATA_VALUE_PREFIX, null, 0),
+ new CacheObjectImpl(SENSITIVE_DATA_VALUE_PREFIX, null),
+ GridCacheOperation.CREATE,
+ new GridCacheVersion(),
+ new GridCacheVersion(),
+ 0,
+ 0,
+ 0
+ );
+
+ byte[] sensitiveDataBytes =
SENSITIVE_DATA_VALUE_PREFIX.getBytes(StandardCharsets.UTF_8);
+
+ walRecords.add(new DataRecord(dataEntry));
+ walRecords.add(new MetastoreDataRecord(SENSITIVE_DATA_VALUE_PREFIX,
sensitiveDataBytes));
+
+ return walRecords;
+ }
+
+ /**
+ * Create {@link QueryEntity} for {@link Person}.
+ *
+ * @return QueryEntity for {@link Person}.
+ */
+ private QueryEntity personQueryEntity() {
+ String orgIdField = "orgId";
+ String nameField = "name";
+
+ return new QueryEntity()
+ .setKeyType(String.class.getName())
+ .setValueType(Person.class.getName())
+ .addQueryField(orgIdField, Integer.class.getName(), null)
+ .addQueryField(nameField, String.class.getName(), null)
+ .setIndexes(asList(new QueryIndex(nameField), new
QueryIndex(orgIdField)));
+ }
+
+ /**
+ * Simple class Person for tests.
+ */
+ private static class Person implements Serializable {
+ /** Id organization. */
+ int orgId;
+
+ /** Name organization. */
+ String name;
+
+ /**
+ * Constructor.
+ *
+ * @param orgId Organization id.
+ * @param name Organization name.
+ */
+ Person(int orgId, String name) {
+ this.orgId = orgId;
+ this.name = name;
+ }
+ }
+}
