This is an automated email from the ASF dual-hosted git repository. irakov pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push: new e7e34f0 IGNITE-12906 Add to IgniteWalConverter possibility output only hashes instead real data - Fixes #7678. e7e34f0 is described below commit e7e34f0c3bcf080bc3d0da9b3e72ee72af9d0a32 Author: Kirill Tkalenko <tkalkir...@yandex.ru> AuthorDate: Wed Apr 22 18:28:14 2020 +0300 IGNITE-12906 Add to IgniteWalConverter possibility output only hashes instead real data - Fixes #7678. Signed-off-by: Ivan Rakov <ivan.glu...@gmail.com> --- .../org/apache/ignite/IgniteSystemProperties.java | 7 +- .../pagemem/wal/record/MetastoreDataRecord.java | 2 + .../pagemem/wal/record/UnwrapDataEntry.java | 14 +- .../pagemem/wal/record/UnwrapMvccDataEntry.java | 14 +- .../wal/record/WALRecordSerializationTest.java | 8 +- .../testframework/wal/record/RecordUtils.java | 10 + modules/dev-utils/pom.xml | 28 ++ .../ignite/development/utils/DataEntryWrapper.java | 89 +++++ .../development/utils/IgniteWalConverter.java | 65 +++- .../utils/MetastoreDataRecordWrapper.java | 53 +++ .../development/utils/ProcessSensitiveData.java | 32 ++ .../utils/ProcessSensitiveDataUtils.java | 56 +++ .../development/utils/DevUtilsTestSuite.java | 31 ++ .../utils/IgniteWalConverterSensitiveDataTest.java | 383 +++++++++++++++++++++ 14 files changed, 772 insertions(+), 20 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java index bcec9ed..66c3811 100644 --- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java +++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java @@ -1347,7 +1347,12 @@ public final class IgniteSystemProperties { if (val == null) return dflt; - return Enum.valueOf(enumCls, val); + try { + return Enum.valueOf(enumCls, val); + } + catch (IllegalArgumentException ignore) { + return dflt; + } } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java index 9e73424..5cff9ba 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/MetastoreDataRecord.java @@ -19,6 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record; import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage; +import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.internal.S; import org.jetbrains.annotations.Nullable; @@ -27,6 +28,7 @@ import org.jetbrains.annotations.Nullable; */ public class MetastoreDataRecord extends WALRecord implements WalRecordCacheGroupAware { /** */ + @GridToStringInclude(sensitive = true) private final String key; /** */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapDataEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapDataEntry.java index 5dd268b..ad5e1d0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapDataEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapDataEntry.java @@ -23,6 +23,8 @@ import org.apache.ignite.internal.processors.cache.CacheObjectValueContext; import org.apache.ignite.internal.processors.cache.GridCacheOperation; import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.SB; /** * Data Entry for automatic unwrapping key and value from Data Entry @@ -108,9 +110,13 @@ public class UnwrapDataEntry extends DataEntry implements UnwrappedDataEntry { /** {@inheritDoc} */ @Override public String toString() { - return getClass().getSimpleName() + "[k = " + unwrappedKey() + ", v = [ " - + unwrappedValue() - + "], super = [" - + super.toString() + "]]"; + SB sb = new SB(); + + sb.a(getClass().getSimpleName()).a('['); + + if (S.includeSensitive()) + sb.a("k = ").a(unwrappedKey()).a(", v = [ ").a(unwrappedValue()).a("], "); + + return sb.a("super = [").a(super.toString()).a("]]").toString(); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapMvccDataEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapMvccDataEntry.java index c3c12a3..47a9d7c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapMvccDataEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/UnwrapMvccDataEntry.java @@ -24,6 +24,8 @@ import org.apache.ignite.internal.processors.cache.GridCacheOperation; import org.apache.ignite.internal.processors.cache.KeyCacheObject; import org.apache.ignite.internal.processors.cache.mvcc.MvccVersion; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.SB; /** * Data Entry for automatic unwrapping key and value from Mvcc Data Entry @@ -112,9 +114,13 @@ public class UnwrapMvccDataEntry extends MvccDataEntry implements UnwrappedDataE /** {@inheritDoc} */ @Override public String toString() { - return getClass().getSimpleName() + "[k = " + unwrappedKey() + ", v = [ " - + unwrappedValue() - + "], super = [" - + super.toString() + "]]"; + SB sb = new SB(); + + sb.a(getClass().getSimpleName()).a('['); + + if (S.includeSensitive()) + sb.a("k = ").a(unwrappedKey()).a(", v = [ ").a(unwrappedValue()).a("], "); + + return sb.a("super = [").a(super.toString()).a("]]").toString(); } } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/wal/record/WALRecordSerializationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/wal/record/WALRecordSerializationTest.java index edd348d..bc3470f 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/wal/record/WALRecordSerializationTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/wal/record/WALRecordSerializationTest.java @@ -36,7 +36,6 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.testframework.wal.record.RecordUtils; -import org.apache.ignite.testframework.wal.record.UnsupportedWalRecord; import org.junit.Test; import org.mockito.internal.matchers.apachecommons.ReflectionEquals; @@ -108,7 +107,7 @@ public class WALRecordSerializationTest extends GridCommonAbstractTest { for (WALRecord.RecordType recordType : recordTypes) { WALRecord record = RecordUtils.buildWalRecord(recordType); - if (!(record instanceof UnsupportedWalRecord) && !(record instanceof SwitchSegmentRecord)) { + if (RecordUtils.isIncludeIntoLog(record)) { serializedRecords.add(new ReflectionEquals(record, "prev", "pos", "updateCounter" //updateCounter for PartitionMetaStateRecord isn't serialized. )); @@ -165,11 +164,8 @@ public class WALRecordSerializationTest extends GridCommonAbstractTest { for (WALRecord.RecordType recordType : recordTypes) { WALRecord record = RecordUtils.buildWalRecord(recordType); - if (!(record instanceof UnsupportedWalRecord) - && !(record instanceof SwitchSegmentRecord) - && (recordType.purpose() == WALRecord.RecordPurpose.LOGICAL || + if (RecordUtils.isIncludeIntoLog(record) && (recordType.purpose() == WALRecord.RecordPurpose.LOGICAL || recordType == WALRecord.RecordType.CHECKPOINT_RECORD)) { - serializedRecords.add(new ReflectionEquals(record, "prev", "pos", "updateCounter" //updateCounter for PartitionMetaStateRecord isn't serialized. )); diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/wal/record/RecordUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/wal/record/RecordUtils.java index 37dbeb8..793825e 100644 --- a/modules/core/src/test/java/org/apache/ignite/testframework/wal/record/RecordUtils.java +++ b/modules/core/src/test/java/org/apache/ignite/testframework/wal/record/RecordUtils.java @@ -571,4 +571,14 @@ public class RecordUtils { public static UnsupportedWalRecord buildBtreeMetaPageInitRootV3() { return new UnsupportedWalRecord(BTREE_META_PAGE_INIT_ROOT_V3); } + + /** + * Return {@code true} if include to write-ahead log. + * + * @param walRecord Instance of {@link WALRecord}. + * @return {@code True} if include to write-ahead log. + */ + public static boolean isIncludeIntoLog(WALRecord walRecord) { + return !UnsupportedWalRecord.class.isInstance(walRecord) && !SwitchSegmentRecord.class.isInstance(walRecord); + } } diff --git a/modules/dev-utils/pom.xml b/modules/dev-utils/pom.xml index e5f830f..d6cf4e0 100644 --- a/modules/dev-utils/pom.xml +++ b/modules/dev-utils/pom.xml @@ -46,6 +46,34 @@ <artifactId>ignite-indexing</artifactId> <version>${project.version}</version> </dependency> + + <dependency> + <groupId>org.apache.ignite</groupId> + <artifactId>ignite-core</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-context</artifactId> + <version>${spring.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-beans</artifactId> + <version>${spring.version}</version> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/DataEntryWrapper.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/DataEntryWrapper.java new file mode 100644 index 0000000..f96620a --- /dev/null +++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/DataEntryWrapper.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.development.utils; + +import org.apache.ignite.internal.pagemem.wal.record.DataEntry; +import org.apache.ignite.internal.pagemem.wal.record.UnwrappedDataEntry; +import org.apache.ignite.internal.util.typedef.internal.SB; +import org.jetbrains.annotations.Nullable; + +import static java.lang.String.valueOf; +import static java.util.Objects.isNull; +import static org.apache.ignite.development.utils.ProcessSensitiveData.HASH; +import static org.apache.ignite.development.utils.ProcessSensitiveData.MD5; +import static org.apache.ignite.development.utils.ProcessSensitiveDataUtils.md5; + +/** + * Wrapper {@link DataEntry} for sensitive data output. + */ +class DataEntryWrapper extends DataEntry { + /** Unwrapped DataEntry. */ + @Nullable private final UnwrappedDataEntry unwrappedDataEntry; + + /** Strategy for the processing of sensitive data. */ + private final ProcessSensitiveData sensitiveData; + + /** + * Constructor. + * + * @param dataEntry Instance of {@link DataEntry}. + * @param sensitiveData Strategy for the processing of sensitive data. + */ + public DataEntryWrapper( + DataEntry dataEntry, + ProcessSensitiveData sensitiveData + ) { + super( + dataEntry.cacheId(), + dataEntry.key(), + dataEntry.value(), + dataEntry.op(), + dataEntry.nearXidVersion(), + dataEntry.writeVersion(), + dataEntry.expireTime(), + dataEntry.partitionId(), + dataEntry.partitionCounter() + ); + + this.sensitiveData = sensitiveData; + + this.unwrappedDataEntry = UnwrappedDataEntry.class.isInstance(dataEntry) ? + (UnwrappedDataEntry) dataEntry : null; + } + + /** {@inheritDoc} */ + @Override public String toString() { + if (isNull(unwrappedDataEntry)) + return super.toString(); + + Object key = unwrappedDataEntry.unwrappedKey(); + Object value = unwrappedDataEntry.unwrappedValue(); + + if (HASH == sensitiveData) { + key = valueOf(key).hashCode(); + value = valueOf(value).hashCode(); + } + else if (MD5 == sensitiveData) { + key = md5(valueOf(key)); + value = md5(valueOf(value)); + } + + return new SB().a(unwrappedDataEntry.getClass().getSimpleName()) + .a("[k = ").a(key).a(", v = [ ").a(value).a("], super = [").a(super.toString()).a("]]").toString(); + } +} diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java index ca144ab..d9e9423 100644 --- a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java +++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java @@ -18,9 +18,12 @@ package org.apache.ignite.development.utils; import java.io.File; -import org.apache.ignite.IgniteSystemProperties; +import java.util.List; import org.apache.ignite.internal.pagemem.wal.WALIterator; import org.apache.ignite.internal.pagemem.wal.WALPointer; +import org.apache.ignite.internal.pagemem.wal.record.DataEntry; +import org.apache.ignite.internal.pagemem.wal.record.DataRecord; +import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; @@ -35,11 +38,34 @@ import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.logger.NullLogger; import org.jetbrains.annotations.Nullable; +import static java.util.stream.Collectors.toList; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_INCLUDE_SENSITIVE; +import static org.apache.ignite.IgniteSystemProperties.getBoolean; +import static org.apache.ignite.IgniteSystemProperties.getEnum; +import static org.apache.ignite.development.utils.ProcessSensitiveData.HIDE; +import static org.apache.ignite.development.utils.ProcessSensitiveData.SHOW; + /** * Print WAL log data in human-readable form. */ public class IgniteWalConverter { /** + * System property for printing {@link WALRecord}. By default, {@code false}. + */ + static final String PRINT_RECORDS = "PRINT_RECORDS"; + + /** + * System property for printing {@link WalStat}. By default, {@code true}. + */ + static final String PRINT_STAT = "PRINT_STAT"; + + /** + * System property for setting {@link ProcessSensitiveData strategy} of output sensitive data. + * By default, {@link ProcessSensitiveData#SHOW}. + */ + static final String SENSITIVE_DATA = "SENSITIVE_DATA"; + + /** * @param args Args. * @throws Exception If failed. */ @@ -54,8 +80,12 @@ public class IgniteWalConverter { H2ExtrasInnerIO.register(); H2ExtrasLeafIO.register(); - boolean printRecords = IgniteSystemProperties.getBoolean("PRINT_RECORDS", false); //TODO read them from argumetns - boolean printStat = IgniteSystemProperties.getBoolean("PRINT_STAT", true); //TODO read them from argumetns + boolean printRecords = getBoolean(PRINT_RECORDS, false); //TODO read them from argumetns + boolean printStat = getBoolean(PRINT_STAT, true); //TODO read them from argumetns + ProcessSensitiveData sensitiveData = getEnum(SENSITIVE_DATA, SHOW); //TODO read them from argumetns + + if (printRecords && HIDE == sensitiveData) + System.setProperty(IGNITE_TO_STRING_INCLUDE_SENSITIVE, Boolean.FALSE.toString()); final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(new NullLogger()); @@ -83,7 +113,7 @@ public class IgniteWalConverter { stat.registerRecord(record, pointer, true); if (printRecords) - System.out.println("[W] " + record); + System.out.println("[W] " + toString(record, sensitiveData)); } } @@ -101,7 +131,7 @@ public class IgniteWalConverter { stat.registerRecord(record, pointer, false); if (printRecords) - System.out.println("[A] " + record); + System.out.println("[A] " + toString(record, sensitiveData)); } } } @@ -111,4 +141,29 @@ public class IgniteWalConverter { if (stat != null) System.out.println("Statistic collected:\n" + stat.toString()); } + + /** + * Converting {@link WALRecord} to a string with sensitive data. + * + * @param walRecord Instance of {@link WALRecord}. + * @param sensitiveData Strategy for processing of sensitive data. + * @return String representation of {@link WALRecord}. + */ + private static String toString(WALRecord walRecord, ProcessSensitiveData sensitiveData) { + if (SHOW == sensitiveData || HIDE == sensitiveData) + return walRecord.toString(); + + if (MetastoreDataRecord.class.isInstance(walRecord)) + walRecord = new MetastoreDataRecordWrapper((MetastoreDataRecord)walRecord, sensitiveData); + else if (DataRecord.class.isInstance(walRecord)) { + DataRecord dataRecord = (DataRecord)walRecord; + + List<DataEntry> entryWrappers = dataRecord.writeEntries().stream() + .map(dataEntry -> new DataEntryWrapper(dataEntry, sensitiveData)).collect(toList()); + + dataRecord.setWriteEntries(entryWrappers); + } + + return walRecord.toString(); + } } diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/MetastoreDataRecordWrapper.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/MetastoreDataRecordWrapper.java new file mode 100644 index 0000000..f32c8f4 --- /dev/null +++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/MetastoreDataRecordWrapper.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.development.utils; + +import java.util.Arrays; +import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord; + +import static java.lang.String.valueOf; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.ignite.development.utils.ProcessSensitiveData.HASH; +import static org.apache.ignite.development.utils.ProcessSensitiveData.MD5; +import static org.apache.ignite.development.utils.ProcessSensitiveDataUtils.md5; + +/** + * Wrapper {@link MetastoreDataRecord} for sensitive data output. + */ +class MetastoreDataRecordWrapper extends MetastoreDataRecord { + /** + * Constructor. + * + * @param metastoreRecord Instance of {@link MetastoreDataRecord}. + * @param sensitiveData Strategy for the processing of sensitive data. + */ + public MetastoreDataRecordWrapper(MetastoreDataRecord metastoreRecord, ProcessSensitiveData sensitiveData) { + super( + HASH == sensitiveData ? valueOf(metastoreRecord.key().hashCode()) : + MD5 == sensitiveData ? md5(metastoreRecord.key()) : metastoreRecord.key(), + HASH == sensitiveData ? valueOf(Arrays.hashCode(metastoreRecord.value())).getBytes(UTF_8) : + MD5 == sensitiveData ? md5(Arrays.toString(metastoreRecord.value())).getBytes(UTF_8) : + metastoreRecord.value() + ); + + size(metastoreRecord.size()); + chainSize(metastoreRecord.chainSize()); + previous(metastoreRecord.previous()); + position(metastoreRecord.position()); + } +} diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/ProcessSensitiveData.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/ProcessSensitiveData.java new file mode 100644 index 0000000..8b30055 --- /dev/null +++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/ProcessSensitiveData.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.development.utils; + +/** + * Strategy for the processing of sensitive data. + */ +enum ProcessSensitiveData { + /** Show sensitive data. */ + SHOW, + /** Hide sensitive data. */ + HIDE, + /** Replace sensitive data with {@link Object#hashCode}. */ + HASH, + /** Replace sensitive data with MD5 hash. */ + MD5; +} diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/ProcessSensitiveDataUtils.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/ProcessSensitiveDataUtils.java new file mode 100644 index 0000000..4355568 --- /dev/null +++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/ProcessSensitiveDataUtils.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.development.utils; + +import java.security.MessageDigest; +import java.util.Base64; +import org.apache.ignite.IgniteException; + +import static java.nio.charset.StandardCharsets.UTF_8; + +/** + * Utility class for processing sensitive data. + */ +class ProcessSensitiveDataUtils { + /** + * Conversion to md5 hash string. + * + * @param val String value. + * @return MD5 hash string. + * */ + public static String md5(String val) { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + md.update(val.getBytes(UTF_8)); + + byte[] digest = md.digest(); + + return Base64.getEncoder().encodeToString(digest); + } + catch (Exception e) { + throw new IgniteException(e); + } + } + + /** + * Private constructor. + */ + private ProcessSensitiveDataUtils(){ + throw new RuntimeException("Don't create."); + } +} diff --git a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/DevUtilsTestSuite.java b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/DevUtilsTestSuite.java new file mode 100644 index 0000000..5f424d9 --- /dev/null +++ b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/DevUtilsTestSuite.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.development.utils; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** + * Test suite for dev utils. + */ +@RunWith(Suite.class) +@Suite.SuiteClasses({ + IgniteWalConverterSensitiveDataTest.class +}) +public class DevUtilsTestSuite { +} diff --git a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterSensitiveDataTest.java b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterSensitiveDataTest.java new file mode 100644 index 0000000..e2b2dd4 --- /dev/null +++ b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterSensitiveDataTest.java @@ -0,0 +1,383 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.development.utils; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.PrintStream; +import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.function.Function; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.QueryEntity; +import org.apache.ignite.cache.QueryIndex; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; +import org.apache.ignite.internal.pagemem.wal.record.DataEntry; +import org.apache.ignite.internal.pagemem.wal.record.DataRecord; +import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord; +import org.apache.ignite.internal.pagemem.wal.record.WALRecord; +import org.apache.ignite.internal.processors.cache.CacheObjectImpl; +import org.apache.ignite.internal.processors.cache.GridCacheOperation; +import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl; +import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.testframework.junits.WithSystemProperty; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.transactions.Transaction; +import org.junit.Test; + +import static java.lang.String.valueOf; +import static java.lang.System.setOut; +import static java.lang.System.setProperty; +import static java.util.Arrays.asList; +import static java.util.Objects.requireNonNull; +import static java.util.function.Function.identity; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_TO_STRING_INCLUDE_SENSITIVE; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.development.utils.IgniteWalConverter.PRINT_RECORDS; +import static org.apache.ignite.development.utils.IgniteWalConverter.SENSITIVE_DATA; +import static org.apache.ignite.testframework.GridTestUtils.assertContains; +import static org.apache.ignite.testframework.GridTestUtils.assertNotContains; +import static org.apache.ignite.testframework.wal.record.RecordUtils.isIncludeIntoLog; + +/** + * Class for testing sensitive data when reading {@link WALRecord} using + * {@link IgniteWalConverter}. + */ +public class IgniteWalConverterSensitiveDataTest extends GridCommonAbstractTest { + /** Sensitive data prefix. */ + private static final String SENSITIVE_DATA_VALUE_PREFIX = "must_hide_it_"; + + /** Path to directory where WAL is stored. */ + private static String walDirPath; + + /** Page size. */ + private static int pageSize; + + /** System out. */ + private static PrintStream sysOut; + + /** Sensitive data values. */ + private static List<String> sensitiveValues = new ArrayList<>(); + + /** + * Test out - can be injected via {@link #injectTestSystemOut()} instead + * of System.out and analyzed in test. + */ + private static ByteArrayOutputStream testOut; + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + sysOut = System.out; + testOut = new ByteArrayOutputStream(16 * 1024); + + int nodeId = 0; + + IgniteEx crd = startGrid(nodeId); + crd.cluster().active(true); + + try (Transaction tx = crd.transactions().txStart()) { + IgniteCache<Object, Object> cache = crd.cache(DEFAULT_CACHE_NAME); + + sensitiveValues.add(SENSITIVE_DATA_VALUE_PREFIX + 0); + sensitiveValues.add(SENSITIVE_DATA_VALUE_PREFIX + 1); + sensitiveValues.add(SENSITIVE_DATA_VALUE_PREFIX + 2); + + String val0 = sensitiveValues.get(0); + String val1 = sensitiveValues.get(1); + String val2 = sensitiveValues.get(2); + + cache.put(val0, val0); + cache.withKeepBinary().put(val1, val1); + cache.put(val2, new Person(1, val2)); + + tx.commit(); + } + + GridKernalContext kernalCtx = crd.context(); + IgniteWriteAheadLogManager wal = kernalCtx.cache().context().wal(); + + for (WALRecord walRecord : withSensitiveData()) { + if (isIncludeIntoLog(walRecord)) + wal.log(walRecord); + } + + sensitiveValues.add(SENSITIVE_DATA_VALUE_PREFIX); + + wal.flush(null, true); + + IgniteConfiguration cfg = crd.configuration(); + + String wd = cfg.getWorkDirectory(); + String wp = cfg.getDataStorageConfiguration().getWalPath(); + String fn = kernalCtx.pdsFolderResolver().resolveFolders().folderName(); + + walDirPath = wd + File.separator + wp + File.separator + fn; + pageSize = cfg.getDataStorageConfiguration().getPageSize(); + + stopGrid(nodeId); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + super.beforeTest(); + + clearGridToStringClassCache(); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + log.info("Test output for " + currentTestMethod()); + log.info("----------------------------------------"); + + setOut(sysOut); + + log.info(testOut.toString()); + resetTestOut(); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + super.afterTestsStopped(); + + stopAllGrids(); + + cleanPersistenceDir(); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName) + .setCacheConfiguration( + new CacheConfiguration<>(DEFAULT_CACHE_NAME) + .setAtomicityMode(TRANSACTIONAL) + .setQueryEntities(asList(personQueryEntity())) + ) + .setDataStorageConfiguration( + new DataStorageConfiguration().setDefaultDataRegionConfiguration( + new DataRegionConfiguration().setPersistenceEnabled(true) + ) + ); + } + + /** + * Test checks that by default {@link WALRecord} will not be output without + * system option {@link IgniteWalConverter#PRINT_RECORDS}. + * + * @throws Exception If failed. + */ + @Test + public void testNotPrintRecordsByDefault() throws Exception { + exeWithCheck(false, false, identity()); + } + + /** + * Test checks that by default sensitive data is displayed. + * + * @throws Exception If failed. + */ + @Test + @WithSystemProperty(key = PRINT_RECORDS, value = "true") + public void testShowSensitiveDataByDefault() throws Exception { + exeWithCheck(true, true, identity()); + } + + /** + * Test checks that sensitive data is displayed. + * + * @throws Exception If failed. + */ + @Test + @WithSystemProperty(key = PRINT_RECORDS, value = "true") + @WithSystemProperty(key = SENSITIVE_DATA, value = "SHOW") + public void testShowSensitiveData() throws Exception { + exeWithCheck(true, true, identity()); + + setProperty(SENSITIVE_DATA, currentTestMethod().getName()); + resetTestOut(); + + exeWithCheck(true, true, identity()); + } + + /** + * Test verifies that sensitive data will be hidden. + * + * @throws Exception If failed. + */ + @Test + @WithSystemProperty(key = PRINT_RECORDS, value = "true") + @WithSystemProperty(key = SENSITIVE_DATA, value = "HIDE") + @WithSystemProperty(key = IGNITE_TO_STRING_INCLUDE_SENSITIVE, value = "true") + public void testHideSensitiveData() throws Exception { + exeWithCheck(false, false, identity()); + } + + /** + * Test verifies that sensitive data should be replaced with hash. + * + * @throws Exception If failed. + */ + @Test + @WithSystemProperty(key = PRINT_RECORDS, value = "true") + @WithSystemProperty(key = SENSITIVE_DATA, value = "HASH") + public void testHashSensitiveData() throws Exception { + exeWithCheck(true, false, s -> valueOf(s.hashCode())); + } + + /** + * Test verifies that sensitive data should be replaced with MD5 hash. + * + * @throws Exception If failed. + */ + @Test + @WithSystemProperty(key = PRINT_RECORDS, value = "true") + @WithSystemProperty(key = SENSITIVE_DATA, value = "MD5") + public void testMd5HashSensitiveData() throws Exception { + exeWithCheck(true, false, ProcessSensitiveDataUtils::md5); + } + + /** + * Executing {@link IgniteWalConverter} with checking the content of its output. + * + * @param containsData Contains or not elements {@link #sensitiveValues} in utility output. + * @param containsPrefix Contains or not {@link #SENSITIVE_DATA_VALUE_PREFIX} in utility output. + * @param converter Converting elements {@link #sensitiveValues} for checking in utility output. + * @throws Exception If failed. + */ + private void exeWithCheck( + boolean containsData, + boolean containsPrefix, + Function<String, String> converter + ) throws Exception { + requireNonNull(converter); + + injectTestSystemOut(); + + IgniteWalConverter.main(new String[] {valueOf(pageSize), walDirPath}); + + String testOutStr = testOut.toString(); + + if (containsPrefix) + assertContains(log, testOutStr, SENSITIVE_DATA_VALUE_PREFIX); + else + assertNotContains(log, testOutStr, SENSITIVE_DATA_VALUE_PREFIX); + + for (String sensitiveDataValue : sensitiveValues) { + if (containsData) + assertContains(log, testOutStr, converter.apply(sensitiveDataValue)); + else + assertNotContains(log, testOutStr, converter.apply(sensitiveDataValue)); + } + } + + /** + * Inject {@link #testOut} to System.out for analyze in test. + */ + private void injectTestSystemOut() { + setOut(new PrintStream(testOut)); + } + + /** + * Reset {@link #testOut}. + */ + private void resetTestOut() { + testOut.reset(); + } + + /** + * Creating {@link WALRecord} instances with sensitive data. + * + * @return {@link WALRecord} instances with sensitive data. + */ + private Collection<WALRecord> withSensitiveData() { + List<WALRecord> walRecords = new ArrayList<>(); + + int cacheId = CU.cacheId(DEFAULT_CACHE_NAME); + + DataEntry dataEntry = new DataEntry( + cacheId, + new KeyCacheObjectImpl(SENSITIVE_DATA_VALUE_PREFIX, null, 0), + new CacheObjectImpl(SENSITIVE_DATA_VALUE_PREFIX, null), + GridCacheOperation.CREATE, + new GridCacheVersion(), + new GridCacheVersion(), + 0, + 0, + 0 + ); + + byte[] sensitiveDataBytes = SENSITIVE_DATA_VALUE_PREFIX.getBytes(StandardCharsets.UTF_8); + + walRecords.add(new DataRecord(dataEntry)); + walRecords.add(new MetastoreDataRecord(SENSITIVE_DATA_VALUE_PREFIX, sensitiveDataBytes)); + + return walRecords; + } + + /** + * Create {@link QueryEntity} for {@link Person}. + * + * @return QueryEntity for {@link Person}. + */ + private QueryEntity personQueryEntity() { + String orgIdField = "orgId"; + String nameField = "name"; + + return new QueryEntity() + .setKeyType(String.class.getName()) + .setValueType(Person.class.getName()) + .addQueryField(orgIdField, Integer.class.getName(), null) + .addQueryField(nameField, String.class.getName(), null) + .setIndexes(asList(new QueryIndex(nameField), new QueryIndex(orgIdField))); + } + + /** + * Simple class Person for tests. + */ + private static class Person implements Serializable { + /** Id organization. */ + int orgId; + + /** Name organization. */ + String name; + + /** + * Constructor. + * + * @param orgId Organization id. + * @param name Organization name. + */ + Person(int orgId, String name) { + this.orgId = orgId; + this.name = name; + } + } +}