Repository: ignite Updated Branches: refs/heads/master 2bdc89827 -> 091ace90e
IGNITE-9272: j.u.zip.CRC32algo instead of PureJavaCrc32 - Fixes #4619. Signed-off-by: Nikolay Izhikov <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/091ace90 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/091ace90 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/091ace90 Branch: refs/heads/master Commit: 091ace90e56bac1f7bc76886a5dccff796cbe34e Parents: 2bdc898 Author: Evgeny Stanilovskiy <[email protected]> Authored: Wed Oct 17 17:56:08 2018 +0300 Committer: Nikolay Izhikov <[email protected]> Committed: Wed Oct 17 17:56:08 2018 +0300 ---------------------------------------------------------------------- .../benchmarks/jmh/algo/BenchmarkCRC.java | 95 +++++++++++++++++ .../GridCacheDatabaseSharedManager.java | 4 +- .../cache/persistence/file/EncryptedFileIO.java | 7 +- .../cache/persistence/file/FilePageStore.java | 7 +- .../wal/FileWriteAheadLogManager.java | 4 +- .../wal/FsyncModeFileWriteAheadLogManager.java | 4 +- .../cache/persistence/wal/crc/FastCrc.java | 101 +++++++++++++++++++ .../persistence/wal/crc/PureJavaCrc32.java | 2 + .../cache/persistence/wal/io/FileInput.java | 10 +- .../persistence/wal/io/SimpleFileInput.java | 3 +- .../wal/serializer/RecordV1Serializer.java | 5 +- .../db/wal/crc/IgniteDataIntegrityTests.java | 6 +- .../wal/crc/IgnitePureJavaCrcCompatibility.java | 55 ++++++++++ .../ignite/testsuites/IgnitePdsTestSuite2.java | 2 + 14 files changed, 284 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/091ace90/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/algo/BenchmarkCRC.java ---------------------------------------------------------------------- diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/algo/BenchmarkCRC.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/algo/BenchmarkCRC.java new file mode 100644 index 0000000..5c922fe --- /dev/null +++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/algo/BenchmarkCRC.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.benchmarks.jmh.algo; + +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.nio.ByteBuffer; +import java.util.Random; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; +import static org.openjdk.jmh.annotations.Mode.AverageTime; +import static org.openjdk.jmh.annotations.Scope.Thread; + +/** + * + */ +@State(Thread) +@OutputTimeUnit(NANOSECONDS) +@BenchmarkMode(AverageTime) +@Fork(value = 1, jvmArgsAppend = {"-XX:+UnlockDiagnosticVMOptions"}) +@Warmup(iterations = 5) +@Measurement(iterations = 5) +public class BenchmarkCRC { + /** */ + static final int SIZE = 1024; + + /** */ + static final int BUF_LEN = 4096; + + /** */ + @State(Thread) + public static class Context { + /** */ + final int[] results = new int[SIZE]; + + /** */ + final ByteBuffer bb = ByteBuffer.allocate(BUF_LEN); + + /** */ + @Setup + public void setup() { + new Random().ints(BUF_LEN, Byte.MIN_VALUE, Byte.MAX_VALUE).forEach(k -> bb.put((byte) k)); + } + } + + /** */ + @Benchmark + public int[] pureJavaCrc32(Context context) { + for (int i = 0; i < SIZE; i++) { + context.bb.rewind(); + + context.results[i] = PureJavaCrc32.calcCrc32(context.bb, BUF_LEN); + } + + return context.results; + } + + /** */ + @Benchmark + public int[] crc32(Context context) { + for (int i = 0; i < SIZE; i++) { + context.bb.rewind(); + + context.results[i] = FastCrc.calcCrc(context.bb, BUF_LEN); + } + + return context.results; + } +} + + http://git-wip-us.apache.org/repos/asf/ignite/blob/091ace90/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java index ea3e314..33f593d 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java @@ -132,8 +132,8 @@ import org.apache.ignite.internal.processors.cache.persistence.snapshot.Snapshot import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.apache.ignite.internal.processors.port.GridPortRecord; import org.apache.ignite.internal.util.GridMultiCollectionWrapper; import org.apache.ignite.internal.util.GridUnsafe; @@ -4142,7 +4142,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan } if (!skipCrc) { - PageIO.setCrc(writeAddr, PureJavaCrc32.calcCrc32(tmpWriteBuf, pageSize())); + PageIO.setCrc(writeAddr, FastCrc.calcCrc(tmpWriteBuf, pageSize())); tmpWriteBuf.rewind(); } http://git-wip-us.apache.org/repos/asf/ignite/blob/091ace90/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java index 008a728..86d9bbc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/EncryptedFileIO.java @@ -21,10 +21,11 @@ import java.io.IOException; import java.io.Serializable; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; + +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.spi.encryption.EncryptionSpi; import org.apache.ignite.internal.managers.encryption.GridEncryptionManager; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.apache.ignite.internal.util.typedef.internal.CU; /** @@ -260,7 +261,7 @@ public class EncryptedFileIO implements FileIO { * @param res Destination buffer. */ private void storeCRC(ByteBuffer res) { - int crc = PureJavaCrc32.calcCrc32(res, encryptedDataSize()); + int crc = FastCrc.calcCrc(res, encryptedDataSize()); res.put((byte) (crc >> 24)); res.put((byte) (crc >> 16)); @@ -274,7 +275,7 @@ public class EncryptedFileIO implements FileIO { * @param encrypted Encrypted data buffer. */ private void checkCRC(ByteBuffer encrypted) throws IOException { - int crc = PureJavaCrc32.calcCrc32(encrypted, encryptedDataSize()); + int crc = FastCrc.calcCrc(encrypted, encryptedDataSize()); int storedCrc = 0; http://git-wip-us.apache.org/repos/asf/ignite/blob/091ace90/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java index 2e07867..fdf4705 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java @@ -27,6 +27,7 @@ import java.nio.file.Files; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; + import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.configuration.DataStorageConfiguration; @@ -35,8 +36,8 @@ import org.apache.ignite.internal.pagemem.store.PageStore; import org.apache.ignite.internal.processors.cache.persistence.AllocatedPageTracker; import org.apache.ignite.internal.processors.cache.persistence.StorageException; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.apache.ignite.internal.util.typedef.internal.U; import static java.nio.file.StandardOpenOption.CREATE; @@ -358,7 +359,7 @@ public class FilePageStore implements PageStore { pageBuf.position(0); if (!skipCrc) { - int curCrc32 = PureJavaCrc32.calcCrc32(pageBuf, pageSize); + int curCrc32 = FastCrc.calcCrc(pageBuf, pageSize); if ((savedCrc32 ^ curCrc32) != 0) throw new IgniteDataIntegrityViolationException("Failed to read page (CRC validation failed) " + @@ -613,7 +614,7 @@ public class FilePageStore implements PageStore { try { pageBuf.position(0); - return PureJavaCrc32.calcCrc32(pageBuf, pageSize); + return FastCrc.calcCrc(pageBuf, pageSize); } finally { pageBuf.position(0); http://git-wip-us.apache.org/repos/asf/ignite/blob/091ace90/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java index 5c6502b..8198149 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java @@ -98,8 +98,8 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStor import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings; import org.apache.ignite.internal.processors.cache.persistence.wal.aware.SegmentAware; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.io.LockedSegmentFileInputFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory; @@ -2438,7 +2438,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl buf.position(0); // This call will move buffer position to the end of the record again. - int crcVal = PureJavaCrc32.calcCrc32(buf, curPos); + int crcVal = FastCrc.calcCrc(buf, curPos); buf.putInt(crcVal); } http://git-wip-us.apache.org/repos/asf/ignite/blob/091ace90/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java index 7bfd10b..917640d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java @@ -91,8 +91,8 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO; @@ -2339,7 +2339,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda buf.position(0); // This call will move buffer position to the end of the record again. - int crcVal = PureJavaCrc32.calcCrc32(buf, curPos); + int crcVal = FastCrc.calcCrc(buf, curPos); buf.putInt(crcVal); } http://git-wip-us.apache.org/repos/asf/ignite/blob/091ace90/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/FastCrc.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/FastCrc.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/FastCrc.java new file mode 100644 index 0000000..0dcbafd --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/FastCrc.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.wal.crc; + +import java.nio.ByteBuffer; +import java.util.zip.CRC32; + +/** + * This CRC calculation implementation workf much faster then {@link PureJavaCrc32} + */ +public final class FastCrc { + /** CRC algo. */ + private static final ThreadLocal<CRC32> CRC = ThreadLocal.withInitial(CRC32::new); + + /** */ + private final CRC32 crc = new CRC32(); + + /** + * Current value. + */ + private int val; + + /** */ + public FastCrc() { + reset(); + } + + /** + * Preparation for further calculations. + */ + public void reset() { + val = 0xffffffff; + + crc.reset(); + } + + /** + * @return crc value. + */ + public int getValue() { + return val; + } + + /** + * @param buf Input buffer. + * @param len Data length. + */ + public void update(final ByteBuffer buf, final int len) { + val = calcCrc(crc, buf, len); + } + + /** + * @param buf Input buffer. + * @param len Data length. + * + * @return Crc checksum. + */ + public static int calcCrc(ByteBuffer buf, int len) { + CRC32 crcAlgo = CRC.get(); + + int res = calcCrc(crcAlgo, buf, len); + + crcAlgo.reset(); + + return res; + } + + /** + * @param crcAlgo CRC algorithm. + * @param buf Input buffer. + * @param len Buffer length. + * + * @return Crc checksum. + */ + private static int calcCrc(CRC32 crcAlgo, ByteBuffer buf, int len) { + int initLimit = buf.limit(); + + buf.limit(buf.position() + len); + + crcAlgo.update(buf); + + buf.limit(initLimit); + + return (int)crcAlgo.getValue() ^ 0xFFFFFFFF; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/091ace90/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java index 5e89a40..3f348d7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java @@ -29,7 +29,9 @@ import java.nio.ByteBuffer; * succession. * * The current version is ~10x to 1.8x as fast as Sun's native java.util.zip.CRC32 in Java 1.6 + * @deprecated Use {@link FastCrc} instead. */ +@Deprecated public class PureJavaCrc32 { /** * the current CRC value http://git-wip-us.apache.org/repos/asf/ignite/blob/091ace90/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java index d19d17b..c9615f5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/FileInput.java @@ -19,10 +19,12 @@ package org.apache.ignite.internal.processors.cache.persistence.wal.io; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.zip.CRC32; + import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.jetbrains.annotations.NotNull; /** @@ -56,7 +58,7 @@ public interface FileInput extends ByteBufferBackedDataInput { */ public class Crc32CheckingFileInput implements ByteBufferBackedDataInput, AutoCloseable { /** */ - private final PureJavaCrc32 crc32 = new PureJavaCrc32(); + private final FastCrc crc = new FastCrc(); /** Last calc position. */ private int lastCalcPosition; @@ -93,7 +95,7 @@ public interface FileInput extends ByteBufferBackedDataInput { @Override public void close() throws Exception { updateCrc(); - int val = crc32.getValue(); + int val = crc.getValue(); int writtenCrc = this.readInt(); @@ -118,7 +120,7 @@ public interface FileInput extends ByteBufferBackedDataInput { buffer().position(lastCalcPosition); - crc32.update(delegate.buffer(), oldPos - lastCalcPosition); + crc.update(delegate.buffer(), oldPos - lastCalcPosition); lastCalcPosition = oldPos; } http://git-wip-us.apache.org/repos/asf/ignite/blob/091ace90/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleFileInput.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleFileInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleFileInput.java index 5918b0b..1a1562e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleFileInput.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/io/SimpleFileInput.java @@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.cache.persistence.wal.io; import java.io.EOFException; import java.io.IOException; import java.nio.ByteBuffer; + import org.apache.ignite.internal.processors.cache.persistence.file.FileIO; import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander; import org.jetbrains.annotations.NotNull; @@ -264,7 +265,7 @@ public class SimpleFileInput implements FileInput { /** * @param skipCheck If CRC check should be skipped. - * @return autoclosable fileInput, after its closing crc32 will be calculated and compared with saved one + * @return autoclosable fileInput, after its closing crc will be calculated and compared with saved one */ public Crc32CheckingFileInput startRead(boolean skipCheck) { return new Crc32CheckingFileInput(this, skipCheck); http://git-wip-us.apache.org/repos/asf/ignite/blob/091ace90/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java index c65f37c..31efa1f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java @@ -22,6 +22,7 @@ import java.io.EOFException; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; + import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.internal.pagemem.wal.WALPointer; @@ -32,6 +33,7 @@ import org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType; import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO; import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput; import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentFileInputFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; @@ -39,7 +41,6 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentEofExc import org.apache.ignite.internal.processors.cache.persistence.wal.io.SegmentIO; import org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleFileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.WalSegmentTailReachedException; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord; import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.io.RecordIO; import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; @@ -421,7 +422,7 @@ public class RecordV1Serializer implements RecordSerializer { buf.position(startPos); // This call will move buffer position to the end of the record again. - int crcVal = PureJavaCrc32.calcCrc32(buf, curPos - startPos); + int crcVal = FastCrc.calcCrc(buf, curPos - startPos); buf.putInt(crcVal); } http://git-wip-us.apache.org/repos/asf/ignite/blob/091ace90/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java index 59dd3b7..3ad4c90 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java @@ -24,13 +24,14 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.concurrent.ThreadLocalRandom; + import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; import org.apache.ignite.internal.processors.cache.persistence.wal.io.FileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.io.SimpleFileInput; import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException; -import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; /** * @@ -59,6 +60,7 @@ public class IgniteDataIntegrityTests extends TestCase { ); ByteBuffer buf = ByteBuffer.allocate(1024); + ThreadLocalRandom curr = ThreadLocalRandom.current(); for (int i = 0; i < 1024; i+=16) { @@ -66,7 +68,7 @@ public class IgniteDataIntegrityTests extends TestCase { buf.putInt(curr.nextInt()); buf.putInt(curr.nextInt()); buf.position(i); - buf.putInt(PureJavaCrc32.calcCrc32(buf, 12)); + buf.putInt(FastCrc.calcCrc(buf, 12)); } buf.rewind(); http://git-wip-us.apache.org/repos/asf/ignite/blob/091ace90/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgnitePureJavaCrcCompatibility.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgnitePureJavaCrcCompatibility.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgnitePureJavaCrcCompatibility.java new file mode 100644 index 0000000..faafad2 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgnitePureJavaCrcCompatibility.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.db.wal.crc; + +import junit.framework.TestCase; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.FastCrc; +import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32; + +import java.nio.ByteBuffer; +import java.util.concurrent.ThreadLocalRandom; + +/** + * PureJavaCrc32 previous crc algo realization vs java.util.zip.crc32 test. + */ +public class IgnitePureJavaCrcCompatibility extends TestCase { + /** + * Test crc algo equality results. + * @throws Exception + */ + public void testAlgoEqual() throws Exception { + ByteBuffer buf = ByteBuffer.allocate(1024); + + ThreadLocalRandom curr = ThreadLocalRandom.current(); + + for (int i = 0; i < 1024; i+=16) { + buf.putInt(curr.nextInt()); + buf.putInt(curr.nextInt()); + buf.putInt(curr.nextInt()); + buf.position(i); + + buf.position(i); + int crc0 = FastCrc.calcCrc(buf, 12); + + buf.position(i); + int crc1 = PureJavaCrc32.calcCrc32(buf, 12); + + assertEquals(crc0, crc1); + } + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/091ace90/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java index bf83bf5..ede5cab 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java @@ -62,6 +62,7 @@ import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.Ignite import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteFsyncReplayWalIteratorInvalidCrcTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteReplayWalIteratorInvalidCrcTest; import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteStandaloneWalIteratorInvalidCrcTest; +import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgnitePureJavaCrcCompatibility; import org.apache.ignite.internal.processors.cache.persistence.db.wal.reader.IgniteWalReaderTest; import org.apache.ignite.internal.processors.cache.persistence.wal.reader.StandaloneWalRecordsIteratorTest; @@ -80,6 +81,7 @@ public class IgnitePdsTestSuite2 extends TestSuite { suite.addTestSuite(IgniteStandaloneWalIteratorInvalidCrcTest.class); suite.addTestSuite(IgniteReplayWalIteratorInvalidCrcTest.class); suite.addTestSuite(IgniteFsyncReplayWalIteratorInvalidCrcTest.class); + suite.addTestSuite(IgnitePureJavaCrcCompatibility.class); addRealPageStoreTests(suite);
