This is an automated email from the ASF dual-hosted git repository. ivandasch pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push: new 21ffc3e92ea IGNITE-18584 Enlarge compression buffer in order to provide sufficient space for compressors. (#10490) 21ffc3e92ea is described below commit 21ffc3e92ea4ea13ece8a3b45094185665e8cd69 Author: Ivan Daschinskiy <ivanda...@apache.org> AuthorDate: Fri Jan 20 14:07:39 2023 +0300 IGNITE-18584 Enlarge compression buffer in order to provide sufficient space for compressors. (#10490) --- .../compress/CompressionProcessorImpl.java | 13 +- .../compress/CompressionProcessorTest.java | 864 +++------------------ .../platforms/cpp/core/src/jni/os/linux/utils.cpp | 2 +- .../platforms/cpp/core/src/jni/os/win/utils.cpp | 2 +- 4 files changed, 119 insertions(+), 762 deletions(-) diff --git a/modules/compress/src/main/java/org/apache/ignite/internal/processors/compress/CompressionProcessorImpl.java b/modules/compress/src/main/java/org/apache/ignite/internal/processors/compress/CompressionProcessorImpl.java index 623ff50ee3d..d2e22caf702 100644 --- a/modules/compress/src/main/java/org/apache/ignite/internal/processors/compress/CompressionProcessorImpl.java +++ b/modules/compress/src/main/java/org/apache/ignite/internal/processors/compress/CompressionProcessorImpl.java @@ -50,8 +50,8 @@ public class CompressionProcessorImpl extends CompressionProcessor { /** Max page size. */ private final ThreadLocalByteBuffer compactBuf = new ThreadLocalByteBuffer(MAX_PAGE_SIZE); - /** A bit more than max page size. */ - private final ThreadLocalByteBuffer compressBuf = new ThreadLocalByteBuffer(MAX_PAGE_SIZE + 1024); + /** A bit more than max page size, extra space is required by compressors. */ + private final ThreadLocalByteBuffer compressBuf = new ThreadLocalByteBuffer(maxCompressedBufferSize(MAX_PAGE_SIZE)); /** * @param ctx Kernal context. @@ -420,6 +420,15 @@ public class CompressionProcessorImpl extends CompressionProcessor { setCompressionInfo(page, DiskPageCompression.DISABLED, 0, 0); } + /** */ + private static int maxCompressedBufferSize(int baseSz) { + int lz4Sz = Lz4.fastCompressor.maxCompressedLength(baseSz); + int zstdSz = (int)Zstd.compressBound(baseSz); + int snappySz = Snappy.maxCompressedLength(baseSz); + + return Math.max(Math.max(lz4Sz, zstdSz), snappySz); + } + /** */ static class Lz4 { /** */ diff --git a/modules/compress/src/test/java/org/apache/ignite/internal/processors/compress/CompressionProcessorTest.java b/modules/compress/src/test/java/org/apache/ignite/internal/processors/compress/CompressionProcessorTest.java index ff328e89bff..f499059a1fe 100644 --- a/modules/compress/src/test/java/org/apache/ignite/internal/processors/compress/CompressionProcessorTest.java +++ b/modules/compress/src/test/java/org/apache/ignite/internal/processors/compress/CompressionProcessorTest.java @@ -42,7 +42,10 @@ import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.testframework.junits.GridTestKernalContext; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import static org.apache.ignite.configuration.DataStorageConfiguration.MAX_PAGE_SIZE; import static org.apache.ignite.configuration.DiskPageCompression.LZ4; import static org.apache.ignite.configuration.DiskPageCompression.SKIP_GARBAGE; import static org.apache.ignite.configuration.DiskPageCompression.SNAPPY; @@ -50,741 +53,165 @@ import static org.apache.ignite.configuration.DiskPageCompression.ZSTD; import static org.apache.ignite.internal.processors.compress.CompressionProcessor.LZ4_MAX_LEVEL; import static org.apache.ignite.internal.processors.compress.CompressionProcessor.LZ4_MIN_LEVEL; import static org.apache.ignite.internal.processors.compress.CompressionProcessor.UNCOMPRESSED_PAGE; +import static org.apache.ignite.internal.processors.compress.CompressionProcessor.ZSTD_DEFAULT_LEVEL; import static org.apache.ignite.internal.processors.compress.CompressionProcessor.ZSTD_MAX_LEVEL; import static org.apache.ignite.internal.processors.compress.CompressionProcessorImpl.allocateDirectBuffer; -import static org.apache.ignite.internal.processors.compress.CompressionProcessorTest.TestInnerIO.INNER_IO; -import static org.apache.ignite.internal.processors.compress.CompressionProcessorTest.TestLeafIO.LEAF_IO; import static org.apache.ignite.internal.util.GridUnsafe.bufferAddress; /** */ +@RunWith(Parameterized.class) public class CompressionProcessorTest extends GridCommonAbstractTest { /** */ - private static final int ITEM_SIZE = 6; // To fill the whole page. - - /** */ - private int blockSize = 16; - - /** */ - private int pageSize = 4 * 1024; - - /** */ - private DiskPageCompression compression; - - /** */ - private int compressLevel; - - /** */ - private CompressionProcessor p; - - /** {@inheritDoc} */ - @Override protected void beforeTestsStarted() throws Exception { - super.beforeTestsStarted(); - - PageIO.registerTest(INNER_IO, LEAF_IO); - } - - /** {@inheritDoc} */ - @Override protected void beforeTest() { - p = new CompressionProcessorImpl(new GridTestKernalContext(log)); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageCompact16() throws IgniteCheckedException { - blockSize = 16; - compression = SKIP_GARBAGE; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageCompact128() throws IgniteCheckedException { - blockSize = 128; - compression = SKIP_GARBAGE; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageCompact1k() throws IgniteCheckedException { - blockSize = 1024; - compression = SKIP_GARBAGE; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageCompact2k() throws IgniteCheckedException { - blockSize = 2 * 1024; - compression = SKIP_GARBAGE; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageZstd16() throws IgniteCheckedException { - blockSize = 16; - compression = ZSTD; - compressLevel = ZSTD_MAX_LEVEL; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageZstd128() throws IgniteCheckedException { - blockSize = 128; - compression = ZSTD; - compressLevel = ZSTD_MAX_LEVEL; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageZstd1k() throws IgniteCheckedException { - blockSize = 1024; - compression = ZSTD; - compressLevel = ZSTD_MAX_LEVEL; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageZstd2k() throws IgniteCheckedException { - blockSize = 2 * 1024; - compression = ZSTD; - compressLevel = ZSTD_MAX_LEVEL; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageSnappy16() throws IgniteCheckedException { - blockSize = 16; - compression = SNAPPY; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageSnappy128() throws IgniteCheckedException { - blockSize = 128; - compression = SNAPPY; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageSnappy1k() throws IgniteCheckedException { - blockSize = 1024; - compression = SNAPPY; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageSnappy2k() throws IgniteCheckedException { - blockSize = 2 * 1024; - compression = SNAPPY; - - doTestDataPage(); - } - - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageLz4Fast16() throws IgniteCheckedException { - blockSize = 16; - compression = LZ4; - compressLevel = LZ4_MIN_LEVEL; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageLz4Fast128() throws IgniteCheckedException { - blockSize = 128; - compression = LZ4; - compressLevel = LZ4_MIN_LEVEL; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageLz4Fast1k() throws IgniteCheckedException { - blockSize = 1024; - compression = LZ4; - compressLevel = LZ4_MIN_LEVEL; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageLz4Fast2k() throws IgniteCheckedException { - blockSize = 2 * 1024; - compression = LZ4; - compressLevel = LZ4_MIN_LEVEL; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageLz4Slow16() throws IgniteCheckedException { - blockSize = 16; - compression = LZ4; - compressLevel = LZ4_MAX_LEVEL; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageLz4Slow128() throws IgniteCheckedException { - blockSize = 128; - compression = LZ4; - compressLevel = LZ4_MAX_LEVEL; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageLz4Slow1k() throws IgniteCheckedException { - blockSize = 1024; - compression = LZ4; - compressLevel = LZ4_MAX_LEVEL; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testDataPageLz4Slow2k() throws IgniteCheckedException { - blockSize = 2 * 1024; - compression = LZ4; - compressLevel = LZ4_MAX_LEVEL; - - doTestDataPage(); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageCompact16() throws IgniteCheckedException { - blockSize = 16; - compression = SKIP_GARBAGE; - - doTestBTreePage(INNER_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageCompact16() throws IgniteCheckedException { - blockSize = 16; - compression = SKIP_GARBAGE; - - doTestBTreePage(LEAF_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageZstd16() throws IgniteCheckedException { - blockSize = 16; - compression = ZSTD; - compressLevel = ZSTD_MAX_LEVEL; - - doTestBTreePage(INNER_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageZstd16() throws IgniteCheckedException { - blockSize = 16; - compression = ZSTD; - compressLevel = ZSTD_MAX_LEVEL; - - doTestBTreePage(LEAF_IO); - } - - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageLz4Fast16() throws IgniteCheckedException { - blockSize = 16; - compression = LZ4; - compressLevel = LZ4_MIN_LEVEL; - - doTestBTreePage(INNER_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageLz4Fast16() throws IgniteCheckedException { - blockSize = 16; - compression = LZ4; - compressLevel = LZ4_MIN_LEVEL; - - doTestBTreePage(LEAF_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageLz4Slow16() throws IgniteCheckedException { - blockSize = 16; - compression = LZ4; - compressLevel = LZ4_MAX_LEVEL; - - doTestBTreePage(INNER_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageLz4Slow16() throws IgniteCheckedException { - blockSize = 16; - compression = LZ4; - compressLevel = LZ4_MAX_LEVEL; - - doTestBTreePage(LEAF_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageSnappy16() throws IgniteCheckedException { - blockSize = 16; - compression = SNAPPY; - - doTestBTreePage(INNER_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageSnappy16() throws IgniteCheckedException { - blockSize = 16; - compression = SNAPPY; - - doTestBTreePage(LEAF_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageCompact128() throws IgniteCheckedException { - blockSize = 128; - compression = SKIP_GARBAGE; - - doTestBTreePage(INNER_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageCompact128() throws IgniteCheckedException { - blockSize = 128; - compression = SKIP_GARBAGE; - - doTestBTreePage(LEAF_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageZstd128() throws IgniteCheckedException { - blockSize = 128; - compression = ZSTD; - compressLevel = ZSTD_MAX_LEVEL; - - doTestBTreePage(INNER_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageZstd128() throws IgniteCheckedException { - blockSize = 128; - compression = ZSTD; - compressLevel = ZSTD_MAX_LEVEL; - - doTestBTreePage(LEAF_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageLz4Fast128() throws IgniteCheckedException { - blockSize = 128; - compression = LZ4; - compressLevel = LZ4_MIN_LEVEL; - - doTestBTreePage(INNER_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageLz4Fast128() throws IgniteCheckedException { - blockSize = 128; - compression = LZ4; - compressLevel = LZ4_MIN_LEVEL; - - doTestBTreePage(LEAF_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageLz4Slow128() throws IgniteCheckedException { - blockSize = 128; - compression = LZ4; - compressLevel = LZ4_MAX_LEVEL; - - doTestBTreePage(INNER_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageLz4Slow128() throws IgniteCheckedException { - blockSize = 128; - compression = LZ4; - compressLevel = LZ4_MAX_LEVEL; - - doTestBTreePage(LEAF_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageSnappy128() throws IgniteCheckedException { - blockSize = 128; - compression = SNAPPY; - - doTestBTreePage(INNER_IO); - } - - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageSnappy128() throws IgniteCheckedException { - blockSize = 128; - compression = SNAPPY; + @Parameterized.Parameters(name = "page_size = {0}, block_size = {1}, compression = {2}, level = {3}") + public static Iterable<Object[]> testParameters() { + List<Object[]> params = new ArrayList<>(); + + for (int pageSz = 4 * 1024; pageSz <= MAX_PAGE_SIZE; pageSz *= 2) { + for (int blockSize : Arrays.asList(16, 128, 1024, 2048)) { + params.add(new Object[] {pageSz, blockSize, SKIP_GARBAGE, 0}); + params.add(new Object[] {pageSz, blockSize, SNAPPY, 0}); + params.add(new Object[] {pageSz, blockSize, LZ4, LZ4_MIN_LEVEL}); + params.add(new Object[] {pageSz, blockSize, LZ4, LZ4_MAX_LEVEL}); + params.add(new Object[] {pageSz, blockSize, ZSTD, ZSTD_DEFAULT_LEVEL}); + params.add(new Object[] {pageSz, blockSize, ZSTD, ZSTD_MAX_LEVEL}); + } + } - doTestBTreePage(LEAF_IO); + return params; } - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageCompact1k() throws IgniteCheckedException { - blockSize = 1024; - compression = SKIP_GARBAGE; - - doTestBTreePage(INNER_IO); - } + /** */ + @Parameterized.Parameter + public int pageSize; - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageCompact1k() throws IgniteCheckedException { - blockSize = 1024; - compression = SKIP_GARBAGE; + /** */ + @Parameterized.Parameter(1) + public int blockSize; - doTestBTreePage(LEAF_IO); - } + /** */ + @Parameterized.Parameter(2) + public DiskPageCompression compression; - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageZstd1k() throws IgniteCheckedException { - blockSize = 1024; - compression = ZSTD; - compressLevel = ZSTD_MAX_LEVEL; + /** */ + @Parameterized.Parameter(3) + public int compressLevel; - doTestBTreePage(INNER_IO); - } + /** */ + private CompressionProcessor p; - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageZstd1k() throws IgniteCheckedException { - blockSize = 1024; - compression = ZSTD; - compressLevel = ZSTD_MAX_LEVEL; + /** */ + private TestInnerIO innerIo; - doTestBTreePage(LEAF_IO); - } + /** */ + private TestLeafIO leafIo; - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageLz4Fast1k() throws IgniteCheckedException { - blockSize = 1024; - compression = LZ4; - compressLevel = LZ4_MIN_LEVEL; + /** {@inheritDoc} */ + @Override protected void beforeTest() { + /** + * 7 is divisor of 8192 - 58, 6 is divisor of 4096 - 58 and 16384 - 58. So an item size should be + * 7 for a 8k page and 6 for 4k and 16k pages respectively in order to fill the page fully. + * See {@see org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO#getMaxCount} + */ + int itemSize = pageSize == 8192 ? 7 : 6; - doTestBTreePage(INNER_IO); - } + innerIo = new TestInnerIO(itemSize); + leafIo = new TestLeafIO(itemSize); - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageLz4Fast1k() throws IgniteCheckedException { - blockSize = 1024; - compression = LZ4; - compressLevel = LZ4_MIN_LEVEL; + PageIO.registerTest(innerIo, leafIo); - doTestBTreePage(LEAF_IO); + p = new CompressionProcessorImpl(new GridTestKernalContext(log)); } /** * @throws IgniteCheckedException If failed. */ @Test - public void testInnerPageLz4Slow1k() throws IgniteCheckedException { - blockSize = 1024; - compression = LZ4; - compressLevel = LZ4_MAX_LEVEL; - - doTestBTreePage(INNER_IO); - } + public void testDataPage() throws IgniteCheckedException { + Random rnd = ThreadLocalRandom.current(); - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageLz4Slow1k() throws IgniteCheckedException { - blockSize = 1024; - compression = LZ4; - compressLevel = LZ4_MAX_LEVEL; + final byte[][] rows = new byte[][]{ + new byte[17], new byte[37], new byte[71] + }; - doTestBTreePage(LEAF_IO); - } + for (int i = 0; i < rows.length; i++) + rnd.nextBytes(rows[i]); - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageSnappy1k() throws IgniteCheckedException { - blockSize = 1024; - compression = SNAPPY; + ByteBuffer page = allocateDirectBuffer(pageSize); + long pageAddr = bufferAddress(page); - doTestBTreePage(INNER_IO); - } + SimpleDataPageIO io = SimpleDataPageIO.VERSIONS.latest(); - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageSnappy1k() throws IgniteCheckedException { - blockSize = 1024; - compression = SNAPPY; + long pageId = PageIdUtils.pageId(PageIdAllocator.MAX_PARTITION_ID, PageIdAllocator.FLAG_DATA, 171717); - doTestBTreePage(LEAF_IO); - } + io.initNewPage(pageAddr, pageId, pageSize, null); - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageCompact2k() throws IgniteCheckedException { - blockSize = 2 * 1024; - compression = SKIP_GARBAGE; + checkIo(io, page); - doTestBTreePage(INNER_IO); - } + Function<ByteBuffer, List<Bytes>> getContents = (buf) -> { + try { + long addr = bufferAddress(buf); - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageCompact2k() throws IgniteCheckedException { - blockSize = 2 * 1024; - compression = SKIP_GARBAGE; + return io.forAllItems(addr, (link) -> { + DataPagePayload payload = io.readPayload(addr, PageIdUtils.itemId(link), pageSize); - doTestBTreePage(LEAF_IO); - } + return new Bytes(payload.getBytes(addr)); + }); + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + }; - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageZstd2k() throws IgniteCheckedException { - blockSize = 2 * 1024; - compression = ZSTD; - compressLevel = ZSTD_MAX_LEVEL; + // Empty data page. + checkCompressDecompress(page, getContents, false); - doTestBTreePage(INNER_IO); - } + GridIntList itemIds = new GridIntList(); - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageZstd2k() throws IgniteCheckedException { - blockSize = 2 * 1024; - compression = ZSTD; - compressLevel = ZSTD_MAX_LEVEL; + for (;;) { + byte[] row = rows[rnd.nextInt(rows.length)]; - doTestBTreePage(LEAF_IO); - } + if (io.getFreeSpace(pageAddr) < row.length) + break; - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageLz4Fast2k() throws IgniteCheckedException { - blockSize = 2 * 1024; - compression = LZ4; - compressLevel = LZ4_MIN_LEVEL; + itemIds.add(io.addRow(pageAddr, row, pageSize)); + } - doTestBTreePage(INNER_IO); - } + int freeSpace = io.getFreeSpace(pageAddr); - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageLz4Fast2k() throws IgniteCheckedException { - blockSize = 2 * 1024; - compression = LZ4; - compressLevel = LZ4_MIN_LEVEL; + if (freeSpace != 0) { + byte[] lastRow = new byte[freeSpace]; + rnd.nextBytes(lastRow); - doTestBTreePage(LEAF_IO); - } + io.addRowFragment(pageId, pageAddr, lastRow, 777L, pageSize); - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testInnerPageLz4Slow2k() throws IgniteCheckedException { - blockSize = 2 * 1024; - compression = LZ4; - compressLevel = LZ4_MAX_LEVEL; + assertEquals(0, io.getRealFreeSpace(pageAddr)); + } - doTestBTreePage(INNER_IO); - } + // Full data page. + checkCompressDecompress(page, getContents, io.getRealFreeSpace(pageAddr) == 0); - /** - * @throws IgniteCheckedException If failed. - */ - @Test - public void testLeafPageLz4Slow2k() throws IgniteCheckedException { - blockSize = 2 * 1024; - compression = LZ4; - compressLevel = LZ4_MAX_LEVEL; + for (int i = 0; i < itemIds.size(); i += 2) + io.removeRow(pageAddr, itemIds.get(i), pageSize); - doTestBTreePage(LEAF_IO); + // Half-filled data page. + checkCompressDecompress(page, getContents, false); } /** * @throws IgniteCheckedException If failed. */ @Test - public void testInnerPageSnappy2k() throws IgniteCheckedException { - blockSize = 2 * 1024; - compression = SNAPPY; - - doTestBTreePage(INNER_IO); + public void testInnerPage() throws IgniteCheckedException { + doTestBTreePage(innerIo); } /** * @throws IgniteCheckedException If failed. */ @Test - public void testLeafPageSnappy2k() throws IgniteCheckedException { - blockSize = 2 * 1024; - compression = SNAPPY; - - doTestBTreePage(LEAF_IO); + public void testLeafPage() throws IgniteCheckedException { + doTestBTreePage(leafIo); } /** @@ -855,80 +282,6 @@ public class CompressionProcessorTest extends GridCommonAbstractTest { checkCompressDecompress(page, getContents, false); } - /** - * @throws IgniteCheckedException If failed. - */ - private void doTestDataPage() throws IgniteCheckedException { - Random rnd = ThreadLocalRandom.current(); - - final byte[][] rows = new byte[][]{ - new byte[17], new byte[37], new byte[71] - }; - - for (int i = 0; i < rows.length; i++) - rnd.nextBytes(rows[i]); - - ByteBuffer page = allocateDirectBuffer(pageSize); - long pageAddr = bufferAddress(page); - - SimpleDataPageIO io = SimpleDataPageIO.VERSIONS.latest(); - - long pageId = PageIdUtils.pageId(PageIdAllocator.MAX_PARTITION_ID, PageIdAllocator.FLAG_DATA, 171717); - - io.initNewPage(pageAddr, pageId, pageSize, null); - - checkIo(io, page); - - Function<ByteBuffer, List<Bytes>> getContents = (buf) -> { - try { - long addr = bufferAddress(buf); - - return io.forAllItems(addr, (link) -> { - DataPagePayload payload = io.readPayload(addr, PageIdUtils.itemId(link), pageSize); - - return new Bytes(payload.getBytes(addr)); - }); - } - catch (IgniteCheckedException e) { - throw new IgniteException(e); - } - }; - - // Empty data page. - checkCompressDecompress(page, getContents, false); - - GridIntList itemIds = new GridIntList(); - - for (;;) { - byte[] row = rows[rnd.nextInt(rows.length)]; - - if (io.getFreeSpace(pageAddr) < row.length) - break; - - itemIds.add(io.addRow(pageAddr, row, pageSize)); - } - - int freeSpace = io.getFreeSpace(pageAddr); - - if (freeSpace != 0) { - byte[] lastRow = new byte[freeSpace]; - rnd.nextBytes(lastRow); - - io.addRowFragment(pageId, pageAddr, lastRow, 777L, pageSize); - - assertEquals(0, io.getRealFreeSpace(pageAddr)); - } - - // Full data page. - checkCompressDecompress(page, getContents, io.getRealFreeSpace(pageAddr) == 0); - - for (int i = 0; i < itemIds.size(); i += 2) - io.removeRow(pageAddr, itemIds.get(i), pageSize); - - // Half-filled data page. - checkCompressDecompress(page, getContents, false); - } - /** */ private void checkIo(PageIO io, ByteBuffer page) throws IgniteCheckedException { assertSame(io, PageIO.getPageIO(bufferAddress(page))); @@ -1027,13 +380,10 @@ public class CompressionProcessorTest extends GridCommonAbstractTest { /** */ static class TestLeafIO extends BPlusLeafIO<byte[]> { - /** */ - static final TestLeafIO LEAF_IO = new TestLeafIO(); - /** */ - TestLeafIO() { - super(29_501, 1, ITEM_SIZE); + TestLeafIO(int itemSz) { + super(29_501, 1, itemSz); } /** {@inheritDoc} */ @@ -1056,13 +406,10 @@ public class CompressionProcessorTest extends GridCommonAbstractTest { /** */ static class TestInnerIO extends BPlusInnerIO<byte[]> { - /** */ - static TestInnerIO INNER_IO = new TestInnerIO(); - /** */ - TestInnerIO() { - super(29_502, 1, true, ITEM_SIZE); + TestInnerIO(int itemSz) { + super(29_502, 1, true, itemSz); } /** {@inheritDoc} */ @@ -1082,3 +429,4 @@ public class CompressionProcessorTest extends GridCommonAbstractTest { } } } + diff --git a/modules/platforms/cpp/core/src/jni/os/linux/utils.cpp b/modules/platforms/cpp/core/src/jni/os/linux/utils.cpp index dab7849c033..b6f455ad4c2 100644 --- a/modules/platforms/cpp/core/src/jni/os/linux/utils.cpp +++ b/modules/platforms/cpp/core/src/jni/os/linux/utils.cpp @@ -47,7 +47,7 @@ namespace ignite const char* IGNITE_NATIVE_TEST_CLASSPATH = "IGNITE_NATIVE_TEST_CLASSPATH"; /** Excluded modules from test classpath. */ - const char* TEST_EXCLUDED_MODULES[] = { "rest-http" }; + const char* TEST_EXCLUDED_MODULES[] = { "rest-http", "compress" }; /** Key indicating that the thread is attached. */ static pthread_key_t attachKey; diff --git a/modules/platforms/cpp/core/src/jni/os/win/utils.cpp b/modules/platforms/cpp/core/src/jni/os/win/utils.cpp index 40945d0ef6a..330f662aa04 100644 --- a/modules/platforms/cpp/core/src/jni/os/win/utils.cpp +++ b/modules/platforms/cpp/core/src/jni/os/win/utils.cpp @@ -43,7 +43,7 @@ namespace ignite const char* IGNITE_NATIVE_TEST_CLASSPATH = "IGNITE_NATIVE_TEST_CLASSPATH"; /** Excluded modules from test classpath. */ - const char* TEST_EXCLUDED_MODULES[] = { "rest-http" }; + const char* TEST_EXCLUDED_MODULES[] = { "rest-http", "compress" }; AttachHelper::~AttachHelper() {