GEODE-306: support printing of off-heap compressed nodes Before when a chunk of off-heap compressed memory tried to get the type of the data stored in the chunk it needed to decompress the data. Since it did not know what decompressor to use it threw an UnsupportedOperationException. Now for a compressed chunk it returns the DataType as either a "compressed object of size xxx" or "compressed byte[xxx]". It does not do a decompression. Unit test coverage was added.
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/f0f61766 Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/f0f61766 Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/f0f61766 Branch: refs/heads/feature/GEODE-12 Commit: f0f61766d0fa9737369b6910d94d1668e011299b Parents: 141512c Author: Darrel Schneider <dschnei...@pivotal.io> Authored: Tue Sep 1 16:40:43 2015 -0700 Committer: Darrel Schneider <dschnei...@pivotal.io> Committed: Wed Sep 2 09:41:06 2015 -0700 ---------------------------------------------------------------------- .../offheap/SimpleMemoryAllocatorImpl.java | 8 +++- .../offheap/OffHeapValidationJUnitTest.java | 45 +++++++++++++++----- 2 files changed, 41 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/f0f61766/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java index 29319e0..7cf1656 100644 --- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java +++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/offheap/SimpleMemoryAllocatorImpl.java @@ -3341,7 +3341,13 @@ public final class SimpleMemoryAllocatorImpl implements MemoryAllocator, MemoryI } if (!isSerialized()) { // byte array - return "byte[" + ((Chunk)this.block).getDataSize() + "]"; + if (isCompressed()) { + return "compressed byte[" + ((Chunk)this.block).getDataSize() + "]"; + } else { + return "byte[" + ((Chunk)this.block).getDataSize() + "]"; + } + } else if (isCompressed()) { + return "compressed object of size " + ((Chunk)this.block).getDataSize(); } //Object obj = EntryEventImpl.deserialize(((Chunk)this.block).getRawBytes()); byte[] bytes = ((Chunk)this.block).getRawBytes(); http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/f0f61766/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/OffHeapValidationJUnitTest.java ---------------------------------------------------------------------- diff --git a/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/OffHeapValidationJUnitTest.java b/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/OffHeapValidationJUnitTest.java index 7991872..1b40f2a 100755 --- a/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/OffHeapValidationJUnitTest.java +++ b/gemfire-core/src/test/java/com/gemstone/gemfire/internal/offheap/OffHeapValidationJUnitTest.java @@ -5,10 +5,13 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import java.io.IOException; import java.io.Serializable; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -33,9 +36,13 @@ import org.junit.Before; import org.junit.Test; import org.junit.experimental.categories.Category; +import com.gemstone.gemfire.DataSerializer; import com.gemstone.gemfire.cache.CacheFactory; import com.gemstone.gemfire.cache.Region; import com.gemstone.gemfire.cache.RegionShortcut; +import com.gemstone.gemfire.compression.SnappyCompressor; +import com.gemstone.gemfire.internal.HeapDataOutputStream; +import com.gemstone.gemfire.internal.Version; import com.gemstone.gemfire.internal.cache.GemFireCacheImpl; import com.gemstone.gemfire.internal.cache.LocalRegion; import com.gemstone.gemfire.internal.offheap.SimpleMemoryAllocatorImpl.Chunk; @@ -87,7 +94,7 @@ public class OffHeapValidationJUnitTest { } @Test - public void testMemoryInspection() { + public void testMemoryInspection() throws IOException { // validate initial state MemoryAllocator allocator = this.cache.getOffHeapStore(); assertNotNull(allocator); @@ -113,6 +120,7 @@ public class OffHeapValidationJUnitTest { // create off-heap region Region<Object, Object> region = this.cache.createRegionFactory(getRegionShortcut()).setOffHeap(true).create(getRegionName()); + Region<Object, Object> compressedRegion = this.cache.createRegionFactory(getRegionShortcut()).setOffHeap(true).setCompressor(SnappyCompressor.getDefaultInstance()).create(getRegionName()+"Compressed"); // perform some ops List<ExpectedValues> expected = new ArrayList<ExpectedValues>(); @@ -120,8 +128,10 @@ public class OffHeapValidationJUnitTest { // Chunk.OFF_HEAP_HEADER_SIZE + 4 ? putString(region, expected); + putCompressedString(compressedRegion, expected); putDate(region, expected); putByteArray(region, expected); + putCompressedByteArray(compressedRegion, expected); putByteArrayArray(region, expected); putShortArray(region, expected); putStringArray(region, expected); @@ -189,8 +199,11 @@ public class OffHeapValidationJUnitTest { assertTrue(obj instanceof String); assertEquals("this is a string", (String)obj); } - if (values.dataType.contains("[")) { //for (int j = 0; j < ((byte[])values.dataValue).length; j++) { - // TODO + if ((values.dataType.contains("byte [") && values.dataType.lastIndexOf('[') == values.dataType.indexOf('[')) || values.dataType.startsWith("compressed")) { + assertTrue("for dataType=" + values.dataType + " expected " + Arrays.toString((byte[])values.dataValue) + " but was " + Arrays.toString((byte[])block.getDataValue()), + Arrays.equals((byte[])values.dataValue, (byte[])block.getDataValue())); + } else if (values.dataType.contains("[")) { + // TODO: multiple dimension arrays or non-byte arrays } else if (values.dataValue instanceof Collection) { int diff = joint((Collection<?>)values.dataValue, (Collection<?>)block.getDataValue()); assertEquals(i + ":" + values.dataType, 0, diff); @@ -216,14 +229,6 @@ public class OffHeapValidationJUnitTest { } - @Test - public void testCompaction() { - // create fragmented state - // validate fragmented - // perform compaction - // validate freed fragments - } - /** * Returns -1 if c1 is missing an element in c2, 1 if c2 is missing an element * in c1, or 0 is they contain the exact same elements. @@ -289,6 +294,17 @@ public class OffHeapValidationJUnitTest { expected.add(new ExpectedValues(value, value.length()*2, "java.lang.String", -1, getMemoryAddress(region, key), 1, 0, false, true)); } + private void putCompressedString(Region<Object, Object> region, List<ExpectedValues> expected) throws IOException { + String key = "keyString"; + String value = "this is a string"; + region.put(key, value); + HeapDataOutputStream hdos = new HeapDataOutputStream(Version.CURRENT); + DataSerializer.writeObject(value, hdos); + byte[] uncompressedBytes = hdos.toByteArray(); + byte[] expectedValue = SnappyCompressor.getDefaultInstance().compress(uncompressedBytes); + expected.add(new ExpectedValues(expectedValue, 32, "compressed object of size " + expectedValue.length, -1, getMemoryAddress(region, key), 1, 0, true, true)); + } + private void putDate(Region<Object, Object> region, List<ExpectedValues> expected) { String key = "keyDate"; Date value = new Date(); @@ -302,6 +318,13 @@ public class OffHeapValidationJUnitTest { region.put(key, value); expected.add(new ExpectedValues(value, 24, "byte[10]", -1, getMemoryAddress(region, key), 1, 0, false, false)); } + private void putCompressedByteArray(Region<Object, Object> region, List<ExpectedValues> expected) throws IOException { + String key = "keyByteArray"; + byte[] value = new byte[10]; + region.put(key, value); + byte[] expectedValue = SnappyCompressor.getDefaultInstance().compress(value); + expected.add(new ExpectedValues(expectedValue, 24, "compressed byte[" + expectedValue.length + "]", -1, getMemoryAddress(region, key), 1, 0, true, false)); + } private void putByteArrayArray(Region<Object, Object> region, List<ExpectedValues> expected) { String key = "keyByteArrayArray";