Repository: hbase Updated Branches: refs/heads/master 3b6db2686 -> ccb22bd80
http://git-wip-us.apache.org/repos/asf/hbase/blob/ccb22bd8/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java index 122b7fc..97d5917 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/CacheTestUtils.java @@ -270,7 +270,7 @@ public class CacheTestUtils { } @Override - public Cacheable deserialize(ByteBuff b, boolean reuse) + public Cacheable deserialize(ByteBuff b, boolean reuse, MemoryType memType) throws IOException { return deserialize(b); } @@ -315,6 +315,11 @@ public class CacheTestUtils { public BlockType getBlockType() { return BlockType.DATA; } + + @Override + public MemoryType getMemoryType() { + return MemoryType.EXCLUSIVE; + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/ccb22bd8/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java index 110d92b..959cf4d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.io.hfile.Cacheable.MemoryType; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.nio.MultiByteBuff; @@ -73,7 +74,7 @@ public class TestCacheConfig { } @Override - public Cacheable deserialize(ByteBuff b, boolean reuse) throws IOException { + public Cacheable deserialize(ByteBuff b, boolean reuse, MemoryType memType) throws IOException { LOG.info("Deserialized " + b + ", reuse=" + reuse); return cacheable; } @@ -140,6 +141,11 @@ public class TestCacheConfig { public BlockType getBlockType() { return BlockType.DATA; } + + @Override + public MemoryType getMemoryType() { + return MemoryType.EXCLUSIVE; + } }; static class MetaCacheEntry extends DataCacheEntry { http://git-wip-us.apache.org/repos/asf/hbase/blob/ccb22bd8/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java index 77311bc..5823dfb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java @@ -28,7 +28,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.EnumMap; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Random; import org.apache.commons.logging.Log; @@ -290,6 +293,8 @@ public class TestCacheOnWrite { DataBlockEncoding encodingInCache = encoderType.getEncoder().getDataBlockEncoding(); + List<Long> cachedBlocksOffset = new ArrayList<Long>(); + Map<Long, HFileBlock> cachedBlocks = new HashMap<Long, HFileBlock>(); while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { long onDiskSize = -1; if (prevBlock != null) { @@ -303,6 +308,8 @@ public class TestCacheOnWrite { offset); HFileBlock fromCache = (HFileBlock) blockCache.getBlock(blockCacheKey, true, false, true); boolean isCached = fromCache != null; + cachedBlocksOffset.add(offset); + cachedBlocks.put(offset, fromCache); boolean shouldBeCached = cowType.shouldBeCached(block.getBlockType()); assertTrue("shouldBeCached: " + shouldBeCached+ "\n" + "isCached: " + isCached + "\n" + @@ -355,6 +362,28 @@ public class TestCacheOnWrite { while (scanner.next()) { scanner.getCell(); } + Iterator<Long> iterator = cachedBlocksOffset.iterator(); + while(iterator.hasNext()) { + Long entry = iterator.next(); + BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(), + entry); + HFileBlock hFileBlock = cachedBlocks.get(entry); + if (hFileBlock != null) { + // call return twice because for the isCache cased the counter would have got incremented + // twice + blockCache.returnBlock(blockCacheKey, hFileBlock); + if(cacheCompressedData) { + if (this.compress == Compression.Algorithm.NONE + || cowType == CacheOnWriteType.INDEX_BLOCKS + || cowType == CacheOnWriteType.BLOOM_BLOCKS) { + blockCache.returnBlock(blockCacheKey, hFileBlock); + } + } else { + blockCache.returnBlock(blockCacheKey, hFileBlock); + } + } + } + scanner.shipped(); reader.close(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/ccb22bd8/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java index 600b407..fcb179b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCachedBlockQueue.java @@ -141,6 +141,11 @@ public class TestCachedBlockQueue extends TestCase { return BlockType.DATA; } + @Override + public MemoryType getMemoryType() { + return MemoryType.EXCLUSIVE; + } + }, accessTime, false); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/ccb22bd8/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java index 3258991..1264fa0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java @@ -328,7 +328,7 @@ public class TestHFile extends HBaseTestCase { private void readNumMetablocks(Reader reader, int n) throws IOException { for (int i = 0; i < n; i++) { - ByteBuff actual = reader.getMetaBlock("HFileMeta" + i, false); + ByteBuff actual = reader.getMetaBlock("HFileMeta" + i, false).getBufferWithoutHeader(); ByteBuffer expected = ByteBuffer.wrap(("something to test" + i).getBytes()); assertEquals( http://git-wip-us.apache.org/repos/asf/hbase/blob/ccb22bd8/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java index c6aba43..12fb584 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlock.java @@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.Cacheable.MemoryType; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.nio.MultiByteBuff; import org.apache.hadoop.hbase.nio.SingleByteBuff; @@ -463,8 +464,9 @@ public class TestHFileBlock { for (boolean reuseBuffer : new boolean[] { false, true }) { ByteBuffer serialized = ByteBuffer.allocate(blockFromHFile.getSerializedLength()); blockFromHFile.serialize(serialized); - HFileBlock deserialized = (HFileBlock) blockFromHFile.getDeserializer().deserialize( - new SingleByteBuff(serialized), reuseBuffer); + HFileBlock deserialized = + (HFileBlock) blockFromHFile.getDeserializer().deserialize( + new SingleByteBuff(serialized), reuseBuffer, MemoryType.EXCLUSIVE); assertEquals( "Serialization did not preserve block state. reuseBuffer=" + reuseBuffer, blockFromHFile, deserialized); http://git-wip-us.apache.org/repos/asf/hbase/blob/ccb22bd8/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java index 5d9ac9d..8a843a3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java @@ -169,6 +169,10 @@ public class TestHFileBlockIndex { } @Override + public void returnBlock(HFileBlock block) { + } + + @Override public HFileBlock readBlock(long offset, long onDiskSize, boolean cacheBlock, boolean pread, boolean isCompaction, boolean updateCacheMetrics, BlockType expectedBlockType, http://git-wip-us.apache.org/repos/asf/hbase/blob/ccb22bd8/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java index ec60bcd..cff88d5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java @@ -779,6 +779,11 @@ public class TestLruBlockCache { return BlockType.DATA; } + @Override + public MemoryType getMemoryType() { + return MemoryType.EXCLUSIVE; + } + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/ccb22bd8/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestByteBufferIOEngine.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestByteBufferIOEngine.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestByteBufferIOEngine.java index f68271e..41669fd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestByteBufferIOEngine.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestByteBufferIOEngine.java @@ -22,10 +22,12 @@ import static org.junit.Assert.assertTrue; import java.nio.ByteBuffer; +import org.apache.hadoop.hbase.io.hfile.Cacheable.MemoryType; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.nio.MultiByteBuff; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Pair; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -65,11 +67,10 @@ public class TestByteBufferIOEngine { offset = (int) (Math.random() * (capacity - maxBlockSize)); } ioEngine.write(srcBuffer, offset); - ByteBuffer dstBuffer = ByteBuffer.allocate(blockSize); - ioEngine.read(dstBuffer, offset); - byte[] byteArray2 = dstBuffer.array(); + Pair<ByteBuff, MemoryType> pair = ioEngine.read(offset, blockSize); + ByteBuff dstBuffer = pair.getFirst(); for (int j = 0; j < byteArray.length; ++j) { - assertTrue(byteArray[j] == byteArray2[j]); + assertTrue(byteArray[j] == dstBuffer.get(j)); } } assert testOffsetAtStartNum == 0; @@ -110,9 +111,9 @@ public class TestByteBufferIOEngine { //ioEngine.read(dstBuffer, offset); //MultiByteBuffer read = new MultiByteBuffer(dstBuffer); // TODO : this will get changed after HBASE-12295 goes in - ByteBuff read = ioEngine.read(offset, blockSize); + Pair<ByteBuff, MemoryType> read = ioEngine.read(offset, blockSize); for (int j = 0; j < byteArray.length; ++j) { - assertTrue(srcBuffer.get(j) == read.get(j)); + assertTrue(srcBuffer.get(j) == read.getFirst().get(j)); } } assert testOffsetAtStartNum == 0; http://git-wip-us.apache.org/repos/asf/hbase/blob/ccb22bd8/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java index 8306114..6a290ac 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/bucket/TestFileIOEngine.java @@ -24,8 +24,11 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import org.apache.hadoop.hbase.io.hfile.Cacheable.MemoryType; +import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Pair; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -47,9 +50,9 @@ public class TestFileIOEngine { for (int j = 0; j < data1.length; ++j) { data1[j] = (byte) (Math.random() * 255); } - byte[] data2 = new byte[len]; fileIOEngine.write(ByteBuffer.wrap(data1), offset); - fileIOEngine.read(ByteBuffer.wrap(data2), offset); + Pair<ByteBuff, MemoryType> pair = fileIOEngine.read(offset, len); + byte[] data2 = pair.getFirst().array(); for (int j = 0; j < data1.length; ++j) { assertTrue(data1[j] == data2[j]); } http://git-wip-us.apache.org/repos/asf/hbase/blob/ccb22bd8/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java index fbda6b8..5a92846 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java @@ -39,6 +39,7 @@ import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; import org.apache.hadoop.hbase.io.hfile.CacheStats; import org.apache.hadoop.hbase.io.hfile.Cacheable; import org.apache.hadoop.hbase.io.hfile.CachedBlock; +import org.apache.hadoop.hbase.io.hfile.HFileBlock; import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache; import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil; import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext; @@ -578,9 +579,13 @@ public class TestHeapMemoryManager { return null; } - public void setTestBlockSize(long testBlockSize) { - this.testBlockSize = testBlockSize; - } + @Override + public void returnBlock(BlockCacheKey cacheKey, Cacheable buf) { + } + + public void setTestBlockSize(long testBlockSize) { + this.testBlockSize = testBlockSize; + } } private static class MemstoreFlusherStub implements FlushRequester { http://git-wip-us.apache.org/repos/asf/hbase/blob/ccb22bd8/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java index a3c6685..14c71d2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestScannerHeartbeatMessages.java @@ -425,14 +425,14 @@ public class TestScannerHeartbeatMessages { // Instantiate the custom heartbeat region scanners @Override protected RegionScanner instantiateRegionScanner(Scan scan, - List<KeyValueScanner> additionalScanners) throws IOException { + List<KeyValueScanner> additionalScanners, boolean copyCells) throws IOException { if (scan.isReversed()) { if (scan.getFilter() != null) { scan.getFilter().setReversed(true); } - return new HeartbeatReversedRegionScanner(scan, additionalScanners, this); + return new HeartbeatReversedRegionScanner(scan, additionalScanners, this, copyCells); } - return new HeartbeatRegionScanner(scan, additionalScanners, this); + return new HeartbeatRegionScanner(scan, additionalScanners, this, copyCells); } } @@ -442,8 +442,8 @@ public class TestScannerHeartbeatMessages { */ private static class HeartbeatReversedRegionScanner extends ReversedRegionScannerImpl { HeartbeatReversedRegionScanner(Scan scan, List<KeyValueScanner> additionalScanners, - HRegion region) throws IOException { - super(scan, additionalScanners, region); + HRegion region, boolean copyCells) throws IOException { + super(scan, additionalScanners, region, copyCells); } @Override @@ -469,9 +469,9 @@ public class TestScannerHeartbeatMessages { * column family cells */ private static class HeartbeatRegionScanner extends RegionScannerImpl { - HeartbeatRegionScanner(Scan scan, List<KeyValueScanner> additionalScanners, HRegion region) - throws IOException { - region.super(scan, additionalScanners, region); + HeartbeatRegionScanner(Scan scan, List<KeyValueScanner> additionalScanners, HRegion region, + boolean copyCells) throws IOException { + region.super(scan, additionalScanners, region, copyCells); } @Override
