Repository: hbase Updated Branches: refs/heads/HBASE-7912 b4e0ad6ee -> acd9a7061
HBASE-17199 Back-port HBASE-17151 to HBASE-7912 branch (Vladimir Rodionov) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/acd9a706 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/acd9a706 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/acd9a706 Branch: refs/heads/HBASE-7912 Commit: acd9a7061c9f9ca1c4d67f24b51c30278777c4f4 Parents: b4e0ad6 Author: tedyu <[email protected]> Authored: Mon Dec 5 10:33:16 2016 -0800 Committer: tedyu <[email protected]> Committed: Mon Dec 5 10:33:16 2016 -0800 ---------------------------------------------------------------------- .../hadoop/hbase/io/hfile/CacheConfig.java | 21 ++++- .../org/apache/hadoop/hbase/io/hfile/HFile.java | 40 ++++++++- .../apache/hadoop/hbase/io/hfile/TestHFile.java | 89 +++++++++++++++++--- 3 files changed, 131 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/acd9a706/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index d6bdec0..6c99e08 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -26,10 +26,10 @@ import java.lang.management.MemoryUsage; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; import org.apache.hadoop.hbase.util.ReflectionUtils; @@ -44,6 +44,14 @@ import com.google.common.annotations.VisibleForTesting; public class CacheConfig { private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName()); + + /** + * Disabled cache configuration + */ + + public static final CacheConfig DISABLED = new CacheConfig(); + + /** * Configuration key to cache data blocks on write. There are separate * switches for bloom blocks and non-root index blocks. @@ -85,7 +93,7 @@ public class CacheConfig { * If the chosen ioengine can persist its state across restarts, the path to the file to * persist to. */ - public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY = + public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY = "hbase.bucketcache.persistent.path"; /** @@ -93,11 +101,11 @@ public class CacheConfig { * as indices and blooms are kept in the lru blockcache and the data blocks in the * bucket cache). */ - public static final String BUCKET_CACHE_COMBINED_KEY = + public static final String BUCKET_CACHE_COMBINED_KEY = "hbase.bucketcache.combinedcache.enabled"; public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads"; - public static final String BUCKET_CACHE_WRITER_QUEUE_KEY = + public static final String BUCKET_CACHE_WRITER_QUEUE_KEY = "hbase.bucketcache.writer.queuelength"; /** @@ -304,6 +312,11 @@ public class CacheConfig { cacheConf.cacheDataInL1, cacheConf.dropBehindCompaction); } + private CacheConfig() { + this(null, false, false, false, false, false, + false, false, false, false, false); + } + /** * Checks whether the block cache is enabled. */ http://git-wip-us.apache.org/repos/asf/hbase/blob/acd9a706/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index e9fa05c..660bce4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -249,7 +249,7 @@ public class HFile { protected FileSystem fs; protected Path path; protected FSDataOutputStream ostream; - protected CellComparator comparator = + protected CellComparator comparator = CellComparator.COMPARATOR; protected InetSocketAddress[] favoredNodes; private HFileContext fileContext; @@ -460,9 +460,9 @@ public class HFile { * Return the file context of the HFile this reader belongs to */ HFileContext getFileContext(); - + boolean isPrimaryReplicaReader(); - + void setPrimaryReplicaReader(boolean isPrimaryReplicaReader); boolean shouldIncludeMemstoreTS(); @@ -547,6 +547,20 @@ public class HFile { } /** + * Creates reader with cache configuration disabled + * @param fs filesystem + * @param path Path to file to read + * @param conf the Configuration instance + * @return an active Reader instance + * @throws IOException Will throw a CorruptHFileException + * (DoNotRetryIOException subtype) if hfile is corrupt/invalid. + */ + public static Reader createReader( + FileSystem fs, Path path, Configuration conf) throws IOException { + return createReader(fs, path, CacheConfig.DISABLED, conf); + } + + /** * * @param fs filesystem * @param path Path to file to read @@ -656,82 +670,102 @@ public class HFile { return this; } + @Override public void clear() { this.map.clear(); } + @Override public Comparator<? super byte[]> comparator() { return map.comparator(); } + @Override public boolean containsKey(Object key) { return map.containsKey(key); } + @Override public boolean containsValue(Object value) { return map.containsValue(value); } + @Override public Set<java.util.Map.Entry<byte[], byte[]>> entrySet() { return map.entrySet(); } + @Override public boolean equals(Object o) { return map.equals(o); } + @Override public byte[] firstKey() { return map.firstKey(); } + @Override public byte[] get(Object key) { return map.get(key); } + @Override public int hashCode() { return map.hashCode(); } + @Override public SortedMap<byte[], byte[]> headMap(byte[] toKey) { return this.map.headMap(toKey); } + @Override public boolean isEmpty() { return map.isEmpty(); } + @Override public Set<byte[]> keySet() { return map.keySet(); } + @Override public byte[] lastKey() { return map.lastKey(); } + @Override public byte[] put(byte[] key, byte[] value) { return this.map.put(key, value); } + @Override public void putAll(Map<? extends byte[], ? extends byte[]> m) { this.map.putAll(m); } + @Override public byte[] remove(Object key) { return this.map.remove(key); } + @Override public int size() { return map.size(); } + @Override public SortedMap<byte[], byte[]> subMap(byte[] fromKey, byte[] toKey) { return this.map.subMap(fromKey, toKey); } + @Override public SortedMap<byte[], byte[]> tailMap(byte[] fromKey) { return this.map.tailMap(fromKey); } + @Override public Collection<byte[]> values() { return map.values(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/acd9a706/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java index 66fb49c..a4d0d36 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java @@ -18,12 +18,18 @@ */ package org.apache.hadoop.hbase.io.hfile; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; -import java.util.Map; +import java.util.Random; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -33,6 +39,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.ArrayBackedTag; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; @@ -42,21 +49,21 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.Tag; -import org.apache.hadoop.hbase.ArrayBackedTag; import org.apache.hadoop.hbase.io.compress.Compression; import org.apache.hadoop.hbase.io.hfile.HFile.Reader; import org.apache.hadoop.hbase.io.hfile.HFile.Writer; import org.apache.hadoop.hbase.nio.ByteBuff; +import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.testclassification.IOTests; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.Writable; -import org.junit.*; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; -import static org.junit.Assert.*; - /** * test hfile features. */ @@ -66,14 +73,13 @@ public class TestHFile { @Rule public TestName testName = new TestName(); private static final Log LOG = LogFactory.getLog(TestHFile.class); - + private static final int NUM_VALID_KEY_TYPES = KeyValue.Type.values().length - 2; private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static String ROOT_DIR = TEST_UTIL.getDataTestDir("TestHFile").toString(); private final int minBlockSize = 512; private static String localFormatter = "%010d"; private static CacheConfig cacheConf = null; - private Map<String, Long> startingMetrics; private static Configuration conf ; private static FileSystem fs; @@ -82,7 +88,66 @@ public class TestHFile { conf = TEST_UTIL.getConfiguration(); fs = TEST_UTIL.getTestFileSystem(); } - + + @Test + public void testReaderWithoutBlockCache() throws Exception { + Path path = writeStoreFile(); + try{ + readStoreFile(path); + } catch (Exception e) { + // fail test + assertTrue(false); + } + } + + + private void readStoreFile(Path storeFilePath) throws Exception { + // Open the file reader with block cache disabled + HFile.Reader reader = HFile.createReader(fs, storeFilePath, conf); + long offset = 0; + while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { + HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, null); + offset += block.getOnDiskSizeWithHeader(); + } + } + + private Path writeStoreFile() throws IOException { + Path storeFileParentDir = new Path(TEST_UTIL.getDataTestDir(), "TestHFile"); + HFileContext meta = new HFileContextBuilder().withBlockSize(64 * 1024).build(); + StoreFile.Writer sfw = + new StoreFile.WriterBuilder(conf, cacheConf, fs).withOutputDir(storeFileParentDir) + .withComparator(CellComparator.COMPARATOR).withFileContext(meta).build(); + + final int rowLen = 32; + Random RNG = new Random(); + for (int i = 0; i < 1000; ++i) { + byte[] k = RandomKeyValueUtil.randomOrderedKey(RNG, i); + byte[] v = RandomKeyValueUtil.randomValue(RNG); + int cfLen = RNG.nextInt(k.length - rowLen + 1); + KeyValue kv = + new KeyValue(k, 0, rowLen, k, rowLen, cfLen, k, rowLen + cfLen, + k.length - rowLen - cfLen, RNG.nextLong(), generateKeyType(RNG), v, 0, v.length); + sfw.append(kv); + } + + sfw.close(); + return sfw.getPath(); + } + + public static KeyValue.Type generateKeyType(Random rand) { + if (rand.nextBoolean()) { + // Let's make half of KVs puts. + return KeyValue.Type.Put; + } else { + KeyValue.Type keyType = KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)]; + if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum) { + throw new RuntimeException("Generated an invalid key type: " + keyType + ". " + + "Probably the layout of KeyValue.Type has changed."); + } + return keyType; + } + } + /** * Test empty HFile. * Test all features work reasonably when hfile is empty of entries. @@ -430,7 +495,7 @@ public class TestHFile { mid = HFileWriterImpl.getMidpoint(CellComparator.COMPARATOR, left, right); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(left, mid) < 0); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(mid, right) < 0); - assertEquals(1, (int) mid.getRowLength()); + assertEquals(1, mid.getRowLength()); left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("a")); @@ -443,21 +508,21 @@ public class TestHFile { mid = HFileWriterImpl.getMidpoint(CellComparator.COMPARATOR, left, right); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(left, mid) < 0); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(mid, right) < 0); - assertEquals(2, (int) mid.getFamilyLength()); + assertEquals(2, mid.getFamilyLength()); left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("aaaaaaaaa")); mid = HFileWriterImpl.getMidpoint(CellComparator.COMPARATOR, left, right); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(left, mid) < 0); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(mid, right) < 0); - assertEquals(2, (int) mid.getQualifierLength()); + assertEquals(2, mid.getQualifierLength()); left = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("a")); right = CellUtil.createCell(Bytes.toBytes("a"), Bytes.toBytes("a"), Bytes.toBytes("b")); mid = HFileWriterImpl.getMidpoint(CellComparator.COMPARATOR, left, right); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(left, mid) < 0); assertTrue(CellComparator.COMPARATOR.compareKeyIgnoresMvcc(mid, right) <= 0); - assertEquals(1, (int) mid.getQualifierLength()); + assertEquals(1, mid.getQualifierLength()); // Assert that if meta comparator, it returns the right cell -- i.e. no // optimization done.
