HBASE-16691 Optimize KeyOnlyFilter by utilizing KeyOnlyCell.(binlijin)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b0fcca6d Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b0fcca6d Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b0fcca6d Branch: refs/heads/hbase-14439 Commit: b0fcca6d7bfb7a897d51537b7485acb6e2bb4714 Parents: aa4a678 Author: anoopsamjohn <anoopsamj...@gmail.com> Authored: Tue Sep 27 22:48:04 2016 +0530 Committer: anoopsamjohn <anoopsamj...@gmail.com> Committed: Tue Sep 27 22:48:04 2016 +0530 ---------------------------------------------------------------------- .../hadoop/hbase/filter/KeyOnlyFilter.java | 294 ++++++++++++++++++- .../hadoop/hbase/filter/TestKeyOnlyFilter.java | 127 ++++++++ .../apache/hadoop/hbase/filter/TestFilter.java | 8 +- 3 files changed, 410 insertions(+), 19 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/b0fcca6d/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java index 2fd5aba..1a0d2af 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java @@ -20,11 +20,12 @@ package org.apache.hadoop.hbase.filter; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; +import org.apache.hadoop.hbase.ByteBufferedCell; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.exceptions.DeserializationException; @@ -61,19 +62,11 @@ public class KeyOnlyFilter extends FilterBase { } private Cell createKeyOnlyCell(Cell c) { - // KV format: <keylen:4><valuelen:4><key:keylen><value:valuelen> - // Rebuild as: <keylen:4><0:4><key:keylen> - int dataLen = lenAsVal ? Bytes.SIZEOF_INT : 0; - int keyOffset = (2 * Bytes.SIZEOF_INT); - int keyLen = KeyValueUtil.keyLength(c); - byte[] newBuffer = new byte[keyLen + keyOffset + dataLen]; - Bytes.putInt(newBuffer, 0, keyLen); - Bytes.putInt(newBuffer, Bytes.SIZEOF_INT, dataLen); - KeyValueUtil.appendKeyTo(c, newBuffer, keyOffset); - if (lenAsVal) { - Bytes.putInt(newBuffer, newBuffer.length - dataLen, c.getValueLength()); - } - return new KeyValue(newBuffer); + if (c instanceof ByteBufferedCell) { + return new KeyOnlyByteBufferedCell((ByteBufferedCell) c, lenAsVal); + } else { + return new KeyOnlyCell(c, lenAsVal); + } } @Override @@ -130,4 +123,275 @@ public class KeyOnlyFilter extends FilterBase { KeyOnlyFilter other = (KeyOnlyFilter)o; return this.lenAsVal == other.lenAsVal; } + + static class KeyOnlyCell implements Cell { + private Cell cell; + private boolean lenAsVal; + + public KeyOnlyCell(Cell c, boolean lenAsVal) { + this.cell = c; + this.lenAsVal = lenAsVal; + } + + @Override + public byte[] getRowArray() { + return cell.getRowArray(); + } + + @Override + public int getRowOffset() { + return cell.getRowOffset(); + } + + @Override + public short getRowLength() { + return cell.getRowLength(); + } + + @Override + public byte[] getFamilyArray() { + return cell.getFamilyArray(); + } + + @Override + public int getFamilyOffset() { + return cell.getFamilyOffset(); + } + + @Override + public byte getFamilyLength() { + return cell.getFamilyLength(); + } + + @Override + public byte[] getQualifierArray() { + return cell.getQualifierArray(); + } + + @Override + public int getQualifierOffset() { + return cell.getQualifierOffset(); + } + + @Override + public int getQualifierLength() { + return cell.getQualifierLength(); + } + + @Override + public long getTimestamp() { + return cell.getTimestamp(); + } + + @Override + public byte getTypeByte() { + return cell.getTypeByte(); + } + + @Override + public long getSequenceId() { + return 0; + } + + @Override + public byte[] getValueArray() { + if (lenAsVal) { + return Bytes.toBytes(cell.getValueLength()); + } else { + return HConstants.EMPTY_BYTE_ARRAY; + } + } + + @Override + public int getValueOffset() { + return 0; + } + + @Override + public int getValueLength() { + if (lenAsVal) { + return Bytes.SIZEOF_INT; + } else { + return 0; + } + } + + @Override + public byte[] getTagsArray() { + return HConstants.EMPTY_BYTE_ARRAY; + } + + @Override + public int getTagsOffset() { + return 0; + } + + @Override + public int getTagsLength() { + return 0; + } + } + + static class KeyOnlyByteBufferedCell extends ByteBufferedCell { + private ByteBufferedCell cell; + private boolean lenAsVal; + + public KeyOnlyByteBufferedCell(ByteBufferedCell c, boolean lenAsVal) { + this.cell = c; + this.lenAsVal = lenAsVal; + } + + @Override + public byte[] getRowArray() { + return cell.getRowArray(); + } + + @Override + public int getRowOffset() { + return cell.getRowOffset(); + } + + @Override + public short getRowLength() { + return cell.getRowLength(); + } + + @Override + public byte[] getFamilyArray() { + return cell.getFamilyArray(); + } + + @Override + public int getFamilyOffset() { + return cell.getFamilyOffset(); + } + + @Override + public byte getFamilyLength() { + return cell.getFamilyLength(); + } + + @Override + public byte[] getQualifierArray() { + return cell.getQualifierArray(); + } + + @Override + public int getQualifierOffset() { + return cell.getQualifierOffset(); + } + + @Override + public int getQualifierLength() { + return cell.getQualifierLength(); + } + + @Override + public long getTimestamp() { + return cell.getTimestamp(); + } + + @Override + public byte getTypeByte() { + return cell.getTypeByte(); + } + + @Override + public long getSequenceId() { + return 0; + } + + @Override + public byte[] getValueArray() { + if (lenAsVal) { + return Bytes.toBytes(cell.getValueLength()); + } else { + return HConstants.EMPTY_BYTE_ARRAY; + } + } + + @Override + public int getValueOffset() { + return 0; + } + + @Override + public int getValueLength() { + if (lenAsVal) { + return Bytes.SIZEOF_INT; + } else { + return 0; + } + } + + @Override + public byte[] getTagsArray() { + return HConstants.EMPTY_BYTE_ARRAY; + } + + @Override + public int getTagsOffset() { + return 0; + } + + @Override + public int getTagsLength() { + return 0; + } + + @Override + public ByteBuffer getRowByteBuffer() { + return cell.getRowByteBuffer(); + } + + @Override + public int getRowPosition() { + return cell.getRowPosition(); + } + + @Override + public ByteBuffer getFamilyByteBuffer() { + return cell.getFamilyByteBuffer(); + } + + @Override + public int getFamilyPosition() { + return cell.getFamilyPosition(); + } + + @Override + public ByteBuffer getQualifierByteBuffer() { + return cell.getQualifierByteBuffer(); + } + + @Override + public int getQualifierPosition() { + return cell.getQualifierPosition(); + } + + @Override + public ByteBuffer getValueByteBuffer() { + if (lenAsVal) { + return ByteBuffer.wrap(Bytes.toBytes(cell.getValueLength())); + } else { + return HConstants.EMPTY_BYTE_BUFFER; + } + } + + @Override + public int getValuePosition() { + return 0; + } + + @Override + public ByteBuffer getTagsByteBuffer() { + return HConstants.EMPTY_BYTE_BUFFER; + } + + @Override + public int getTagsPosition() { + return 0; + } + } + } http://git-wip-us.apache.org/repos/asf/hbase/blob/b0fcca6d/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestKeyOnlyFilter.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestKeyOnlyFilter.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestKeyOnlyFilter.java new file mode 100644 index 0000000..31f7904 --- /dev/null +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestKeyOnlyFilter.java @@ -0,0 +1,127 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.filter; + +import static org.junit.Assert.assertTrue; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.TestCellUtil.ByteBufferedCellImpl; +import org.apache.hadoop.hbase.filter.KeyOnlyFilter.KeyOnlyByteBufferedCell; +import org.apache.hadoop.hbase.filter.KeyOnlyFilter.KeyOnlyCell; +import org.apache.hadoop.hbase.testclassification.MiscTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; + +@Category({ MiscTests.class, SmallTests.class }) +@RunWith(Parameterized.class) +public class TestKeyOnlyFilter { + + private final boolean lenAsVal; + + @Parameters + public static Collection<Object[]> parameters() { + List<Object[]> paramList = new ArrayList<Object[]>(); + { + paramList.add(new Object[] { false }); + paramList.add(new Object[] { true }); + } + return paramList; + } + + public TestKeyOnlyFilter(boolean lenAsVal) { + this.lenAsVal = lenAsVal; + } + + @Test + public void testKeyOnly() throws Exception { + byte[] r = Bytes.toBytes("row1"); + byte[] f = Bytes.toBytes("cf1"); + byte[] q = Bytes.toBytes("qual1"); + byte[] v = Bytes.toBytes("val1"); + byte[] tags = Bytes.toBytes("tag1"); + KeyValue kv = new KeyValue(r, f, q, 0, q.length, 1234L, Type.Put, v, 0, + v.length, tags); + + ByteBuffer buffer = ByteBuffer.wrap(kv.getBuffer()); + ByteBufferedCellImpl bbCell = new ByteBufferedCellImpl(buffer, 0, + buffer.remaining()); + + // KV format: <keylen:4><valuelen:4><key:keylen><value:valuelen> + // Rebuild as: <keylen:4><0:4><key:keylen> + int dataLen = lenAsVal ? Bytes.SIZEOF_INT : 0; + int keyOffset = (2 * Bytes.SIZEOF_INT); + int keyLen = KeyValueUtil.keyLength(kv); + byte[] newBuffer = new byte[keyLen + keyOffset + dataLen]; + Bytes.putInt(newBuffer, 0, keyLen); + Bytes.putInt(newBuffer, Bytes.SIZEOF_INT, dataLen); + KeyValueUtil.appendKeyTo(kv, newBuffer, keyOffset); + if (lenAsVal) { + Bytes.putInt(newBuffer, newBuffer.length - dataLen, kv.getValueLength()); + } + KeyValue KeyOnlyKeyValue = new KeyValue(newBuffer); + + KeyOnlyCell keyOnlyCell = new KeyOnlyCell(kv, lenAsVal); + KeyOnlyByteBufferedCell keyOnlyByteBufferedCell = new KeyOnlyByteBufferedCell( + bbCell, lenAsVal); + + assertTrue(CellUtil.matchingRows(KeyOnlyKeyValue, keyOnlyCell)); + assertTrue(CellUtil.matchingRows(KeyOnlyKeyValue, keyOnlyByteBufferedCell)); + + assertTrue(CellUtil.matchingFamily(KeyOnlyKeyValue, keyOnlyCell)); + assertTrue(CellUtil + .matchingFamily(KeyOnlyKeyValue, keyOnlyByteBufferedCell)); + + assertTrue(CellUtil.matchingQualifier(KeyOnlyKeyValue, keyOnlyCell)); + assertTrue(CellUtil.matchingQualifier(KeyOnlyKeyValue, + keyOnlyByteBufferedCell)); + + assertTrue(CellUtil.matchingValue(KeyOnlyKeyValue, keyOnlyCell)); + assertTrue(KeyOnlyKeyValue.getValueLength() == keyOnlyByteBufferedCell + .getValueLength()); + if (keyOnlyByteBufferedCell.getValueLength() > 0) { + assertTrue(CellUtil.matchingValue(KeyOnlyKeyValue, + keyOnlyByteBufferedCell)); + } + + assertTrue(KeyOnlyKeyValue.getTimestamp() == keyOnlyCell.getTimestamp()); + assertTrue(KeyOnlyKeyValue.getTimestamp() == keyOnlyByteBufferedCell + .getTimestamp()); + + assertTrue(KeyOnlyKeyValue.getTypeByte() == keyOnlyCell.getTypeByte()); + assertTrue(KeyOnlyKeyValue.getTypeByte() == keyOnlyByteBufferedCell + .getTypeByte()); + + assertTrue(KeyOnlyKeyValue.getTagsLength() == keyOnlyCell.getTagsLength()); + assertTrue(KeyOnlyKeyValue.getTagsLength() == keyOnlyByteBufferedCell + .getTagsLength()); + } + +} http://git-wip-us.apache.org/repos/asf/hbase/blob/b0fcca6d/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java index 4b8da96..6b22164 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/filter/TestFilter.java @@ -1660,7 +1660,7 @@ public class TestFilter { int i = 0; for (boolean done = true; done; i++) { done = scanner.next(results); - Arrays.sort(results.toArray(new KeyValue[results.size()]), + Arrays.sort(results.toArray(new Cell[results.size()]), CellComparator.COMPARATOR); LOG.info("counter=" + i + ", " + results); if (results.isEmpty()) break; @@ -1682,7 +1682,7 @@ public class TestFilter { int i = 0; for (boolean done = true; done; i++) { done = scanner.next(results); - Arrays.sort(results.toArray(new KeyValue[results.size()]), + Arrays.sort(results.toArray(new Cell[results.size()]), CellComparator.COMPARATOR); LOG.info("counter=" + i + ", " + results); if(results.isEmpty()) break; @@ -1704,7 +1704,7 @@ public class TestFilter { int idx = 0; for (boolean done = true; done; row++) { done = scanner.next(results); - Arrays.sort(results.toArray(new KeyValue[results.size()]), + Arrays.sort(results.toArray(new Cell[results.size()]), CellComparator.COMPARATOR); if(results.isEmpty()) break; assertTrue("Scanned too many keys! Only expected " + kvs.length + @@ -1735,7 +1735,7 @@ public class TestFilter { int idx = 0; for (boolean more = true; more; row++) { more = scanner.next(results); - Arrays.sort(results.toArray(new KeyValue[results.size()]), + Arrays.sort(results.toArray(new Cell[results.size()]), CellComparator.COMPARATOR); if(results.isEmpty()) break; assertTrue("Scanned too many keys! Only expected " + kvs.length +
