Repository: hbase Updated Branches: refs/heads/branch-1 6a1a97422 -> deeda60c6
HBASE-17617 Backport HBASE-16731 (Inconsistent results from the Get/Scan if we use the empty FilterList) to branch-1 Signed-off-by: Chia-Ping Tsai <chia7...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/deeda60c Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/deeda60c Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/deeda60c Branch: refs/heads/branch-1 Commit: deeda60c69f564e3a54d85380deace5aebfd6d94 Parents: 6a1a974 Author: Pankaj Kumar <pankaj...@huawei.com> Authored: Thu Aug 17 15:58:17 2017 +0800 Committer: Chia-Ping Tsai <chia7...@gmail.com> Committed: Thu Aug 17 18:31:57 2017 +0800 ---------------------------------------------------------------------- .../org/apache/hadoop/hbase/client/Get.java | 5 + .../org/apache/hadoop/hbase/client/Query.java | 35 ++++ .../org/apache/hadoop/hbase/client/Scan.java | 36 +--- .../hadoop/hbase/protobuf/ProtobufUtil.java | 9 +- .../apache/hadoop/hbase/client/TestScan.java | 32 ++++ .../hbase/protobuf/generated/ClientProtos.java | 185 +++++++++++++++---- hbase-protocol/src/main/protobuf/Client.proto | 1 + .../hadoop/hbase/regionserver/HRegion.java | 3 + .../hadoop/hbase/client/TestFromClientSide.java | 44 +++++ .../hadoop/hbase/protobuf/TestProtobufUtil.java | 1 + 10 files changed, 285 insertions(+), 66 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java index 3286d57..f8c34bd 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java @@ -110,6 +110,7 @@ public class Get extends Query this.storeOffset = get.getRowOffsetPerColumnFamily(); this.tr = get.getTimeRange(); this.checkExistenceOnly = get.isCheckExistenceOnly(); + this.loadColumnFamiliesOnDemand = get.getLoadColumnFamiliesOnDemandValue(); this.closestRowBefore = get.isClosestRowBefore(); Map<byte[], NavigableSet<byte[]>> fams = get.getFamilyMap(); for (Map.Entry<byte[],NavigableSet<byte[]>> entry : fams.entrySet()) { @@ -242,6 +243,10 @@ public class Get extends Query return this; } + public Get setLoadColumnFamiliesOnDemand(boolean value) { + return (Get) super.setLoadColumnFamiliesOnDemand(value); + } + /** * Set the maximum number of values to return per row per Column Family * @param limit the maximum number of values returned / row / CF http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java index 99d5a6a..7d3b628 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java @@ -42,6 +42,7 @@ public abstract class Query extends OperationWithAttributes { protected int targetReplicaId = -1; protected Consistency consistency = Consistency.STRONG; protected Map<byte[], TimeRange> colFamTimeRangeMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); + protected Boolean loadColumnFamiliesOnDemand = null; /** * @return Filter @@ -177,6 +178,40 @@ public abstract class Query extends OperationWithAttributes { IsolationLevel.fromBytes(attr); } + /** + * Set the value indicating whether loading CFs on demand should be allowed (cluster + * default is false). On-demand CF loading doesn't load column families until necessary, e.g. + * if you filter on one column, the other column family data will be loaded only for the rows + * that are included in result, not all rows like in normal case. + * With column-specific filters, like SingleColumnValueFilter w/filterIfMissing == true, + * this can deliver huge perf gains when there's a cf with lots of data; however, it can + * also lead to some inconsistent results, as follows: + * - if someone does a concurrent update to both column families in question you may get a row + * that never existed, e.g. for { rowKey = 5, { cat_videos => 1 }, { video => "my cat" } } + * someone puts rowKey 5 with { cat_videos => 0 }, { video => "my dog" }, concurrent scan + * filtering on "cat_videos == 1" can get { rowKey = 5, { cat_videos => 1 }, + * { video => "my dog" } }. + * - if there's a concurrent split and you have more than 2 column families, some rows may be + * missing some column families. + */ + public Query setLoadColumnFamiliesOnDemand(boolean value) { + this.loadColumnFamiliesOnDemand = value; + return this; + } + + /** + * Get the raw loadColumnFamiliesOnDemand setting; if it's not set, can be null. + */ + public Boolean getLoadColumnFamiliesOnDemandValue() { + return this.loadColumnFamiliesOnDemand; + } + + /** + * Get the logical value indicating whether on-demand CF loading should be allowed. + */ + public boolean doLoadColumnFamiliesOnDemand() { + return (this.loadColumnFamiliesOnDemand != null) && this.loadColumnFamiliesOnDemand; + } /** * Get versions of columns only within the specified timestamp range, http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java index 1fbf23e..1340602 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java @@ -148,7 +148,6 @@ public class Scan extends Query { private TimeRange tr = new TimeRange(); private Map<byte [], NavigableSet<byte []>> familyMap = new TreeMap<byte [], NavigableSet<byte []>>(Bytes.BYTES_COMPARATOR); - private Boolean loadColumnFamiliesOnDemand = null; /** * Set it true for small scan to get better performance @@ -300,6 +299,7 @@ public class Scan extends Query { this.getScan = true; this.consistency = get.getConsistency(); this.setIsolationLevel(get.getIsolationLevel()); + this.loadColumnFamiliesOnDemand = get.getLoadColumnFamiliesOnDemandValue(); for (Map.Entry<String, byte[]> attr : get.getAttributesMap().entrySet()) { setAttribute(attr.getKey(), attr.getValue()); } @@ -871,40 +871,8 @@ public class Scan extends Query { return allowPartialResults; } - /** - * Set the value indicating whether loading CFs on demand should be allowed (cluster - * default is false). On-demand CF loading doesn't load column families until necessary, e.g. - * if you filter on one column, the other column family data will be loaded only for the rows - * that are included in result, not all rows like in normal case. - * With column-specific filters, like SingleColumnValueFilter w/filterIfMissing == true, - * this can deliver huge perf gains when there's a cf with lots of data; however, it can - * also lead to some inconsistent results, as follows: - * - if someone does a concurrent update to both column families in question you may get a row - * that never existed, e.g. for { rowKey = 5, { cat_videos => 1 }, { video => "my cat" } } - * someone puts rowKey 5 with { cat_videos => 0 }, { video => "my dog" }, concurrent scan - * filtering on "cat_videos == 1" can get { rowKey = 5, { cat_videos => 1 }, - * { video => "my dog" } }. - * - if there's a concurrent split and you have more than 2 column families, some rows may be - * missing some column families. - */ public Scan setLoadColumnFamiliesOnDemand(boolean value) { - this.loadColumnFamiliesOnDemand = value; - return this; - } - - /** - * Get the raw loadColumnFamiliesOnDemand setting; if it's not set, can be null. - */ - public Boolean getLoadColumnFamiliesOnDemandValue() { - return this.loadColumnFamiliesOnDemand; - } - - /** - * Get the logical value indicating whether on-demand CF loading should be allowed. - */ - public boolean doLoadColumnFamiliesOnDemand() { - return (this.loadColumnFamiliesOnDemand != null) - && this.loadColumnFamiliesOnDemand.booleanValue(); + return (Scan) super.setLoadColumnFamiliesOnDemand(value); } /** http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java index fa44cdd..f0715da 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java @@ -532,6 +532,9 @@ public final class ProtobufUtil { if (proto.hasConsistency()) { get.setConsistency(toConsistency(proto.getConsistency())); } + if (proto.hasLoadColumnFamiliesOnDemand()) { + get.setLoadColumnFamiliesOnDemand(proto.getLoadColumnFamiliesOnDemand()); + } return get; } @@ -980,7 +983,7 @@ public final class ProtobufUtil { } Boolean loadColumnFamiliesOnDemand = scan.getLoadColumnFamiliesOnDemandValue(); if (loadColumnFamiliesOnDemand != null) { - scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand.booleanValue()); + scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand); } scanBuilder.setMaxVersions(scan.getMaxVersions()); for (Entry<byte[], TimeRange> cftr : scan.getColumnFamilyTimeRange().entrySet()) { @@ -1264,6 +1267,10 @@ public final class ProtobufUtil { if (get.getConsistency() != null && get.getConsistency() != Consistency.STRONG) { builder.setConsistency(toConsistency(get.getConsistency())); } + Boolean loadColumnFamiliesOnDemand = get.getLoadColumnFamiliesOnDemandValue(); + if (loadColumnFamiliesOnDemand != null) { + builder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand); + } return builder.build(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java index 5f771cd..77d624d 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hbase.client; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.io.IOException; @@ -26,6 +28,7 @@ import java.util.Arrays; import java.util.Set; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.filter.FilterList; import org.apache.hadoop.hbase.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.security.visibility.Authorizations; @@ -57,6 +60,35 @@ public class TestScan { } @Test + public void testGetToScan() throws IOException { + Get get = new Get(Bytes.toBytes(1)); + get.setCacheBlocks(true).setConsistency(Consistency.TIMELINE).setFilter(new FilterList()) + .setId("get").setIsolationLevel(IsolationLevel.READ_COMMITTED) + .setLoadColumnFamiliesOnDemand(false).setMaxResultsPerColumnFamily(1000) + .setMaxVersions(9999).setRowOffsetPerColumnFamily(5).setTimeRange(0, 13) + .setAttribute("att_v0", Bytes.toBytes("att_v0")) + .setColumnFamilyTimeRange(Bytes.toBytes("cf"), 0, 123); + Scan scan = new Scan(get); + assertEquals(get.getCacheBlocks(), scan.getCacheBlocks()); + assertEquals(get.getConsistency(), scan.getConsistency()); + assertEquals(get.getFilter(), scan.getFilter()); + assertEquals(get.getId(), scan.getId()); + assertEquals(get.getIsolationLevel(), scan.getIsolationLevel()); + assertEquals(get.getLoadColumnFamiliesOnDemandValue(), + scan.getLoadColumnFamiliesOnDemandValue()); + assertEquals(get.getMaxResultsPerColumnFamily(), scan.getMaxResultsPerColumnFamily()); + assertEquals(get.getMaxVersions(), scan.getMaxVersions()); + assertEquals(get.getRowOffsetPerColumnFamily(), scan.getRowOffsetPerColumnFamily()); + assertEquals(get.getTimeRange().getMin(), scan.getTimeRange().getMin()); + assertEquals(get.getTimeRange().getMax(), scan.getTimeRange().getMax()); + assertTrue(Bytes.equals(get.getAttribute("att_v0"), scan.getAttribute("att_v0"))); + assertEquals(get.getColumnFamilyTimeRange().get(Bytes.toBytes("cf")).getMin(), + scan.getColumnFamilyTimeRange().get(Bytes.toBytes("cf")).getMin()); + assertEquals(get.getColumnFamilyTimeRange().get(Bytes.toBytes("cf")).getMax(), + scan.getColumnFamilyTimeRange().get(Bytes.toBytes("cf")).getMax()); + } + + @Test public void testScanAttributes() { Scan scan = new Scan(); Assert.assertTrue(scan.getAttributesMap().isEmpty()); http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java ---------------------------------------------------------------------- diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java index 4bd3f5f..06ebc65 100644 --- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java @@ -1980,6 +1980,24 @@ public final class ClientProtos { */ org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ColumnFamilyTimeRangeOrBuilder getCfTimeRangeOrBuilder( int index); + + // optional bool load_column_families_on_demand = 14; + /** + * <code>optional bool load_column_families_on_demand = 14;</code> + * + * <pre> + * DO NOT add defaults to load_column_families_on_demand. + * </pre> + */ + boolean hasLoadColumnFamiliesOnDemand(); + /** + * <code>optional bool load_column_families_on_demand = 14;</code> + * + * <pre> + * DO NOT add defaults to load_column_families_on_demand. + * </pre> + */ + boolean getLoadColumnFamiliesOnDemand(); } /** * Protobuf type {@code hbase.pb.Get} @@ -2136,6 +2154,11 @@ public final class ClientProtos { cfTimeRange_.add(input.readMessage(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ColumnFamilyTimeRange.PARSER, extensionRegistry)); break; } + case 112: { + bitField0_ |= 0x00000400; + loadColumnFamiliesOnDemand_ = input.readBool(); + break; + } } } } catch (com.google.protobuf.InvalidProtocolBufferException e) { @@ -2485,6 +2508,30 @@ public final class ClientProtos { return cfTimeRange_.get(index); } + // optional bool load_column_families_on_demand = 14; + public static final int LOAD_COLUMN_FAMILIES_ON_DEMAND_FIELD_NUMBER = 14; + private boolean loadColumnFamiliesOnDemand_; + /** + * <code>optional bool load_column_families_on_demand = 14;</code> + * + * <pre> + * DO NOT add defaults to load_column_families_on_demand. + * </pre> + */ + public boolean hasLoadColumnFamiliesOnDemand() { + return ((bitField0_ & 0x00000400) == 0x00000400); + } + /** + * <code>optional bool load_column_families_on_demand = 14;</code> + * + * <pre> + * DO NOT add defaults to load_column_families_on_demand. + * </pre> + */ + public boolean getLoadColumnFamiliesOnDemand() { + return loadColumnFamiliesOnDemand_; + } + private void initFields() { row_ = com.google.protobuf.ByteString.EMPTY; column_ = java.util.Collections.emptyList(); @@ -2499,6 +2546,7 @@ public final class ClientProtos { closestRowBefore_ = false; consistency_ = org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Consistency.STRONG; cfTimeRange_ = java.util.Collections.emptyList(); + loadColumnFamiliesOnDemand_ = false; } private byte memoizedIsInitialized = -1; public final boolean isInitialized() { @@ -2579,6 +2627,9 @@ public final class ClientProtos { for (int i = 0; i < cfTimeRange_.size(); i++) { output.writeMessage(13, cfTimeRange_.get(i)); } + if (((bitField0_ & 0x00000400) == 0x00000400)) { + output.writeBool(14, loadColumnFamiliesOnDemand_); + } getUnknownFields().writeTo(output); } @@ -2640,6 +2691,10 @@ public final class ClientProtos { size += com.google.protobuf.CodedOutputStream .computeMessageSize(13, cfTimeRange_.get(i)); } + if (((bitField0_ & 0x00000400) == 0x00000400)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(14, loadColumnFamiliesOnDemand_); + } size += getUnknownFields().getSerializedSize(); memoizedSerializedSize = size; return size; @@ -2719,6 +2774,11 @@ public final class ClientProtos { } result = result && getCfTimeRangeList() .equals(other.getCfTimeRangeList()); + result = result && (hasLoadColumnFamiliesOnDemand() == other.hasLoadColumnFamiliesOnDemand()); + if (hasLoadColumnFamiliesOnDemand()) { + result = result && (getLoadColumnFamiliesOnDemand() + == other.getLoadColumnFamiliesOnDemand()); + } result = result && getUnknownFields().equals(other.getUnknownFields()); return result; @@ -2784,6 +2844,10 @@ public final class ClientProtos { hash = (37 * hash) + CF_TIME_RANGE_FIELD_NUMBER; hash = (53 * hash) + getCfTimeRangeList().hashCode(); } + if (hasLoadColumnFamiliesOnDemand()) { + hash = (37 * hash) + LOAD_COLUMN_FAMILIES_ON_DEMAND_FIELD_NUMBER; + hash = (53 * hash) + hashBoolean(getLoadColumnFamiliesOnDemand()); + } hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -2952,6 +3016,8 @@ public final class ClientProtos { } else { cfTimeRangeBuilder_.clear(); } + loadColumnFamiliesOnDemand_ = false; + bitField0_ = (bitField0_ & ~0x00002000); return this; } @@ -3055,6 +3121,10 @@ public final class ClientProtos { } else { result.cfTimeRange_ = cfTimeRangeBuilder_.build(); } + if (((from_bitField0_ & 0x00002000) == 0x00002000)) { + to_bitField0_ |= 0x00000400; + } + result.loadColumnFamiliesOnDemand_ = loadColumnFamiliesOnDemand_; result.bitField0_ = to_bitField0_; onBuilt(); return result; @@ -3179,6 +3249,9 @@ public final class ClientProtos { } } } + if (other.hasLoadColumnFamiliesOnDemand()) { + setLoadColumnFamiliesOnDemand(other.getLoadColumnFamiliesOnDemand()); + } this.mergeUnknownFields(other.getUnknownFields()); return this; } @@ -4498,6 +4571,55 @@ public final class ClientProtos { return cfTimeRangeBuilder_; } + // optional bool load_column_families_on_demand = 14; + private boolean loadColumnFamiliesOnDemand_ ; + /** + * <code>optional bool load_column_families_on_demand = 14;</code> + * + * <pre> + * DO NOT add defaults to load_column_families_on_demand. + * </pre> + */ + public boolean hasLoadColumnFamiliesOnDemand() { + return ((bitField0_ & 0x00002000) == 0x00002000); + } + /** + * <code>optional bool load_column_families_on_demand = 14;</code> + * + * <pre> + * DO NOT add defaults to load_column_families_on_demand. + * </pre> + */ + public boolean getLoadColumnFamiliesOnDemand() { + return loadColumnFamiliesOnDemand_; + } + /** + * <code>optional bool load_column_families_on_demand = 14;</code> + * + * <pre> + * DO NOT add defaults to load_column_families_on_demand. + * </pre> + */ + public Builder setLoadColumnFamiliesOnDemand(boolean value) { + bitField0_ |= 0x00002000; + loadColumnFamiliesOnDemand_ = value; + onChanged(); + return this; + } + /** + * <code>optional bool load_column_families_on_demand = 14;</code> + * + * <pre> + * DO NOT add defaults to load_column_families_on_demand. + * </pre> + */ + public Builder clearLoadColumnFamiliesOnDemand() { + bitField0_ = (bitField0_ & ~0x00002000); + loadColumnFamiliesOnDemand_ = false; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:hbase.pb.Get) } @@ -37168,7 +37290,7 @@ public final class ClientProtos { "o\032\017MapReduce.proto\"\037\n\016Authorizations\022\r\n\005" + "label\030\001 \003(\t\"$\n\016CellVisibility\022\022\n\nexpress" + "ion\030\001 \002(\t\"+\n\006Column\022\016\n\006family\030\001 \002(\014\022\021\n\tq" + - "ualifier\030\002 \003(\014\"\271\003\n\003Get\022\013\n\003row\030\001 \002(\014\022 \n\006c" + + "ualifier\030\002 \003(\014\"\341\003\n\003Get\022\013\n\003row\030\001 \002(\014\022 \n\006c" + "olumn\030\002 \003(\0132\020.hbase.pb.Column\022*\n\tattribu" + "te\030\003 \003(\0132\027.hbase.pb.NameBytesPair\022 \n\006fil" + "ter\030\004 \001(\0132\020.hbase.pb.Filter\022\'\n\ntime_rang" + @@ -37179,10 +37301,11 @@ public final class ClientProtos { "losest_row_before\030\013 \001(\010:\005false\0222\n\013consis" + "tency\030\014 \001(\0162\025.hbase.pb.Consistency:\006STRO" + "NG\0226\n\rcf_time_range\030\r \003(\0132\037.hbase.pb.Col" + - "umnFamilyTimeRange\"\203\001\n\006Result\022\034\n\004cell\030\001 " + + "umnFamilyTimeRange\022&\n\036load_column_famili" + + "es_on_demand\030\016 \001(\010\"\203\001\n\006Result\022\034\n\004cell\030\001 " + "\003(\0132\016.hbase.pb.Cell\022\035\n\025associated_cell_c" + - "ount\030\002 \001(\005\022\016\n\006exists\030\003 \001(\010\022\024\n\005stale\030\004 \001(" + - "\010:\005false\022\026\n\007partial\030\005 \001(\010:\005false\"S\n\nGetR", + "ount\030\002 \001(\005\022\016\n\006exists\030\003 \001(\010\022\024\n\005stale\030\004 \001(", + "\010:\005false\022\026\n\007partial\030\005 \001(\010:\005false\"S\n\nGetR" + "equest\022)\n\006region\030\001 \002(\0132\031.hbase.pb.Region" + "Specifier\022\032\n\003get\030\002 \002(\0132\r.hbase.pb.Get\"/\n" + "\013GetResponse\022 \n\006result\030\001 \001(\0132\020.hbase.pb." + @@ -37191,8 +37314,8 @@ public final class ClientProtos { "type\030\004 \002(\0162\025.hbase.pb.CompareType\022(\n\ncom" + "parator\030\005 \002(\0132\024.hbase.pb.Comparator\"\364\006\n\r" + "MutationProto\022\013\n\003row\030\001 \001(\014\0229\n\013mutate_typ" + - "e\030\002 \001(\0162$.hbase.pb.MutationProto.Mutatio" + - "nType\0229\n\014column_value\030\003 \003(\0132#.hbase.pb.M", + "e\030\002 \001(\0162$.hbase.pb.MutationProto.Mutatio", + "nType\0229\n\014column_value\030\003 \003(\0132#.hbase.pb.M" + "utationProto.ColumnValue\022\021\n\ttimestamp\030\004 " + "\001(\004\022*\n\tattribute\030\005 \003(\0132\027.hbase.pb.NameBy" + "tesPair\022C\n\ndurability\030\006 \001(\0162\".hbase.pb.M" + @@ -37201,8 +37324,8 @@ public final class ClientProtos { "\025associated_cell_count\030\010 \001(\005\022\r\n\005nonce\030\t " + "\001(\004\032\371\001\n\013ColumnValue\022\016\n\006family\030\001 \002(\014\022K\n\017q" + "ualifier_value\030\002 \003(\01322.hbase.pb.Mutation" + - "Proto.ColumnValue.QualifierValue\032\214\001\n\016Qua" + - "lifierValue\022\021\n\tqualifier\030\001 \001(\014\022\r\n\005value\030", + "Proto.ColumnValue.QualifierValue\032\214\001\n\016Qua", + "lifierValue\022\021\n\tqualifier\030\001 \001(\014\022\r\n\005value\030" + "\002 \001(\014\022\021\n\ttimestamp\030\003 \001(\004\0227\n\013delete_type\030" + "\004 \001(\0162\".hbase.pb.MutationProto.DeleteTyp" + "e\022\014\n\004tags\030\005 \001(\014\"W\n\nDurability\022\017\n\013USE_DEF" + @@ -37211,8 +37334,8 @@ public final class ClientProtos { "\022\n\n\006APPEND\020\000\022\r\n\tINCREMENT\020\001\022\007\n\003PUT\020\002\022\n\n\006" + "DELETE\020\003\"p\n\nDeleteType\022\026\n\022DELETE_ONE_VER" + "SION\020\000\022\034\n\030DELETE_MULTIPLE_VERSIONS\020\001\022\021\n\r" + - "DELETE_FAMILY\020\002\022\031\n\025DELETE_FAMILY_VERSION" + - "\020\003\"\242\001\n\rMutateRequest\022)\n\006region\030\001 \002(\0132\031.h", + "DELETE_FAMILY\020\002\022\031\n\025DELETE_FAMILY_VERSION", + "\020\003\"\242\001\n\rMutateRequest\022)\n\006region\030\001 \002(\0132\031.h" + "base.pb.RegionSpecifier\022)\n\010mutation\030\002 \002(" + "\0132\027.hbase.pb.MutationProto\022&\n\tcondition\030" + "\003 \001(\0132\023.hbase.pb.Condition\022\023\n\013nonce_grou" + @@ -37221,8 +37344,8 @@ public final class ClientProtos { "\006\n\004Scan\022 \n\006column\030\001 \003(\0132\020.hbase.pb.Colum" + "n\022*\n\tattribute\030\002 \003(\0132\027.hbase.pb.NameByte" + "sPair\022\021\n\tstart_row\030\003 \001(\014\022\020\n\010stop_row\030\004 \001" + - "(\014\022 \n\006filter\030\005 \001(\0132\020.hbase.pb.Filter\022\'\n\n" + - "time_range\030\006 \001(\0132\023.hbase.pb.TimeRange\022\027\n", + "(\014\022 \n\006filter\030\005 \001(\0132\020.hbase.pb.Filter\022\'\n\n", + "time_range\030\006 \001(\0132\023.hbase.pb.TimeRange\022\027\n" + "\014max_versions\030\007 \001(\r:\0011\022\032\n\014cache_blocks\030\010" + " \001(\010:\004true\022\022\n\nbatch_size\030\t \001(\r\022\027\n\017max_re" + "sult_size\030\n \001(\004\022\023\n\013store_limit\030\013 \001(\r\022\024\n\014" + @@ -37231,8 +37354,8 @@ public final class ClientProtos { "ersed\030\017 \001(\010:\005false\0222\n\013consistency\030\020 \001(\0162" + "\025.hbase.pb.Consistency:\006STRONG\022\017\n\007cachin" + "g\030\021 \001(\r\022\035\n\025allow_partial_results\030\022 \001(\010\0226" + - "\n\rcf_time_range\030\023 \003(\0132\037.hbase.pb.ColumnF" + - "amilyTimeRange\022\032\n\017mvcc_read_point\030\024 \001(\004:", + "\n\rcf_time_range\030\023 \003(\0132\037.hbase.pb.ColumnF", + "amilyTimeRange\022\032\n\017mvcc_read_point\030\024 \001(\004:" + "\0010\022\037\n\021include_start_row\030\025 \001(\010:\004true\022\037\n\020i" + "nclude_stop_row\030\026 \001(\010:\005false\0222\n\010readType" + "\030\027 \001(\0162\027.hbase.pb.Scan.ReadType:\007DEFAULT" + @@ -37241,8 +37364,8 @@ public final class ClientProtos { "D\020\002\"\300\002\n\013ScanRequest\022)\n\006region\030\001 \001(\0132\031.hb" + "ase.pb.RegionSpecifier\022\034\n\004scan\030\002 \001(\0132\016.h" + "base.pb.Scan\022\022\n\nscanner_id\030\003 \001(\004\022\026\n\016numb" + - "er_of_rows\030\004 \001(\r\022\025\n\rclose_scanner\030\005 \001(\010\022" + - "\025\n\rnext_call_seq\030\006 \001(\004\022\037\n\027client_handles", + "er_of_rows\030\004 \001(\r\022\025\n\rclose_scanner\030\005 \001(\010\022", + "\025\n\rnext_call_seq\030\006 \001(\004\022\037\n\027client_handles" + "_partials\030\007 \001(\010\022!\n\031client_handles_heartb" + "eats\030\010 \001(\010\022\032\n\022track_scan_metrics\030\t \001(\010\022\024" + "\n\005renew\030\n \001(\010:\005false\022\030\n\rlimit_of_rows\030\013 " + @@ -37251,8 +37374,8 @@ public final class ClientProtos { "ner_id\030\002 \001(\004\022\024\n\014more_results\030\003 \001(\010\022\013\n\003tt" + "l\030\004 \001(\r\022!\n\007results\030\005 \003(\0132\020.hbase.pb.Resu" + "lt\022\r\n\005stale\030\006 \001(\010\022\037\n\027partial_flag_per_re" + - "sult\030\007 \003(\010\022\036\n\026more_results_in_region\030\010 \001" + - "(\010\022\031\n\021heartbeat_message\030\t \001(\010\022+\n\014scan_me", + "sult\030\007 \003(\010\022\036\n\026more_results_in_region\030\010 \001", + "(\010\022\031\n\021heartbeat_message\030\t \001(\010\022+\n\014scan_me" + "trics\030\n \001(\0132\025.hbase.pb.ScanMetrics\022\032\n\017mv" + "cc_read_point\030\013 \001(\004:\0010\022 \n\006cursor\030\014 \001(\0132\020" + ".hbase.pb.Cursor\"\305\001\n\024BulkLoadHFileReques" + @@ -37261,8 +37384,8 @@ public final class ClientProtos { "kLoadHFileRequest.FamilyPath\022\026\n\016assign_s" + "eq_num\030\003 \001(\010\032*\n\nFamilyPath\022\016\n\006family\030\001 \002" + "(\014\022\014\n\004path\030\002 \002(\t\"\'\n\025BulkLoadHFileRespons" + - "e\022\016\n\006loaded\030\001 \002(\010\"a\n\026CoprocessorServiceC" + - "all\022\013\n\003row\030\001 \002(\014\022\024\n\014service_name\030\002 \002(\t\022\023", + "e\022\016\n\006loaded\030\001 \002(\010\"a\n\026CoprocessorServiceC", + "all\022\013\n\003row\030\001 \002(\014\022\024\n\014service_name\030\002 \002(\t\022\023" + "\n\013method_name\030\003 \002(\t\022\017\n\007request\030\004 \002(\014\"B\n\030" + "CoprocessorServiceResult\022&\n\005value\030\001 \001(\0132" + "\027.hbase.pb.NameBytesPair\"v\n\031CoprocessorS" + @@ -37271,8 +37394,8 @@ public final class ClientProtos { "b.CoprocessorServiceCall\"o\n\032CoprocessorS" + "erviceResponse\022)\n\006region\030\001 \002(\0132\031.hbase.p" + "b.RegionSpecifier\022&\n\005value\030\002 \002(\0132\027.hbase" + - ".pb.NameBytesPair\"\226\001\n\006Action\022\r\n\005index\030\001 " + - "\001(\r\022)\n\010mutation\030\002 \001(\0132\027.hbase.pb.Mutatio", + ".pb.NameBytesPair\"\226\001\n\006Action\022\r\n\005index\030\001 ", + "\001(\r\022)\n\010mutation\030\002 \001(\0132\027.hbase.pb.Mutatio" + "nProto\022\032\n\003get\030\003 \001(\0132\r.hbase.pb.Get\0226\n\014se" + "rvice_call\030\004 \001(\0132 .hbase.pb.CoprocessorS" + "erviceCall\"k\n\014RegionAction\022)\n\006region\030\001 \002" + @@ -37281,8 +37404,8 @@ public final class ClientProtos { "c\n\017RegionLoadStats\022\027\n\014memstoreLoad\030\001 \001(\005" + ":\0010\022\030\n\rheapOccupancy\030\002 \001(\005:\0010\022\035\n\022compact" + "ionPressure\030\003 \001(\005:\0010\"j\n\024MultiRegionLoadS" + - "tats\022)\n\006region\030\001 \003(\0132\031.hbase.pb.RegionSp" + - "ecifier\022\'\n\004stat\030\002 \003(\0132\031.hbase.pb.RegionL", + "tats\022)\n\006region\030\001 \003(\0132\031.hbase.pb.RegionSp", + "ecifier\022\'\n\004stat\030\002 \003(\0132\031.hbase.pb.RegionL" + "oadStats\"\336\001\n\021ResultOrException\022\r\n\005index\030" + "\001 \001(\r\022 \n\006result\030\002 \001(\0132\020.hbase.pb.Result\022" + "*\n\texception\030\003 \001(\0132\027.hbase.pb.NameBytesP" + @@ -37291,8 +37414,8 @@ public final class ClientProtos { "(\0132\031.hbase.pb.RegionLoadStatsB\002\030\001\"x\n\022Reg" + "ionActionResult\0226\n\021resultOrException\030\001 \003" + "(\0132\033.hbase.pb.ResultOrException\022*\n\texcep" + - "tion\030\002 \001(\0132\027.hbase.pb.NameBytesPair\"x\n\014M" + - "ultiRequest\022,\n\014regionAction\030\001 \003(\0132\026.hbas", + "tion\030\002 \001(\0132\027.hbase.pb.NameBytesPair\"x\n\014M", + "ultiRequest\022,\n\014regionAction\030\001 \003(\0132\026.hbas" + "e.pb.RegionAction\022\022\n\nnonceGroup\030\002 \001(\004\022&\n" + "\tcondition\030\003 \001(\0132\023.hbase.pb.Condition\"\226\001" + "\n\rMultiResponse\0228\n\022regionActionResult\030\001 " + @@ -37301,8 +37424,8 @@ public final class ClientProtos { ".hbase.pb.MultiRegionLoadStats*\'\n\013Consis" + "tency\022\n\n\006STRONG\020\000\022\014\n\010TIMELINE\020\0012\203\004\n\rClie" + "ntService\0222\n\003Get\022\024.hbase.pb.GetRequest\032\025" + - ".hbase.pb.GetResponse\022;\n\006Mutate\022\027.hbase." + - "pb.MutateRequest\032\030.hbase.pb.MutateRespon", + ".hbase.pb.GetResponse\022;\n\006Mutate\022\027.hbase.", + "pb.MutateRequest\032\030.hbase.pb.MutateRespon" + "se\0225\n\004Scan\022\025.hbase.pb.ScanRequest\032\026.hbas" + "e.pb.ScanResponse\022P\n\rBulkLoadHFile\022\036.hba" + "se.pb.BulkLoadHFileRequest\032\037.hbase.pb.Bu" + @@ -37311,8 +37434,8 @@ public final class ClientProtos { "pb.CoprocessorServiceResponse\022d\n\027ExecReg" + "ionServerService\022#.hbase.pb.CoprocessorS" + "erviceRequest\032$.hbase.pb.CoprocessorServ" + - "iceResponse\0228\n\005Multi\022\026.hbase.pb.MultiReq" + - "uest\032\027.hbase.pb.MultiResponseBB\n*org.apa", + "iceResponse\0228\n\005Multi\022\026.hbase.pb.MultiReq", + "uest\032\027.hbase.pb.MultiResponseBB\n*org.apa" + "che.hadoop.hbase.protobuf.generatedB\014Cli" + "entProtosH\001\210\001\001\240\001\001" }; @@ -37344,7 +37467,7 @@ public final class ClientProtos { internal_static_hbase_pb_Get_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_hbase_pb_Get_descriptor, - new java.lang.String[] { "Row", "Column", "Attribute", "Filter", "TimeRange", "MaxVersions", "CacheBlocks", "StoreLimit", "StoreOffset", "ExistenceOnly", "ClosestRowBefore", "Consistency", "CfTimeRange", }); + new java.lang.String[] { "Row", "Column", "Attribute", "Filter", "TimeRange", "MaxVersions", "CacheBlocks", "StoreLimit", "StoreOffset", "ExistenceOnly", "ClosestRowBefore", "Consistency", "CfTimeRange", "LoadColumnFamiliesOnDemand", }); internal_static_hbase_pb_Result_descriptor = getDescriptor().getMessageTypes().get(4); internal_static_hbase_pb_Result_fieldAccessorTable = new http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-protocol/src/main/protobuf/Client.proto ---------------------------------------------------------------------- diff --git a/hbase-protocol/src/main/protobuf/Client.proto b/hbase-protocol/src/main/protobuf/Client.proto index 9a68031..83f4e25 100644 --- a/hbase-protocol/src/main/protobuf/Client.proto +++ b/hbase-protocol/src/main/protobuf/Client.proto @@ -88,6 +88,7 @@ message Get { optional Consistency consistency = 12 [default = STRONG]; repeated ColumnFamilyTimeRange cf_time_range = 13; + optional bool load_column_families_on_demand = 14; /* DO NOT add defaults to load_column_families_on_demand. */ } message Result { http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 71c7892..de2461b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -7270,6 +7270,9 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi scan = new Scan(get); } + if (scan.getLoadColumnFamiliesOnDemandValue() == null) { + scan.setLoadColumnFamiliesOnDemand(isLoadingCfsOnDemandDefault()); + } RegionScanner scanner = null; try { scanner = getScanner(scan, null, nonceGroup, nonce); http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java index e21e51a..de6ecab 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java @@ -5730,6 +5730,50 @@ public class TestFromClientSide { } @Test + public void testEmptyFilterList() throws Exception { + // Test Initialization. + TableName TABLE = TableName.valueOf("testEmptyFilterList"); + Table table = TEST_UTIL.createTable(TABLE, FAMILY); + + // Insert one row each region + Put put = new Put(Bytes.toBytes("row")); + put.addColumn(FAMILY, QUALIFIER, VALUE); + table.put(put); + + List<Result> scanResults = new LinkedList<>(); + Scan scan = new Scan(); + scan.setFilter(new FilterList()); + try (ResultScanner scanner = table.getScanner(scan)) { + for (Result r : scanner) { + scanResults.add(r); + } + } + + Get g = new Get(Bytes.toBytes("row")); + g.setFilter(new FilterList()); + Result getResult = table.get(g); + if (scanResults.isEmpty()) { + assertTrue(getResult.isEmpty()); + } else if (scanResults.size() == 1) { + Result scanResult = scanResults.get(0); + assertEquals(scanResult.rawCells().length, getResult.rawCells().length); + for (int i = 0; i != scanResult.rawCells().length; ++i) { + Cell scanCell = scanResult.rawCells()[i]; + Cell getCell = getResult.rawCells()[i]; + assertEquals(0, Bytes.compareTo(CellUtil.cloneRow(scanCell), CellUtil.cloneRow(getCell))); + assertEquals(0, + Bytes.compareTo(CellUtil.cloneFamily(scanCell), CellUtil.cloneFamily(getCell))); + assertEquals(0, + Bytes.compareTo(CellUtil.cloneQualifier(scanCell), CellUtil.cloneQualifier(getCell))); + assertEquals(0, + Bytes.compareTo(CellUtil.cloneValue(scanCell), CellUtil.cloneValue(getCell))); + } + } else { + fail("The result retrieved from SCAN and Get should be same"); + } + } + + @Test public void testSmallScan() throws Exception { // Test Initialization. TableName TABLE = TableName.valueOf("testSmallScan"); http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java index ddbbb74..7cff7fd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java @@ -82,6 +82,7 @@ public class TestProtobufUtil { columnBuilder.clear(); columnBuilder.setFamily(ByteString.copyFromUtf8("f2")); getBuilder.addColumn(columnBuilder.build()); + getBuilder.setLoadColumnFamiliesOnDemand(true); ClientProtos.Get proto = getBuilder.build(); // default fields
