hbase git commit: HBASE-14269 FuzzyRowFilter omits certain rows when multiple fuzzy keys exist (hongbin ma)
Repository: hbase
Updated Branches:
refs/heads/master 506726ed2 -> 6661f2d02
HBASE-14269 FuzzyRowFilter omits certain rows when multiple fuzzy keys exist
(hongbin ma)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6661f2d0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6661f2d0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6661f2d0
Branch: refs/heads/master
Commit: 6661f2d0254f1da9d8cbbd717274421a2ddcb95f
Parents: 506726e
Author: tedyu
Authored: Wed Aug 26 07:23:43 2015 -0700
Committer: tedyu
Committed: Wed Aug 26 07:23:43 2015 -0700
--
.../hadoop/hbase/filter/FuzzyRowFilter.java | 116 +--
.../filter/TestFuzzyRowFilterEndToEnd.java | 94 ++-
2 files changed, 125 insertions(+), 85 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/6661f2d0/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
index 661400b..a9dd596 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
@@ -19,9 +19,9 @@ package org.apache.hadoop.hbase.filter;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.PriorityQueue;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
@@ -160,82 +160,82 @@ public class FuzzyRowFilter extends FilterBase {
@Override
public Cell getNextCellHint(Cell currentCell) {
-boolean result = true;
-if (tracker.needsUpdate()) {
- result = tracker.updateTracker(currentCell);
-}
+boolean result = tracker.updateTracker(currentCell);
if (result == false) {
done = true;
return null;
}
byte[] nextRowKey = tracker.nextRow();
-// We need to compare nextRowKey with currentCell
-int compareResult = CellComparator.COMPARATOR.compareRows(currentCell,
nextRowKey, 0,
-nextRowKey.length);
-if ((reversed && compareResult < 0) || (!reversed && compareResult > 0)) {
- // This can happen when we have multilpe filters and some other filter
- // returns next row with hint which is larger (smaller for reverse)
- // than the current (really?)
- result = tracker.updateTracker(currentCell);
- if (result == false) {
-done = true;
-return null;
- } else {
-nextRowKey = tracker.nextRow();
- }
-}
return KeyValueUtil.createFirstOnRow(nextRowKey);
}
/**
- * If we have multiple fuzzy keys, row tracker should improve overall
performance It calculates
- * all next rows (one per every fuzzy key), sort them accordingly (ascending
for regular and
- * descending for reverse). Next time getNextCellHint is called we check row
tracker first and
- * return next row from the tracker if it exists, if there are no rows in
the tracker we update
- * tracker with a current cell and return first row.
+ * If we have multiple fuzzy keys, row tracker should improve overall
performance. It calculates
+ * all next rows (one per every fuzzy key) and put them (the fuzzy key is
bundled) into a priority
+ * queue so that the smallest row key always appears at queue head, which
helps to decide the
+ * "Next Cell Hint". As scanning going on, the number of candidate rows in
the RowTracker will
+ * remain the size of fuzzy keys until some of the fuzzy keys won't possibly
have matches any
+ * more.
*/
private class RowTracker {
-private final List nextRows;
-private int next = -1;
+private final PriorityQueue>> nextRows;
+private boolean initialized = false;
RowTracker() {
- nextRows = new ArrayList();
-}
-
-boolean needsUpdate() {
- return next == -1 || next == nextRows.size();
+ nextRows =
+ new PriorityQueue>>(fuzzyKeysData.size(),
+ new Comparator>>() {
+@Override
+public int compare(Pair> o1,
+Pair> o2) {
+ int compare = Bytes.compareTo(o1.getFirst(), o2.getFirst());
+ if (!isReversed()) {
+return compare;
+ } else {
+return -compare;
+ }
+}
+ });
}
byte[] nextRow() {
- if (next < 0 || next == nextRows.size()) return null;
- return nextRows.get(next++);
+ if (nextRows.isEmpty()) {
+
[2/4] hbase git commit: HBASE-13212: Procedure V2 - master Create/Modify/Delete namespace (Stephen Yuan Jiang)
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc79b3c5/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
--
diff --git
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
index eb98b42..412d792 100644
---
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
+++
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
@@ -16424,6 +16424,26 @@ public final class MasterProtos {
* required .hbase.pb.NamespaceDescriptor namespaceDescriptor =
1;
*/
org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NamespaceDescriptorOrBuilder
getNamespaceDescriptorOrBuilder();
+
+// optional uint64 nonce_group = 2 [default = 0];
+/**
+ * optional uint64 nonce_group = 2 [default = 0];
+ */
+boolean hasNonceGroup();
+/**
+ * optional uint64 nonce_group = 2 [default = 0];
+ */
+long getNonceGroup();
+
+// optional uint64 nonce = 3 [default = 0];
+/**
+ * optional uint64 nonce = 3 [default = 0];
+ */
+boolean hasNonce();
+/**
+ * optional uint64 nonce = 3 [default = 0];
+ */
+long getNonce();
}
/**
* Protobuf type {@code hbase.pb.CreateNamespaceRequest}
@@ -16489,6 +16509,16 @@ public final class MasterProtos {
bitField0_ |= 0x0001;
break;
}
+case 16: {
+ bitField0_ |= 0x0002;
+ nonceGroup_ = input.readUInt64();
+ break;
+}
+case 24: {
+ bitField0_ |= 0x0004;
+ nonce_ = input.readUInt64();
+ break;
+}
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -16551,8 +16581,42 @@ public final class MasterProtos {
return namespaceDescriptor_;
}
+// optional uint64 nonce_group = 2 [default = 0];
+public static final int NONCE_GROUP_FIELD_NUMBER = 2;
+private long nonceGroup_;
+/**
+ * optional uint64 nonce_group = 2 [default = 0];
+ */
+public boolean hasNonceGroup() {
+ return ((bitField0_ & 0x0002) == 0x0002);
+}
+/**
+ * optional uint64 nonce_group = 2 [default = 0];
+ */
+public long getNonceGroup() {
+ return nonceGroup_;
+}
+
+// optional uint64 nonce = 3 [default = 0];
+public static final int NONCE_FIELD_NUMBER = 3;
+private long nonce_;
+/**
+ * optional uint64 nonce = 3 [default = 0];
+ */
+public boolean hasNonce() {
+ return ((bitField0_ & 0x0004) == 0x0004);
+}
+/**
+ * optional uint64 nonce = 3 [default = 0];
+ */
+public long getNonce() {
+ return nonce_;
+}
+
private void initFields() {
namespaceDescriptor_ =
org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NamespaceDescriptor.getDefaultInstance();
+ nonceGroup_ = 0L;
+ nonce_ = 0L;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -16577,6 +16641,12 @@ public final class MasterProtos {
if (((bitField0_ & 0x0001) == 0x0001)) {
output.writeMessage(1, namespaceDescriptor_);
}
+ if (((bitField0_ & 0x0002) == 0x0002)) {
+output.writeUInt64(2, nonceGroup_);
+ }
+ if (((bitField0_ & 0x0004) == 0x0004)) {
+output.writeUInt64(3, nonce_);
+ }
getUnknownFields().writeTo(output);
}
@@ -16590,6 +16660,14 @@ public final class MasterProtos {
size += com.google.protobuf.CodedOutputStream
.computeMessageSize(1, namespaceDescriptor_);
}
+ if (((bitField0_ & 0x0002) == 0x0002)) {
+size += com.google.protobuf.CodedOutputStream
+ .computeUInt64Size(2, nonceGroup_);
+ }
+ if (((bitField0_ & 0x0004) == 0x0004)) {
+size += com.google.protobuf.CodedOutputStream
+ .computeUInt64Size(3, nonce_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -16618,6 +16696,16 @@ public final class MasterProtos {
result = result && getNamespaceDescriptor()
.equals(other.getNamespaceDescriptor());
}
+ result = result && (hasNonceGroup() == other.hasNonceGroup());
+ if (hasNonceGroup()) {
+result = result && (getNonceGroup()
+== other.getNonceGroup());
+ }
+ result = result && (hasNonce() == other.hasNonce());
+ if (hasNonce()) {
+result = result && (getNonce()
+== other.getNonce());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
re
[1/4] hbase git commit: HBASE-13212: Procedure V2 - master Create/Modify/Delete namespace (Stephen Yuan Jiang)
Repository: hbase
Updated Branches:
refs/heads/master 6661f2d02 -> dc79b3c5c
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc79b3c5/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
new file mode 100644
index 000..c91092a
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
@@ -0,0 +1,364 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master.procedure;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.NamespaceExistException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.TableNamespaceManager;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
+import
org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.CreateNamespaceState;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * The procedure to create a new namespace.
+ */
+@InterfaceAudience.Private
+public class CreateNamespaceProcedure
+extends StateMachineProcedure
+implements TableProcedureInterface {
+ private static final Log LOG =
LogFactory.getLog(CreateNamespaceProcedure.class);
+
+ private final AtomicBoolean aborted = new AtomicBoolean(false);
+
+ private NamespaceDescriptor nsDescriptor;
+ private Boolean traceEnabled;
+
+ public CreateNamespaceProcedure() {
+this.traceEnabled = null;
+ }
+
+ public CreateNamespaceProcedure(
+ final MasterProcedureEnv env,
+ final NamespaceDescriptor nsDescriptor) throws IOException {
+this.nsDescriptor = nsDescriptor;
+this.traceEnabled = null;
+ }
+
+ @Override
+ protected Flow executeFromState(final MasterProcedureEnv env, final
CreateNamespaceState state)
+ throws InterruptedException {
+if (isTraceEnabled()) {
+ LOG.trace(this + " execute state=" + state);
+}
+
+try {
+ switch (state) {
+ case CREATE_NAMESPACE_PREPARE:
+prepareCreate(env);
+setNextState(CreateNamespaceState.CREATE_NAMESPACE_CREATE_DIRECTORY);
+break;
+ case CREATE_NAMESPACE_CREATE_DIRECTORY:
+createDirectory(env, nsDescriptor);
+
setNextState(CreateNamespaceState.CREATE_NAMESPACE_INSERT_INTO_NS_TABLE);
+break;
+ case CREATE_NAMESPACE_INSERT_INTO_NS_TABLE:
+insertIntoNSTable(env, nsDescriptor);
+setNextState(CreateNamespaceState.CREATE_NAMESPACE_UPDATE_ZK);
+break;
+ case CREATE_NAMESPACE_UPDATE_ZK:
+updateZKNamespaceManager(env, nsDescriptor);
+
setNextState(CreateNamespaceState.CREATE_NAMESPACE_SET_NAMESPACE_QUOTA);
+break;
+ case CREATE_NAMESPACE_SET_NAMESPACE_QUOTA:
+setNamespaceQuota(env, nsDescriptor);
+return Flow.NO_MORE_STATE;
+ default:
+throw new UnsupportedOperationException(this + " unhandled state=" +
state);
+ }
+} catch (IOException e) {
+ LOG.warn("Error trying to create the namespace" + nsDescriptor.getName()
++ " (in state=" + state + ")", e);
+
+ setFailure("master-create-namespace", e);
+}
+return Flow.HAS_MORE_STATE;
+ }
+
+ @Override
+ protected void rollbackState(final MasterProcedureEnv env, final
CreateNamespaceState state)
+ throws IOException {
+if (isTraceEnabled()) {
+ LOG.trace(this + " rollback state=" + state);
+}
+try {
+ switch (state) {
+ case
[4/4] hbase git commit: HBASE-13212: Procedure V2 - master Create/Modify/Delete namespace (Stephen Yuan Jiang)
HBASE-13212: Procedure V2 - master Create/Modify/Delete namespace (Stephen Yuan Jiang) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dc79b3c5 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dc79b3c5 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dc79b3c5 Branch: refs/heads/master Commit: dc79b3c5c91b7bc0c230199fe60eb51324770084 Parents: 6661f2d Author: Stephen Yuan Jiang Authored: Wed Aug 26 08:18:03 2015 -0700 Committer: Stephen Yuan Jiang Committed: Wed Aug 26 08:18:03 2015 -0700 -- .../generated/MasterProcedureProtos.java| 2910 -- .../hbase/protobuf/generated/MasterProtos.java | 999 -- hbase-protocol/src/main/protobuf/Master.proto |6 + .../src/main/protobuf/MasterProcedure.proto | 36 + .../apache/hadoop/hbase/ZKNamespaceManager.java |9 +- .../org/apache/hadoop/hbase/master/HMaster.java | 56 +- .../hadoop/hbase/master/MasterRpcServices.java | 14 +- .../hadoop/hbase/master/MasterServices.java | 38 +- .../hbase/master/TableNamespaceManager.java | 216 +- .../procedure/CreateNamespaceProcedure.java | 364 +++ .../procedure/DeleteNamespaceProcedure.java | 398 +++ .../procedure/ModifyNamespaceProcedure.java | 281 ++ .../hadoop/hbase/master/TestCatalogJanitor.java | 28 +- .../MasterProcedureTestingUtility.java |2 - .../procedure/TestCreateNamespaceProcedure.java | 292 ++ .../procedure/TestDeleteNamespaceProcedure.java | 282 ++ .../procedure/TestModifyNamespaceProcedure.java | 295 ++ 17 files changed, 5657 insertions(+), 569 deletions(-) --
[3/4] hbase git commit: HBASE-13212: Procedure V2 - master Create/Modify/Delete namespace (Stephen Yuan Jiang)
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc79b3c5/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProcedureProtos.java
--
diff --git
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProcedureProtos.java
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProcedureProtos.java
index 9bf4c98..d40c1f7 100644
---
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProcedureProtos.java
+++
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProcedureProtos.java
@@ -499,6 +499,315 @@ public final class MasterProcedureProtos {
}
/**
+ * Protobuf enum {@code hbase.pb.CreateNamespaceState}
+ */
+ public enum CreateNamespaceState
+ implements com.google.protobuf.ProtocolMessageEnum {
+/**
+ * CREATE_NAMESPACE_PREPARE = 1;
+ */
+CREATE_NAMESPACE_PREPARE(0, 1),
+/**
+ * CREATE_NAMESPACE_CREATE_DIRECTORY = 2;
+ */
+CREATE_NAMESPACE_CREATE_DIRECTORY(1, 2),
+/**
+ * CREATE_NAMESPACE_INSERT_INTO_NS_TABLE = 3;
+ */
+CREATE_NAMESPACE_INSERT_INTO_NS_TABLE(2, 3),
+/**
+ * CREATE_NAMESPACE_UPDATE_ZK = 4;
+ */
+CREATE_NAMESPACE_UPDATE_ZK(3, 4),
+/**
+ * CREATE_NAMESPACE_SET_NAMESPACE_QUOTA = 5;
+ */
+CREATE_NAMESPACE_SET_NAMESPACE_QUOTA(4, 5),
+;
+
+/**
+ * CREATE_NAMESPACE_PREPARE = 1;
+ */
+public static final int CREATE_NAMESPACE_PREPARE_VALUE = 1;
+/**
+ * CREATE_NAMESPACE_CREATE_DIRECTORY = 2;
+ */
+public static final int CREATE_NAMESPACE_CREATE_DIRECTORY_VALUE = 2;
+/**
+ * CREATE_NAMESPACE_INSERT_INTO_NS_TABLE = 3;
+ */
+public static final int CREATE_NAMESPACE_INSERT_INTO_NS_TABLE_VALUE = 3;
+/**
+ * CREATE_NAMESPACE_UPDATE_ZK = 4;
+ */
+public static final int CREATE_NAMESPACE_UPDATE_ZK_VALUE = 4;
+/**
+ * CREATE_NAMESPACE_SET_NAMESPACE_QUOTA = 5;
+ */
+public static final int CREATE_NAMESPACE_SET_NAMESPACE_QUOTA_VALUE = 5;
+
+
+public final int getNumber() { return value; }
+
+public static CreateNamespaceState valueOf(int value) {
+ switch (value) {
+case 1: return CREATE_NAMESPACE_PREPARE;
+case 2: return CREATE_NAMESPACE_CREATE_DIRECTORY;
+case 3: return CREATE_NAMESPACE_INSERT_INTO_NS_TABLE;
+case 4: return CREATE_NAMESPACE_UPDATE_ZK;
+case 5: return CREATE_NAMESPACE_SET_NAMESPACE_QUOTA;
+default: return null;
+ }
+}
+
+public static
com.google.protobuf.Internal.EnumLiteMap
+internalGetValueMap() {
+ return internalValueMap;
+}
+private static
com.google.protobuf.Internal.EnumLiteMap
+internalValueMap =
+ new com.google.protobuf.Internal.EnumLiteMap()
{
+public CreateNamespaceState findValueByNumber(int number) {
+ return CreateNamespaceState.valueOf(number);
+}
+ };
+
+public final com.google.protobuf.Descriptors.EnumValueDescriptor
+getValueDescriptor() {
+ return getDescriptor().getValues().get(index);
+}
+public final com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptorForType() {
+ return getDescriptor();
+}
+public static final com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptor() {
+ return
org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.getDescriptor().getEnumTypes().get(4);
+}
+
+private static final CreateNamespaceState[] VALUES = values();
+
+public static CreateNamespaceState valueOf(
+com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+ if (desc.getType() != getDescriptor()) {
+throw new java.lang.IllegalArgumentException(
+ "EnumValueDescriptor is not for this type.");
+ }
+ return VALUES[desc.getIndex()];
+}
+
+private final int index;
+private final int value;
+
+private CreateNamespaceState(int index, int value) {
+ this.index = index;
+ this.value = value;
+}
+
+// @@protoc_insertion_point(enum_scope:hbase.pb.CreateNamespaceState)
+ }
+
+ /**
+ * Protobuf enum {@code hbase.pb.ModifyNamespaceState}
+ */
+ public enum ModifyNamespaceState
+ implements com.google.protobuf.ProtocolMessageEnum {
+/**
+ * MODIFY_NAMESPACE_PREPARE = 1;
+ */
+MODIFY_NAMESPACE_PREPARE(0, 1),
+/**
+ * MODIFY_NAMESPACE_UPDATE_NS_TABLE = 2;
+ */
+MODIFY_NAMESPACE_UPDATE_NS_TABLE(1, 2),
+/**
+ * MODIFY_NAMESPACE_UPDATE_ZK = 3;
+ */
+MODIFY_NAMESPACE_UPDATE_ZK(2, 3),
+;
+
+/**
+ * MODIFY_NAMESPACE_PREPARE = 1;
+ */
+public static final int MODIFY_NAMESPACE_PREPARE_VALUE = 1;
+/**
+ * MODIFY_NAMESPACE_UPDATE_NS_TABLE = 2;
+ */
+public static final int MODIFY_NAMESPACE_UPDATE_NS_TABLE_VAL
hbase git commit: HBASE-14078 improve error message when HMaster can't bind to port
Repository: hbase
Updated Branches:
refs/heads/master dc79b3c5c -> ff86749ca
HBASE-14078 improve error message when HMaster can't bind to port
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ff86749c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ff86749c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ff86749c
Branch: refs/heads/master
Commit: ff86749caeb63eafcf10cbfba45334757a791384
Parents: dc79b3c
Author: stack
Authored: Wed Aug 26 09:17:33 2015 -0700
Committer: stack
Committed: Wed Aug 26 09:17:46 2015 -0700
--
.../org/apache/hadoop/hbase/master/HMaster.java | 18 --
.../hadoop/hbase/regionserver/RSRpcServices.java | 16
2 files changed, 20 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ff86749c/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 34776d5..7834f25 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -2306,16 +2306,14 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
Constructor c =
masterClass.getConstructor(Configuration.class,
CoordinatedStateManager.class);
return c.newInstance(conf, cp);
-} catch (InvocationTargetException ite) {
- Throwable target = ite.getTargetException() != null?
-ite.getTargetException(): ite;
- if (target.getCause() != null) target = target.getCause();
- throw new RuntimeException("Failed construction of Master: " +
-masterClass.toString(), target);
-} catch (Exception e) {
- throw new RuntimeException("Failed construction of Master: " +
-masterClass.toString() + ((e.getCause() != null)?
- e.getCause().getMessage(): ""), e);
+} catch(Exception e) {
+ Throwable error = e;
+ if (e instanceof InvocationTargetException &&
+ ((InvocationTargetException)e).getTargetException() != null) {
+error = ((InvocationTargetException)e).getTargetException();
+ }
+ throw new RuntimeException("Failed construction of Master: " +
masterClass.toString() + ". "
+, error);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/ff86749c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 65cedee..70ac7a6 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
import java.io.InterruptedIOException;
+import java.net.BindException;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
@@ -946,10 +947,17 @@ public class RSRpcServices implements
HBaseRPCErrorHandler,
String name = rs.getProcessName() + "/" + initialIsa.toString();
// Set how many times to retry talking to another server over HConnection.
ConnectionUtils.setServerSideHConnectionRetriesConfig(rs.conf, name, LOG);
-rpcServer = new RpcServer(rs, name, getServices(),
- bindAddress, // use final bindAddress for this server.
- rs.conf,
- rpcSchedulerFactory.create(rs.conf, this, rs));
+try {
+ rpcServer = new RpcServer(rs, name, getServices(),
+bindAddress, // use final bindAddress for this server.
+rs.conf,
+rpcSchedulerFactory.create(rs.conf, this, rs));
+} catch(BindException be) {
+ String configName = (this instanceof MasterRpcServices) ?
HConstants.MASTER_PORT :
+HConstants.REGIONSERVER_PORT;
+throw new IOException(be.getMessage() + ". To switch ports use the '"
+ configName +
+ "' configuration property.", be.getCause() != null ? be.getCause() :
be);
+}
scannerLeaseTimeoutPeriod = rs.conf.getInt(
HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
hbase git commit: HBASE-14078 improve error message when HMaster can't bind to port
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 268a69f17 -> 180e8b8fd
HBASE-14078 improve error message when HMaster can't bind to port
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/180e8b8f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/180e8b8f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/180e8b8f
Branch: refs/heads/branch-1.2
Commit: 180e8b8fd68f7d6181bbca17183f55bed2fd844f
Parents: 268a69f
Author: stack
Authored: Wed Aug 26 09:17:33 2015 -0700
Committer: stack
Committed: Wed Aug 26 09:18:31 2015 -0700
--
.../org/apache/hadoop/hbase/master/HMaster.java | 18 --
.../hadoop/hbase/regionserver/RSRpcServices.java | 16
2 files changed, 20 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/180e8b8f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 423deaf..8dbc321 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -2290,16 +2290,14 @@ public class HMaster extends HRegionServer implements
MasterServices, Server {
Constructor c =
masterClass.getConstructor(Configuration.class,
CoordinatedStateManager.class);
return c.newInstance(conf, cp);
-} catch (InvocationTargetException ite) {
- Throwable target = ite.getTargetException() != null?
-ite.getTargetException(): ite;
- if (target.getCause() != null) target = target.getCause();
- throw new RuntimeException("Failed construction of Master: " +
-masterClass.toString(), target);
-} catch (Exception e) {
- throw new RuntimeException("Failed construction of Master: " +
-masterClass.toString() + ((e.getCause() != null)?
- e.getCause().getMessage(): ""), e);
+} catch(Exception e) {
+ Throwable error = e;
+ if (e instanceof InvocationTargetException &&
+ ((InvocationTargetException)e).getTargetException() != null) {
+error = ((InvocationTargetException)e).getTargetException();
+ }
+ throw new RuntimeException("Failed construction of Master: " +
masterClass.toString() + ". "
+, error);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/180e8b8f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
index 69d2a89..0b939d5 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.regionserver;
import java.io.IOException;
import java.io.InterruptedIOException;
+import java.net.BindException;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
@@ -864,10 +865,17 @@ public class RSRpcServices implements
HBaseRPCErrorHandler,
String name = rs.getProcessName() + "/" + initialIsa.toString();
// Set how many times to retry talking to another server over HConnection.
ConnectionUtils.setServerSideHConnectionRetriesConfig(rs.conf, name, LOG);
-rpcServer = new RpcServer(rs, name, getServices(),
- bindAddress, // use final bindAddress for this server.
- rs.conf,
- rpcSchedulerFactory.create(rs.conf, this, rs));
+try {
+ rpcServer = new RpcServer(rs, name, getServices(),
+bindAddress, // use final bindAddress for this server.
+rs.conf,
+rpcSchedulerFactory.create(rs.conf, this, rs));
+} catch(BindException be) {
+ String configName = (this instanceof MasterRpcServices) ?
HConstants.MASTER_PORT :
+HConstants.REGIONSERVER_PORT;
+throw new IOException(be.getMessage() + ". To switch ports use the '"
+ configName +
+ "' configuration property.", be.getCause() != null ? be.getCause() :
be);
+}
scannerLeaseTimeoutPeriod = rs.conf.getInt(
HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
hbase git commit: HBASE-14310 test-patch.sh should handle spurious non-zero exit code from maven
Repository: hbase
Updated Branches:
refs/heads/master ff86749ca -> aca8c3b74
HBASE-14310 test-patch.sh should handle spurious non-zero exit code from maven
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/aca8c3b7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/aca8c3b7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/aca8c3b7
Branch: refs/heads/master
Commit: aca8c3b74b09646c72c4e0fe26a4b2103da0d288
Parents: ff86749
Author: tedyu
Authored: Wed Aug 26 09:37:41 2015 -0700
Committer: tedyu
Committed: Wed Aug 26 09:37:41 2015 -0700
--
dev-support/test-patch.sh | 9 ++---
1 file changed, 6 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/aca8c3b7/dev-support/test-patch.sh
--
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index 0b534034..33f84c5 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -335,10 +335,13 @@ setup () {
$MVN clean package checkstyle:checkstyle-aggregate findbugs:findbugs
-DskipTests \
-D${PROJECT_NAME}PatchProcess > $PATCH_DIR/trunkJavacWarnings.txt 2>&1
if [[ $? != 0 ]] ; then
+echo "mvn exit code was $?"
ERR=`$GREP -A 5 'Compilation failure' $PATCH_DIR/trunkJavacWarnings.txt`
-echo "Trunk compilation is broken?
-{code}$ERR{code}"
-cleanupAndExit 1
+if [[ ${#ERR} -ge 1 ]] ; then
+ echo "Trunk compilation is broken?
+ {code}$ERR{code}"
+ cleanupAndExit 1
+fi
fi
mv target/checkstyle-result.xml $PATCH_DIR/trunkCheckstyle.xml
collectFindbugsReports trunk $BASEDIR $PATCH_DIR
[2/2] hbase git commit: HBASE-14312 Forward port some fixes from hbase-6721-0.98 to hbase-6721 (Francis Liu)
HBASE-14312 Forward port some fixes from hbase-6721-0.98 to hbase-6721 (Francis
Liu)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fade887a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fade887a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fade887a
Branch: refs/heads/hbase-6721
Commit: fade887a309e6b12a4e580c5207794b3b05e9b4e
Parents: 16f65ba
Author: Andrew Purtell
Authored: Wed Aug 26 13:04:29 2015 -0700
Committer: Andrew Purtell
Committed: Wed Aug 26 13:04:29 2015 -0700
--
.../hbase/group/IntegrationTestGroup.java |5 -
.../hbase/protobuf/generated/MasterProtos.java | 1348 ++
hbase-protocol/src/main/protobuf/Master.proto |7 -
.../hadoop/hbase/group/GroupAdminServer.java| 94 +-
.../hbase/group/GroupBasedLoadBalancer.java | 37 +-
.../hbase/group/GroupInfoManagerImpl.java | 32 +-
.../org/apache/hadoop/hbase/group/MXBean.java |8 +-
.../apache/hadoop/hbase/group/MXBeanImpl.java | 19 +-
.../hadoop/hbase/master/AssignmentManager.java |8 +-
.../hbase/master/MasterCoprocessorHost.java | 43 +-
.../hadoop/hbase/master/MasterRpcServices.java |8 +
.../apache/hadoop/hbase/group/TestGroups.java | 42 +-
.../hadoop/hbase/group/TestGroupsBase.java |8 +-
.../hbase/group/TestGroupsOfflineMode.java | 181 +++
.../security/access/TestAccessController.java | 50 -
hbase-shell/src/main/ruby/hbase/admin.rb| 20 -
hbase-shell/src/main/ruby/hbase/group_admin.rb |4 +-
hbase-shell/src/main/ruby/shell.rb |2 -
.../commands/list_group_server_transitions.rb | 44 -
.../ruby/shell/commands/list_group_tables.rb| 45 -
20 files changed, 498 insertions(+), 1507 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/fade887a/hbase-it/src/test/java/org/apache/hadoop/hbase/group/IntegrationTestGroup.java
--
diff --git
a/hbase-it/src/test/java/org/apache/hadoop/hbase/group/IntegrationTestGroup.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/group/IntegrationTestGroup.java
index 22cddd7..62f4f8a 100644
---
a/hbase-it/src/test/java/org/apache/hadoop/hbase/group/IntegrationTestGroup.java
+++
b/hbase-it/src/test/java/org/apache/hadoop/hbase/group/IntegrationTestGroup.java
@@ -74,11 +74,6 @@ public class IntegrationTestGroup extends TestGroupsBase {
((IntegrationTestingUtility)TEST_UTIL).restoreCluster();
LOG.info("Done restoring the cluster");
-groupAdmin.addGroup("master");
-groupAdmin.moveServers(
-
Sets.newHashSet(cluster.getInitialClusterStatus().getMaster().getHostPort()),
-"master");
-
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() {
@Override
public boolean evaluate() throws Exception {
http://git-wip-us.apache.org/repos/asf/hbase/blob/fade887a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
--
diff --git
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
index beee110..71cc5d4 100644
---
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
+++
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
@@ -61124,1065 +61124,6 @@ public final class MasterProtos {
//
@@protoc_insertion_point(class_scope:hbase.pb.GetGroupInfoOfServerResponse)
}
- public interface ListServersInTransitionRequestOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
- }
- /**
- * Protobuf type {@code hbase.pb.ListServersInTransitionRequest}
- */
- public static final class ListServersInTransitionRequest extends
- com.google.protobuf.GeneratedMessage
- implements ListServersInTransitionRequestOrBuilder {
-// Use ListServersInTransitionRequest.newBuilder() to construct.
-private
ListServersInTransitionRequest(com.google.protobuf.GeneratedMessage.Builder
builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
-}
-private ListServersInTransitionRequest(boolean noInit) {
this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
-private static final ListServersInTransitionRequest defaultInstance;
-public static ListServersInTransitionRequest getDefaultInstance() {
- return defaultInstance;
-}
-
-public ListServersInTransitionRequest getDefaultInstanceForType() {
- return defaultInstance;
-}
-
-private final com.google.protobuf.UnknownFieldSet unknownFields;
-@jav
[1/2] hbase git commit: HBASE-14312 Forward port some fixes from hbase-6721-0.98 to hbase-6721 (Francis Liu)
Repository: hbase
Updated Branches:
refs/heads/hbase-6721 16f65badc -> fade887a3
http://git-wip-us.apache.org/repos/asf/hbase/blob/fade887a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index 7a3b01f..05d2fb3 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -62,12 +62,15 @@ public class MasterCoprocessorHost
static class MasterEnvironment extends CoprocessorHost.Environment
implements MasterCoprocessorEnvironment {
private MasterServices masterServices;
+final boolean supportGroupCPs;
public MasterEnvironment(final Class implClass, final Coprocessor impl,
final int priority, final int seq, final Configuration conf,
final MasterServices services) {
super(impl, priority, seq, conf);
this.masterServices = services;
+ supportGroupCPs = !useLegacyMethod(impl.getClass(),
+ "preBalanceGroup", ObserverContext.class, String.class);
}
public MasterServices getMasterServices() {
@@ -1110,7 +1113,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserver oserver,
ObserverContext ctx) throws
IOException {
-oserver.preMoveServers(ctx, servers, targetGroup);
+if(((MasterEnvironment)ctx.getEnvironment()).supportGroupCPs) {
+ oserver.preMoveServers(ctx, servers, targetGroup);
+}
}
});
}
@@ -1121,7 +1126,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserver oserver,
ObserverContext ctx) throws
IOException {
-oserver.postMoveServers(ctx, servers, targetGroup);
+if(((MasterEnvironment)ctx.getEnvironment()).supportGroupCPs) {
+ oserver.postMoveServers(ctx, servers, targetGroup);
+}
}
});
}
@@ -1132,7 +1139,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserver oserver,
ObserverContext ctx) throws
IOException {
-oserver.preMoveTables(ctx, tables, targetGroup);
+if(((MasterEnvironment)ctx.getEnvironment()).supportGroupCPs) {
+ oserver.preMoveTables(ctx, tables, targetGroup);
+}
}
});
}
@@ -1143,7 +1152,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserver oserver,
ObserverContext ctx) throws
IOException {
-oserver.postMoveTables(ctx, tables, targetGroup);
+if(((MasterEnvironment)ctx.getEnvironment()).supportGroupCPs) {
+ oserver.postMoveTables(ctx, tables, targetGroup);
+}
}
});
}
@@ -1154,7 +1165,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserver oserver,
ObserverContext ctx) throws
IOException {
-oserver.preAddGroup(ctx, name);
+if(((MasterEnvironment)ctx.getEnvironment()).supportGroupCPs) {
+ oserver.preAddGroup(ctx, name);
+}
}
});
}
@@ -1165,7 +1178,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserver oserver,
ObserverContext ctx) throws
IOException {
-oserver.postAddGroup(ctx, name);
+if (((MasterEnvironment) ctx.getEnvironment()).supportGroupCPs) {
+ oserver.postAddGroup(ctx, name);
+}
}
});
}
@@ -1176,7 +1191,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserver oserver,
ObserverContext ctx) throws
IOException {
-oserver.preRemoveGroup(ctx, name);
+if(((MasterEnvironment)ctx.getEnvironment()).supportGroupCPs) {
+ oserver.preRemoveGroup(ctx, name);
+}
}
});
}
@@ -1187,7 +1204,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserver oserver,
ObserverContext ctx) throws
IOException {
-oserver.postRemoveGroup(ctx, name);
+if(((MasterEnvironment)ctx.getEnvironment()).supportGroupCPs) {
+ oserver.postRemoveGroup(ctx, name);
+}
}
});
}
@@ -1198,7 +1217,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserver oserver,
ObserverContext ctx) throws
IOException {
-oserver.preBalanceGroup(ctx, name);
+if(((MasterEnvironment)ctx.getEnvironment()).supportGroupCPs) {
+ oserver.preBalanceGroup(ctx, name);
+}
}
});
}
@@ -1209,7 +1230,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserv
[2/3] hbase git commit: HBASE-14312 Forward port some fixes from hbase-6721-0.98 to hbase-6721 (Francis Liu)
HBASE-14312 Forward port some fixes from hbase-6721-0.98 to hbase-6721 (Francis
Liu)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f1e6b202
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f1e6b202
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f1e6b202
Branch: refs/heads/hbase-6721
Commit: f1e6b202e969821d974952af2226bfec56f046bd
Parents: 16f65ba
Author: Andrew Purtell
Authored: Wed Aug 26 13:04:29 2015 -0700
Committer: Andrew Purtell
Committed: Wed Aug 26 13:28:22 2015 -0700
--
.../hbase/group/IntegrationTestGroup.java |5 -
.../hbase/protobuf/generated/MasterProtos.java | 1348 ++
hbase-protocol/src/main/protobuf/Master.proto |7 -
.../hadoop/hbase/group/GroupAdminServer.java| 94 +-
.../hbase/group/GroupBasedLoadBalancer.java | 37 +-
.../hbase/group/GroupInfoManagerImpl.java | 32 +-
.../org/apache/hadoop/hbase/group/MXBean.java |8 +-
.../apache/hadoop/hbase/group/MXBeanImpl.java | 19 +-
.../hadoop/hbase/master/AssignmentManager.java |8 +-
.../hadoop/hbase/master/MasterRpcServices.java |8 +
.../apache/hadoop/hbase/group/TestGroups.java | 42 +-
.../hadoop/hbase/group/TestGroupsBase.java |8 +-
.../hbase/group/TestGroupsOfflineMode.java | 181 +++
.../security/access/TestAccessController.java | 50 -
hbase-shell/src/main/ruby/hbase/admin.rb| 20 -
hbase-shell/src/main/ruby/hbase/group_admin.rb |4 +-
hbase-shell/src/main/ruby/shell.rb |2 -
.../commands/list_group_server_transitions.rb | 44 -
.../ruby/shell/commands/list_group_tables.rb| 45 -
19 files changed, 465 insertions(+), 1497 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/f1e6b202/hbase-it/src/test/java/org/apache/hadoop/hbase/group/IntegrationTestGroup.java
--
diff --git
a/hbase-it/src/test/java/org/apache/hadoop/hbase/group/IntegrationTestGroup.java
b/hbase-it/src/test/java/org/apache/hadoop/hbase/group/IntegrationTestGroup.java
index 22cddd7..62f4f8a 100644
---
a/hbase-it/src/test/java/org/apache/hadoop/hbase/group/IntegrationTestGroup.java
+++
b/hbase-it/src/test/java/org/apache/hadoop/hbase/group/IntegrationTestGroup.java
@@ -74,11 +74,6 @@ public class IntegrationTestGroup extends TestGroupsBase {
((IntegrationTestingUtility)TEST_UTIL).restoreCluster();
LOG.info("Done restoring the cluster");
-groupAdmin.addGroup("master");
-groupAdmin.moveServers(
-
Sets.newHashSet(cluster.getInitialClusterStatus().getMaster().getHostPort()),
-"master");
-
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() {
@Override
public boolean evaluate() throws Exception {
http://git-wip-us.apache.org/repos/asf/hbase/blob/f1e6b202/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
--
diff --git
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
index beee110..71cc5d4 100644
---
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
+++
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
@@ -61124,1065 +61124,6 @@ public final class MasterProtos {
//
@@protoc_insertion_point(class_scope:hbase.pb.GetGroupInfoOfServerResponse)
}
- public interface ListServersInTransitionRequestOrBuilder
- extends com.google.protobuf.MessageOrBuilder {
- }
- /**
- * Protobuf type {@code hbase.pb.ListServersInTransitionRequest}
- */
- public static final class ListServersInTransitionRequest extends
- com.google.protobuf.GeneratedMessage
- implements ListServersInTransitionRequestOrBuilder {
-// Use ListServersInTransitionRequest.newBuilder() to construct.
-private
ListServersInTransitionRequest(com.google.protobuf.GeneratedMessage.Builder
builder) {
- super(builder);
- this.unknownFields = builder.getUnknownFields();
-}
-private ListServersInTransitionRequest(boolean noInit) {
this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
-
-private static final ListServersInTransitionRequest defaultInstance;
-public static ListServersInTransitionRequest getDefaultInstance() {
- return defaultInstance;
-}
-
-public ListServersInTransitionRequest getDefaultInstanceForType() {
- return defaultInstance;
-}
-
-private final com.google.protobuf.UnknownFieldSet unknownFields;
-@java.lang.Override
-public final com.google.protobuf.Unkno
[1/3] hbase git commit: HBASE-14312 Forward port some fixes from hbase-6721-0.98 to hbase-6721 (Francis Liu)
Repository: hbase
Updated Branches:
refs/heads/hbase-6721 fade887a3 -> a4821da1f (forced update)
http://git-wip-us.apache.org/repos/asf/hbase/blob/f1e6b202/hbase-server/src/test/java/org/apache/hadoop/hbase/group/TestGroups.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/group/TestGroups.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/group/TestGroups.java
index aa7ef1f..673c48e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/group/TestGroups.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/group/TestGroups.java
@@ -19,6 +19,7 @@
*/
package org.apache.hadoop.hbase.group;
+import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.commons.logging.Log;
@@ -34,6 +35,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter;
import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
@@ -46,15 +48,20 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
import javax.management.MBeanServer;
import javax.management.ObjectName;
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -62,8 +69,6 @@ import static org.junit.Assert.assertNotSame;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.verify;
@Category({MediumTests.class})
public class TestGroups extends TestGroupsBase {
@@ -139,12 +144,12 @@ public class TestGroups extends TestGroupsBase {
TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate() {
@Override
public boolean evaluate() throws Exception {
-LOG.info("Waiting for cleanup to finish "+groupAdmin.listGroups());
+LOG.info("Waiting for cleanup to finish " + groupAdmin.listGroups());
//Might be greater since moving servers back to default
//is after starting a server
return
groupAdmin.getGroupInfo(GroupInfo.DEFAULT_GROUP).getServers().size()
- == NUM_SLAVES_BASE;
+== NUM_SLAVES_BASE;
}
});
}
@@ -156,12 +161,37 @@ public class TestGroups extends TestGroupsBase {
//verify it was loaded properly
assertEquals("hadoop:name=Group,service=Group",
it.next().getCanonicalName());
-final MXBeanImpl info = MXBeanImpl.init(groupAdmin, master);
+final AtomicReference deadServer = new
AtomicReference(null);
+
+//We use mocks to simulate offline servers to avoid
+//the complexity and overhead of killing servers
+MasterServices mockMaster = Mockito.mock(MasterServices.class);
+final ServerManager mockServerManager = Mockito.mock(ServerManager.class);
+Mockito.when(mockMaster.getServerManager()).thenReturn(mockServerManager);
+Mockito.when(mockServerManager.getOnlineServersList()).then(new
Answer>() {
+ @Override
+ public List answer(InvocationOnMock invocation) throws
Throwable {
+GroupInfo groupInfo = groupAdmin.getGroupInfo(GroupInfo.DEFAULT_GROUP);
+List finalList = Lists.newArrayList();
+HostPort lastServer = groupInfo.getServers().last();
+for (ServerName server:
master.getServerManager().getOnlineServersList()) {
+ if (!server.getHostPort().equals(lastServer)) {
+finalList.add(server);
+ }
+}
+deadServer.set(lastServer);
+return finalList;
+ }
+});
+MXBean info = new MXBeanImpl(groupAdmin, mockMaster);
+
GroupInfo defaultGroup = groupAdmin.getGroupInfo(GroupInfo.DEFAULT_GROUP);
assertEquals(2, info.getGroups().size());
assertEquals(defaultGroup.getName(), info.getGroups().get(0).getName());
assertEquals(defaultGroup.getServers(),
Sets.newTreeSet(info.getGroups().get(0).getServers()));
-assertEquals(defaultGroup.getServers(),
Sets.newTreeSet(info.getServersByGroup().get(GroupInfo.DEFAULT_GROUP)));
+assertEquals(defaultGroup.getServers().headSet(deadServer.get()),
+
Sets.newTreeSet(info.getServersByGroup().get(GroupInfo.DEFAULT_GROUP)));
+
GroupInfo barGroup = a
[3/3] hbase git commit: HBASE-14232 Backwards compatiblity support for new MasterObserver APIs (Francis Liu)
HBASE-14232 Backwards compatiblity support for new MasterObserver APIs (Francis
Liu)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a4821da1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a4821da1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a4821da1
Branch: refs/heads/hbase-6721
Commit: a4821da1f2e50660a9dfee30583772dbe70da3c6
Parents: f1e6b20
Author: Andrew Purtell
Authored: Wed Aug 26 13:28:44 2015 -0700
Committer: Andrew Purtell
Committed: Wed Aug 26 13:28:44 2015 -0700
--
.../hbase/master/MasterCoprocessorHost.java | 43 +++-
1 file changed, 33 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a4821da1/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index 7a3b01f..05d2fb3 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -62,12 +62,15 @@ public class MasterCoprocessorHost
static class MasterEnvironment extends CoprocessorHost.Environment
implements MasterCoprocessorEnvironment {
private MasterServices masterServices;
+final boolean supportGroupCPs;
public MasterEnvironment(final Class implClass, final Coprocessor impl,
final int priority, final int seq, final Configuration conf,
final MasterServices services) {
super(impl, priority, seq, conf);
this.masterServices = services;
+ supportGroupCPs = !useLegacyMethod(impl.getClass(),
+ "preBalanceGroup", ObserverContext.class, String.class);
}
public MasterServices getMasterServices() {
@@ -1110,7 +1113,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserver oserver,
ObserverContext ctx) throws
IOException {
-oserver.preMoveServers(ctx, servers, targetGroup);
+if(((MasterEnvironment)ctx.getEnvironment()).supportGroupCPs) {
+ oserver.preMoveServers(ctx, servers, targetGroup);
+}
}
});
}
@@ -1121,7 +1126,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserver oserver,
ObserverContext ctx) throws
IOException {
-oserver.postMoveServers(ctx, servers, targetGroup);
+if(((MasterEnvironment)ctx.getEnvironment()).supportGroupCPs) {
+ oserver.postMoveServers(ctx, servers, targetGroup);
+}
}
});
}
@@ -1132,7 +1139,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserver oserver,
ObserverContext ctx) throws
IOException {
-oserver.preMoveTables(ctx, tables, targetGroup);
+if(((MasterEnvironment)ctx.getEnvironment()).supportGroupCPs) {
+ oserver.preMoveTables(ctx, tables, targetGroup);
+}
}
});
}
@@ -1143,7 +1152,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserver oserver,
ObserverContext ctx) throws
IOException {
-oserver.postMoveTables(ctx, tables, targetGroup);
+if(((MasterEnvironment)ctx.getEnvironment()).supportGroupCPs) {
+ oserver.postMoveTables(ctx, tables, targetGroup);
+}
}
});
}
@@ -1154,7 +1165,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserver oserver,
ObserverContext ctx) throws
IOException {
-oserver.preAddGroup(ctx, name);
+if(((MasterEnvironment)ctx.getEnvironment()).supportGroupCPs) {
+ oserver.preAddGroup(ctx, name);
+}
}
});
}
@@ -1165,7 +1178,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserver oserver,
ObserverContext ctx) throws
IOException {
-oserver.postAddGroup(ctx, name);
+if (((MasterEnvironment) ctx.getEnvironment()).supportGroupCPs) {
+ oserver.postAddGroup(ctx, name);
+}
}
});
}
@@ -1176,7 +1191,9 @@ public class MasterCoprocessorHost
@Override
public void call(MasterObserver oserver,
ObserverContext ctx) throws
IOException {
-oserver.preRemoveGroup(ctx, name);
+if(((MasterEnvironment)ctx.getEnvironment()).supportGroupCPs) {
+ oserver.preRemoveGroup(ctx, name);
+}
}
});
}
@@ -1187,7 +1204,9 @@ public class MasterCoprocessorHost
@Override
public void call
hbase git commit: HBASE-14232 Backwards compatiblity support for new MasterObserver APIs (Francis Liu)
Repository: hbase
Updated Branches:
refs/heads/hbase-6721-0.98 66e16163f -> 8726a60a5
HBASE-14232 Backwards compatiblity support for new MasterObserver APIs (Francis
Liu)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8726a60a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8726a60a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8726a60a
Branch: refs/heads/hbase-6721-0.98
Commit: 8726a60a52728730e46c46f4a6b9992530de2216
Parents: 66e1616
Author: Andrew Purtell
Authored: Wed Aug 26 13:28:44 2015 -0700
Committer: Andrew Purtell
Committed: Wed Aug 26 13:41:06 2015 -0700
--
.../hbase/coprocessor/CoprocessorHost.java | 74
.../hbase/master/MasterCoprocessorHost.java | 47 +
2 files changed, 109 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/8726a60a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
index 1c55738..16f6a86 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/CoprocessorHost.java
@@ -30,6 +30,7 @@ import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.UUID;
+import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicInteger;
@@ -870,4 +871,77 @@ public abstract class CoprocessorHost {
"coprocessor set.", e);
}
}
+
+ /**
+ * Used to gracefully handle fallback to deprecated methods when we
+ * evolve coprocessor APIs.
+ *
+ * When a particular Coprocessor API is updated to change methods, hosts can
support fallback
+ * to the deprecated API by using this method to determine if an instance
implements the new API.
+ * In the event that said support is partial, then in the face of a runtime
issue that prevents
+ * proper operation {@link #legacyWarning(Class, String)} should be used to
let operators know.
+ *
+ * For examples of this in action, see the implementation of
+ *
+ * {@link org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost}
+ * {@link org.apache.hadoop.hbase.regionserver.wal.WALCoprocessorHost}
+ *
+ *
+ * @param clazz Coprocessor you wish to evaluate
+ * @param methodName the name of the non-deprecated method version
+ * @param parameterTypes the Class of the non-deprecated method's arguments
in the order they are
+ * declared.
+ */
+ @InterfaceAudience.Private
+ protected static boolean useLegacyMethod(final Class
clazz,
+ final String methodName, final Class... parameterTypes) {
+boolean useLegacy;
+// Use reflection to see if they implement the non-deprecated version
+try {
+ clazz.getDeclaredMethod(methodName, parameterTypes);
+ LOG.debug("Found an implementation of '" + methodName + "' that uses
updated method " +
+ "signature. Skipping legacy support for invocations in '" + clazz
+"'.");
+ useLegacy = false;
+} catch (NoSuchMethodException exception) {
+ useLegacy = true;
+} catch (SecurityException exception) {
+ LOG.warn("The Security Manager denied our attempt to detect if the
coprocessor '" + clazz +
+ "' requires legacy support; assuming it does. If you get later
errors about legacy " +
+ "coprocessor use, consider updating your security policy to allow
access to the package" +
+ " and declared members of your implementation.");
+ LOG.debug("Details of Security Manager rejection.", exception);
+ useLegacy = true;
+}
+return useLegacy;
+ }
+
+ /**
+ * Used to limit legacy handling to once per Coprocessor class per
classloader.
+ */
+ private static final Set> legacyWarning =
+ new ConcurrentSkipListSet>(
+ new Comparator>() {
+@Override
+public int compare(Class c1, Class c2) {
+ if (c1.equals(c2)) {
+return 0;
+ }
+ return c1.getName().compareTo(c2.getName());
+}
+ });
+
+ /**
+ * limits the amount of logging to once per coprocessor class.
+ * Used in concert with {@link #useLegacyMethod(Class, String, Class[])}
when a runtime issue
+ * prevents properly supporting the legacy version of a coprocessor API.
+ * Since coprocessors can be in tight loops this serves to limit the amount
of log spam we create.
+ */
+ @InterfaceAudien
hbase git commit: HBASE-14313 After a Connection sees ConnectionClosingException on a connection it never recovers
Repository: hbase
Updated Branches:
refs/heads/branch-1.1 ae5439051 -> 0e4aabf99
HBASE-14313 After a Connection sees ConnectionClosingException on a connection
it never recovers
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0e4aabf9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0e4aabf9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0e4aabf9
Branch: refs/heads/branch-1.1
Commit: 0e4aabf99b341e8bfa6cd55ca4182e3e1dbd42cc
Parents: ae54390
Author: Elliott Clark
Authored: Tue Aug 25 18:39:31 2015 -0700
Committer: Elliott Clark
Committed: Wed Aug 26 13:44:49 2015 -0700
--
.../src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/0e4aabf9/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
index 4b96e31..a2d7172 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
@@ -935,7 +935,7 @@ public class RpcClientImpl extends AbstractRpcClient {
} catch (IOException e) {
// We set the value inside the synchronized block, this way the next
in line
// won't even try to write
- shouldCloseConnection.set(true);
+ markClosed(e);
writeException = e;
interrupt();
}
hbase git commit: HBASE-14313 After a Connection sees ConnectionClosingException on a connection it never recovers
Repository: hbase
Updated Branches:
refs/heads/branch-1.0 72b7f475c -> ea018af2e
HBASE-14313 After a Connection sees ConnectionClosingException on a connection
it never recovers
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ea018af2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ea018af2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ea018af2
Branch: refs/heads/branch-1.0
Commit: ea018af2ea1737291916240d054c5c7871bb57c0
Parents: 72b7f47
Author: Elliott Clark
Authored: Tue Aug 25 18:39:31 2015 -0700
Committer: Elliott Clark
Committed: Wed Aug 26 13:46:11 2015 -0700
--
.../src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ea018af2/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
index f4fd844..d399341 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
@@ -933,7 +933,7 @@ public class RpcClientImpl extends AbstractRpcClient {
} catch (IOException e) {
// We set the value inside the synchronized block, this way the next
in line
// won't even try to write
- shouldCloseConnection.set(true);
+ markClosed(e);
writeException = e;
interrupt();
}
hbase git commit: HBASE-14313 After a Connection sees ConnectionClosingException on a connection it never recovers
Repository: hbase
Updated Branches:
refs/heads/master aca8c3b74 -> 56890d9fe
HBASE-14313 After a Connection sees ConnectionClosingException on a connection
it never recovers
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/56890d9f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/56890d9f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/56890d9f
Branch: refs/heads/master
Commit: 56890d9fe148dd192520fab349a66aa3f688e232
Parents: aca8c3b
Author: Elliott Clark
Authored: Tue Aug 25 18:39:31 2015 -0700
Committer: Elliott Clark
Committed: Wed Aug 26 13:44:12 2015 -0700
--
.../src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/56890d9f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
index 5ece8ae..6647615 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
@@ -939,7 +939,7 @@ public class RpcClientImpl extends AbstractRpcClient {
} catch (IOException e) {
// We set the value inside the synchronized block, this way the next
in line
// won't even try to write
- shouldCloseConnection.set(true);
+ markClosed(e);
writeException = e;
interrupt();
}
hbase git commit: HBASE-14313 After a Connection sees ConnectionClosingException on a connection it never recovers
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 180e8b8fd -> 0a1f0cd66
HBASE-14313 After a Connection sees ConnectionClosingException on a connection
it never recovers
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0a1f0cd6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0a1f0cd6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0a1f0cd6
Branch: refs/heads/branch-1.2
Commit: 0a1f0cd66a0f090782726246d44d7e9c611abc68
Parents: 180e8b8
Author: Elliott Clark
Authored: Tue Aug 25 18:39:31 2015 -0700
Committer: Elliott Clark
Committed: Wed Aug 26 13:44:38 2015 -0700
--
.../src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/0a1f0cd6/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
index 5ece8ae..6647615 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClientImpl.java
@@ -939,7 +939,7 @@ public class RpcClientImpl extends AbstractRpcClient {
} catch (IOException e) {
// We set the value inside the synchronized block, this way the next
in line
// won't even try to write
- shouldCloseConnection.set(true);
+ markClosed(e);
writeException = e;
interrupt();
}
[3/4] hbase git commit: HBASE-13212 Procedure V2 - master Create/Modify/Delete namespace (Stephen Yuan Jiang)
http://git-wip-us.apache.org/repos/asf/hbase/blob/e4b96cf7/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProcedureProtos.java
--
diff --git
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProcedureProtos.java
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProcedureProtos.java
index 9bf4c98..d40c1f7 100644
---
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProcedureProtos.java
+++
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProcedureProtos.java
@@ -499,6 +499,315 @@ public final class MasterProcedureProtos {
}
/**
+ * Protobuf enum {@code hbase.pb.CreateNamespaceState}
+ */
+ public enum CreateNamespaceState
+ implements com.google.protobuf.ProtocolMessageEnum {
+/**
+ * CREATE_NAMESPACE_PREPARE = 1;
+ */
+CREATE_NAMESPACE_PREPARE(0, 1),
+/**
+ * CREATE_NAMESPACE_CREATE_DIRECTORY = 2;
+ */
+CREATE_NAMESPACE_CREATE_DIRECTORY(1, 2),
+/**
+ * CREATE_NAMESPACE_INSERT_INTO_NS_TABLE = 3;
+ */
+CREATE_NAMESPACE_INSERT_INTO_NS_TABLE(2, 3),
+/**
+ * CREATE_NAMESPACE_UPDATE_ZK = 4;
+ */
+CREATE_NAMESPACE_UPDATE_ZK(3, 4),
+/**
+ * CREATE_NAMESPACE_SET_NAMESPACE_QUOTA = 5;
+ */
+CREATE_NAMESPACE_SET_NAMESPACE_QUOTA(4, 5),
+;
+
+/**
+ * CREATE_NAMESPACE_PREPARE = 1;
+ */
+public static final int CREATE_NAMESPACE_PREPARE_VALUE = 1;
+/**
+ * CREATE_NAMESPACE_CREATE_DIRECTORY = 2;
+ */
+public static final int CREATE_NAMESPACE_CREATE_DIRECTORY_VALUE = 2;
+/**
+ * CREATE_NAMESPACE_INSERT_INTO_NS_TABLE = 3;
+ */
+public static final int CREATE_NAMESPACE_INSERT_INTO_NS_TABLE_VALUE = 3;
+/**
+ * CREATE_NAMESPACE_UPDATE_ZK = 4;
+ */
+public static final int CREATE_NAMESPACE_UPDATE_ZK_VALUE = 4;
+/**
+ * CREATE_NAMESPACE_SET_NAMESPACE_QUOTA = 5;
+ */
+public static final int CREATE_NAMESPACE_SET_NAMESPACE_QUOTA_VALUE = 5;
+
+
+public final int getNumber() { return value; }
+
+public static CreateNamespaceState valueOf(int value) {
+ switch (value) {
+case 1: return CREATE_NAMESPACE_PREPARE;
+case 2: return CREATE_NAMESPACE_CREATE_DIRECTORY;
+case 3: return CREATE_NAMESPACE_INSERT_INTO_NS_TABLE;
+case 4: return CREATE_NAMESPACE_UPDATE_ZK;
+case 5: return CREATE_NAMESPACE_SET_NAMESPACE_QUOTA;
+default: return null;
+ }
+}
+
+public static
com.google.protobuf.Internal.EnumLiteMap
+internalGetValueMap() {
+ return internalValueMap;
+}
+private static
com.google.protobuf.Internal.EnumLiteMap
+internalValueMap =
+ new com.google.protobuf.Internal.EnumLiteMap()
{
+public CreateNamespaceState findValueByNumber(int number) {
+ return CreateNamespaceState.valueOf(number);
+}
+ };
+
+public final com.google.protobuf.Descriptors.EnumValueDescriptor
+getValueDescriptor() {
+ return getDescriptor().getValues().get(index);
+}
+public final com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptorForType() {
+ return getDescriptor();
+}
+public static final com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptor() {
+ return
org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.getDescriptor().getEnumTypes().get(4);
+}
+
+private static final CreateNamespaceState[] VALUES = values();
+
+public static CreateNamespaceState valueOf(
+com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+ if (desc.getType() != getDescriptor()) {
+throw new java.lang.IllegalArgumentException(
+ "EnumValueDescriptor is not for this type.");
+ }
+ return VALUES[desc.getIndex()];
+}
+
+private final int index;
+private final int value;
+
+private CreateNamespaceState(int index, int value) {
+ this.index = index;
+ this.value = value;
+}
+
+// @@protoc_insertion_point(enum_scope:hbase.pb.CreateNamespaceState)
+ }
+
+ /**
+ * Protobuf enum {@code hbase.pb.ModifyNamespaceState}
+ */
+ public enum ModifyNamespaceState
+ implements com.google.protobuf.ProtocolMessageEnum {
+/**
+ * MODIFY_NAMESPACE_PREPARE = 1;
+ */
+MODIFY_NAMESPACE_PREPARE(0, 1),
+/**
+ * MODIFY_NAMESPACE_UPDATE_NS_TABLE = 2;
+ */
+MODIFY_NAMESPACE_UPDATE_NS_TABLE(1, 2),
+/**
+ * MODIFY_NAMESPACE_UPDATE_ZK = 3;
+ */
+MODIFY_NAMESPACE_UPDATE_ZK(2, 3),
+;
+
+/**
+ * MODIFY_NAMESPACE_PREPARE = 1;
+ */
+public static final int MODIFY_NAMESPACE_PREPARE_VALUE = 1;
+/**
+ * MODIFY_NAMESPACE_UPDATE_NS_TABLE = 2;
+ */
+public static final int MODIFY_NAMESPACE_UPDATE_NS_TABLE_VAL
[2/4] hbase git commit: HBASE-13212 Procedure V2 - master Create/Modify/Delete namespace (Stephen Yuan Jiang)
http://git-wip-us.apache.org/repos/asf/hbase/blob/e4b96cf7/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
--
diff --git
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
index f1420b0..4fbded7 100644
---
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
+++
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterProtos.java
@@ -16232,6 +16232,26 @@ public final class MasterProtos {
* required .hbase.pb.NamespaceDescriptor namespaceDescriptor =
1;
*/
org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NamespaceDescriptorOrBuilder
getNamespaceDescriptorOrBuilder();
+
+// optional uint64 nonce_group = 2 [default = 0];
+/**
+ * optional uint64 nonce_group = 2 [default = 0];
+ */
+boolean hasNonceGroup();
+/**
+ * optional uint64 nonce_group = 2 [default = 0];
+ */
+long getNonceGroup();
+
+// optional uint64 nonce = 3 [default = 0];
+/**
+ * optional uint64 nonce = 3 [default = 0];
+ */
+boolean hasNonce();
+/**
+ * optional uint64 nonce = 3 [default = 0];
+ */
+long getNonce();
}
/**
* Protobuf type {@code hbase.pb.CreateNamespaceRequest}
@@ -16297,6 +16317,16 @@ public final class MasterProtos {
bitField0_ |= 0x0001;
break;
}
+case 16: {
+ bitField0_ |= 0x0002;
+ nonceGroup_ = input.readUInt64();
+ break;
+}
+case 24: {
+ bitField0_ |= 0x0004;
+ nonce_ = input.readUInt64();
+ break;
+}
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -16359,8 +16389,42 @@ public final class MasterProtos {
return namespaceDescriptor_;
}
+// optional uint64 nonce_group = 2 [default = 0];
+public static final int NONCE_GROUP_FIELD_NUMBER = 2;
+private long nonceGroup_;
+/**
+ * optional uint64 nonce_group = 2 [default = 0];
+ */
+public boolean hasNonceGroup() {
+ return ((bitField0_ & 0x0002) == 0x0002);
+}
+/**
+ * optional uint64 nonce_group = 2 [default = 0];
+ */
+public long getNonceGroup() {
+ return nonceGroup_;
+}
+
+// optional uint64 nonce = 3 [default = 0];
+public static final int NONCE_FIELD_NUMBER = 3;
+private long nonce_;
+/**
+ * optional uint64 nonce = 3 [default = 0];
+ */
+public boolean hasNonce() {
+ return ((bitField0_ & 0x0004) == 0x0004);
+}
+/**
+ * optional uint64 nonce = 3 [default = 0];
+ */
+public long getNonce() {
+ return nonce_;
+}
+
private void initFields() {
namespaceDescriptor_ =
org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NamespaceDescriptor.getDefaultInstance();
+ nonceGroup_ = 0L;
+ nonce_ = 0L;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -16385,6 +16449,12 @@ public final class MasterProtos {
if (((bitField0_ & 0x0001) == 0x0001)) {
output.writeMessage(1, namespaceDescriptor_);
}
+ if (((bitField0_ & 0x0002) == 0x0002)) {
+output.writeUInt64(2, nonceGroup_);
+ }
+ if (((bitField0_ & 0x0004) == 0x0004)) {
+output.writeUInt64(3, nonce_);
+ }
getUnknownFields().writeTo(output);
}
@@ -16398,6 +16468,14 @@ public final class MasterProtos {
size += com.google.protobuf.CodedOutputStream
.computeMessageSize(1, namespaceDescriptor_);
}
+ if (((bitField0_ & 0x0002) == 0x0002)) {
+size += com.google.protobuf.CodedOutputStream
+ .computeUInt64Size(2, nonceGroup_);
+ }
+ if (((bitField0_ & 0x0004) == 0x0004)) {
+size += com.google.protobuf.CodedOutputStream
+ .computeUInt64Size(3, nonce_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -16426,6 +16504,16 @@ public final class MasterProtos {
result = result && getNamespaceDescriptor()
.equals(other.getNamespaceDescriptor());
}
+ result = result && (hasNonceGroup() == other.hasNonceGroup());
+ if (hasNonceGroup()) {
+result = result && (getNonceGroup()
+== other.getNonceGroup());
+ }
+ result = result && (hasNonce() == other.hasNonce());
+ if (hasNonce()) {
+result = result && (getNonce()
+== other.getNonce());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
re
[4/4] hbase git commit: HBASE-13212 Procedure V2 - master Create/Modify/Delete namespace (Stephen Yuan Jiang)
HBASE-13212 Procedure V2 - master Create/Modify/Delete namespace (Stephen Yuan Jiang) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e4b96cf7 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e4b96cf7 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e4b96cf7 Branch: refs/heads/branch-1 Commit: e4b96cf78a4e7b15a1bc46a4f060f65065732786 Parents: 9b55f1c Author: Stephen Yuan Jiang Authored: Wed Aug 26 13:54:50 2015 -0700 Committer: Stephen Yuan Jiang Committed: Wed Aug 26 13:54:50 2015 -0700 -- .../generated/MasterProcedureProtos.java| 2910 -- .../hbase/protobuf/generated/MasterProtos.java | 989 -- hbase-protocol/src/main/protobuf/Master.proto |6 + .../src/main/protobuf/MasterProcedure.proto | 36 + .../apache/hadoop/hbase/ZKNamespaceManager.java |9 +- .../org/apache/hadoop/hbase/master/HMaster.java | 56 +- .../hadoop/hbase/master/MasterRpcServices.java | 14 +- .../hadoop/hbase/master/MasterServices.java | 38 +- .../hbase/master/TableNamespaceManager.java | 232 +- .../procedure/CreateNamespaceProcedure.java | 364 +++ .../procedure/DeleteNamespaceProcedure.java | 398 +++ .../procedure/ModifyNamespaceProcedure.java | 281 ++ .../hadoop/hbase/master/TestCatalogJanitor.java | 28 +- .../MasterProcedureTestingUtility.java |2 - .../procedure/TestCreateNamespaceProcedure.java | 291 ++ .../procedure/TestDeleteNamespaceProcedure.java | 281 ++ .../procedure/TestModifyNamespaceProcedure.java | 294 ++ 17 files changed, 5655 insertions(+), 574 deletions(-) --
[1/4] hbase git commit: HBASE-13212 Procedure V2 - master Create/Modify/Delete namespace (Stephen Yuan Jiang)
Repository: hbase
Updated Branches:
refs/heads/branch-1 9b55f1cd2 -> e4b96cf78
http://git-wip-us.apache.org/repos/asf/hbase/blob/e4b96cf7/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
new file mode 100644
index 000..c91092a
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/CreateNamespaceProcedure.java
@@ -0,0 +1,364 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.master.procedure;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.NamespaceExistException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.TableNamespaceManager;
+import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
+import
org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.CreateNamespaceState;
+import org.apache.hadoop.hbase.util.FSUtils;
+
+/**
+ * The procedure to create a new namespace.
+ */
+@InterfaceAudience.Private
+public class CreateNamespaceProcedure
+extends StateMachineProcedure
+implements TableProcedureInterface {
+ private static final Log LOG =
LogFactory.getLog(CreateNamespaceProcedure.class);
+
+ private final AtomicBoolean aborted = new AtomicBoolean(false);
+
+ private NamespaceDescriptor nsDescriptor;
+ private Boolean traceEnabled;
+
+ public CreateNamespaceProcedure() {
+this.traceEnabled = null;
+ }
+
+ public CreateNamespaceProcedure(
+ final MasterProcedureEnv env,
+ final NamespaceDescriptor nsDescriptor) throws IOException {
+this.nsDescriptor = nsDescriptor;
+this.traceEnabled = null;
+ }
+
+ @Override
+ protected Flow executeFromState(final MasterProcedureEnv env, final
CreateNamespaceState state)
+ throws InterruptedException {
+if (isTraceEnabled()) {
+ LOG.trace(this + " execute state=" + state);
+}
+
+try {
+ switch (state) {
+ case CREATE_NAMESPACE_PREPARE:
+prepareCreate(env);
+setNextState(CreateNamespaceState.CREATE_NAMESPACE_CREATE_DIRECTORY);
+break;
+ case CREATE_NAMESPACE_CREATE_DIRECTORY:
+createDirectory(env, nsDescriptor);
+
setNextState(CreateNamespaceState.CREATE_NAMESPACE_INSERT_INTO_NS_TABLE);
+break;
+ case CREATE_NAMESPACE_INSERT_INTO_NS_TABLE:
+insertIntoNSTable(env, nsDescriptor);
+setNextState(CreateNamespaceState.CREATE_NAMESPACE_UPDATE_ZK);
+break;
+ case CREATE_NAMESPACE_UPDATE_ZK:
+updateZKNamespaceManager(env, nsDescriptor);
+
setNextState(CreateNamespaceState.CREATE_NAMESPACE_SET_NAMESPACE_QUOTA);
+break;
+ case CREATE_NAMESPACE_SET_NAMESPACE_QUOTA:
+setNamespaceQuota(env, nsDescriptor);
+return Flow.NO_MORE_STATE;
+ default:
+throw new UnsupportedOperationException(this + " unhandled state=" +
state);
+ }
+} catch (IOException e) {
+ LOG.warn("Error trying to create the namespace" + nsDescriptor.getName()
++ " (in state=" + state + ")", e);
+
+ setFailure("master-create-namespace", e);
+}
+return Flow.HAS_MORE_STATE;
+ }
+
+ @Override
+ protected void rollbackState(final MasterProcedureEnv env, final
CreateNamespaceState state)
+ throws IOException {
+if (isTraceEnabled()) {
+ LOG.trace(this + " rollback state=" + state);
+}
+try {
+ switch (state) {
+ cas
hbase git commit: HBASE-14269 FuzzyRowFilter omits certain rows when multiple fuzzy keys exist (hongbin ma)
Repository: hbase
Updated Branches:
refs/heads/branch-1 e4b96cf78 -> b8c7a08a9
HBASE-14269 FuzzyRowFilter omits certain rows when multiple fuzzy keys exist
(hongbin ma)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b8c7a08a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b8c7a08a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b8c7a08a
Branch: refs/heads/branch-1
Commit: b8c7a08a9ed9a8c99e0de31512966065c4050e99
Parents: e4b96cf
Author: tedyu
Authored: Wed Aug 26 20:49:43 2015 -0700
Committer: tedyu
Committed: Wed Aug 26 20:49:43 2015 -0700
--
.../hadoop/hbase/filter/FuzzyRowFilter.java | 120 +--
.../filter/TestFuzzyRowFilterEndToEnd.java | 97 ++-
2 files changed, 129 insertions(+), 88 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b8c7a08a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
index f112b2e..1f125e5 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
@@ -19,9 +19,9 @@ package org.apache.hadoop.hbase.filter;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.PriorityQueue;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValueUtil;
@@ -94,7 +94,7 @@ public class FuzzyRowFilter extends FilterBase {
private void preprocessSearchKey(Pair p) {
if (UnsafeAccess.isAvailable() == false) {
- return;
+ return;
}
byte[] key = p.getFirst();
byte[] mask = p.getSecond();
@@ -157,83 +157,83 @@ public class FuzzyRowFilter extends FilterBase {
@Override
public Cell getNextCellHint(Cell currentCell) {
-boolean result = true;
-if (tracker.needsUpdate()) {
- result = tracker.updateTracker(currentCell);
-}
+boolean result = tracker.updateTracker(currentCell);
if (result == false) {
done = true;
return null;
}
byte[] nextRowKey = tracker.nextRow();
-// We need to compare nextRowKey with currentCell
-int compareResult =
-Bytes.compareTo(nextRowKey, 0, nextRowKey.length,
currentCell.getRowArray(),
- currentCell.getRowOffset(), currentCell.getRowLength());
-if ((reversed && compareResult > 0) || (!reversed && compareResult < 0)) {
- // This can happen when we have multilpe filters and some other filter
- // returns next row with hint which is larger (smaller for reverse)
- // than the current (really?)
- result = tracker.updateTracker(currentCell);
- if (result == false) {
-done = true;
-return null;
- } else {
-nextRowKey = tracker.nextRow();
- }
-}
return KeyValueUtil.createFirstOnRow(nextRowKey);
}
/**
- * If we have multiple fuzzy keys, row tracker should improve overall
performance It calculates
- * all next rows (one per every fuzzy key), sort them accordingly (ascending
for regular and
- * descending for reverse). Next time getNextCellHint is called we check row
tracker first and
- * return next row from the tracker if it exists, if there are no rows in
the tracker we update
- * tracker with a current cell and return first row.
+ * If we have multiple fuzzy keys, row tracker should improve overall
performance. It calculates
+ * all next rows (one per every fuzzy key) and put them (the fuzzy key is
bundled) into a priority
+ * queue so that the smallest row key always appears at queue head, which
helps to decide the
+ * "Next Cell Hint". As scanning going on, the number of candidate rows in
the RowTracker will
+ * remain the size of fuzzy keys until some of the fuzzy keys won't possibly
have matches any
+ * more.
*/
private class RowTracker {
-private final List nextRows;
-private int next = -1;
+private final PriorityQueue>> nextRows;
+private boolean initialized = false;
RowTracker() {
- nextRows = new ArrayList();
-}
-
-boolean needsUpdate() {
- return next == -1 || next == nextRows.size();
+ nextRows =
+ new PriorityQueue>>(fuzzyKeysData.size(),
+ new Comparator>>() {
+@Override
+public int compare(Pair> o1,
+Pair> o2) {
+ int compare = Bytes.compareTo(o1.getFirst(), o2.getFirst());
+ if (!isReversed()) {
+
hbase git commit: HBASE-14269 FuzzyRowFilter omits certain rows when multiple fuzzy keys exist (hongbin ma)
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 0a1f0cd66 -> 430f31ff4
HBASE-14269 FuzzyRowFilter omits certain rows when multiple fuzzy keys exist
(hongbin ma)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/430f31ff
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/430f31ff
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/430f31ff
Branch: refs/heads/branch-1.2
Commit: 430f31ff4e6a45562a91e997adbfe2c95f8df4a7
Parents: 0a1f0cd
Author: tedyu
Authored: Wed Aug 26 20:53:24 2015 -0700
Committer: tedyu
Committed: Wed Aug 26 20:53:24 2015 -0700
--
.../hadoop/hbase/filter/FuzzyRowFilter.java | 120 +--
.../filter/TestFuzzyRowFilterEndToEnd.java | 97 ++-
2 files changed, 129 insertions(+), 88 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/430f31ff/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
index f112b2e..1f125e5 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
@@ -19,9 +19,9 @@ package org.apache.hadoop.hbase.filter;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.PriorityQueue;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValueUtil;
@@ -94,7 +94,7 @@ public class FuzzyRowFilter extends FilterBase {
private void preprocessSearchKey(Pair p) {
if (UnsafeAccess.isAvailable() == false) {
- return;
+ return;
}
byte[] key = p.getFirst();
byte[] mask = p.getSecond();
@@ -157,83 +157,83 @@ public class FuzzyRowFilter extends FilterBase {
@Override
public Cell getNextCellHint(Cell currentCell) {
-boolean result = true;
-if (tracker.needsUpdate()) {
- result = tracker.updateTracker(currentCell);
-}
+boolean result = tracker.updateTracker(currentCell);
if (result == false) {
done = true;
return null;
}
byte[] nextRowKey = tracker.nextRow();
-// We need to compare nextRowKey with currentCell
-int compareResult =
-Bytes.compareTo(nextRowKey, 0, nextRowKey.length,
currentCell.getRowArray(),
- currentCell.getRowOffset(), currentCell.getRowLength());
-if ((reversed && compareResult > 0) || (!reversed && compareResult < 0)) {
- // This can happen when we have multilpe filters and some other filter
- // returns next row with hint which is larger (smaller for reverse)
- // than the current (really?)
- result = tracker.updateTracker(currentCell);
- if (result == false) {
-done = true;
-return null;
- } else {
-nextRowKey = tracker.nextRow();
- }
-}
return KeyValueUtil.createFirstOnRow(nextRowKey);
}
/**
- * If we have multiple fuzzy keys, row tracker should improve overall
performance It calculates
- * all next rows (one per every fuzzy key), sort them accordingly (ascending
for regular and
- * descending for reverse). Next time getNextCellHint is called we check row
tracker first and
- * return next row from the tracker if it exists, if there are no rows in
the tracker we update
- * tracker with a current cell and return first row.
+ * If we have multiple fuzzy keys, row tracker should improve overall
performance. It calculates
+ * all next rows (one per every fuzzy key) and put them (the fuzzy key is
bundled) into a priority
+ * queue so that the smallest row key always appears at queue head, which
helps to decide the
+ * "Next Cell Hint". As scanning going on, the number of candidate rows in
the RowTracker will
+ * remain the size of fuzzy keys until some of the fuzzy keys won't possibly
have matches any
+ * more.
*/
private class RowTracker {
-private final List nextRows;
-private int next = -1;
+private final PriorityQueue>> nextRows;
+private boolean initialized = false;
RowTracker() {
- nextRows = new ArrayList();
-}
-
-boolean needsUpdate() {
- return next == -1 || next == nextRows.size();
+ nextRows =
+ new PriorityQueue>>(fuzzyKeysData.size(),
+ new Comparator>>() {
+@Override
+public int compare(Pair> o1,
+Pair> o2) {
+ int compare = Bytes.compareTo(o1.getFirst(), o2.getFirst());
+ if (!isReversed()) {
+
hbase git commit: HBASE-14269 FuzzyRowFilter omits certain rows when multiple fuzzy keys exist (hongbin ma)
Repository: hbase
Updated Branches:
refs/heads/branch-1.1 0e4aabf99 -> 4b5c4e44b
HBASE-14269 FuzzyRowFilter omits certain rows when multiple fuzzy keys exist
(hongbin ma)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4b5c4e44
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4b5c4e44
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4b5c4e44
Branch: refs/heads/branch-1.1
Commit: 4b5c4e44b2367d49a71f872828bfc0bf0e8c43db
Parents: 0e4aabf
Author: tedyu
Authored: Wed Aug 26 20:54:26 2015 -0700
Committer: tedyu
Committed: Wed Aug 26 20:54:26 2015 -0700
--
.../hadoop/hbase/filter/FuzzyRowFilter.java | 120 +--
.../filter/TestFuzzyRowFilterEndToEnd.java | 97 ++-
2 files changed, 129 insertions(+), 88 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/4b5c4e44/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
index f112b2e..1f125e5 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
@@ -19,9 +19,9 @@ package org.apache.hadoop.hbase.filter;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.PriorityQueue;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValueUtil;
@@ -94,7 +94,7 @@ public class FuzzyRowFilter extends FilterBase {
private void preprocessSearchKey(Pair p) {
if (UnsafeAccess.isAvailable() == false) {
- return;
+ return;
}
byte[] key = p.getFirst();
byte[] mask = p.getSecond();
@@ -157,83 +157,83 @@ public class FuzzyRowFilter extends FilterBase {
@Override
public Cell getNextCellHint(Cell currentCell) {
-boolean result = true;
-if (tracker.needsUpdate()) {
- result = tracker.updateTracker(currentCell);
-}
+boolean result = tracker.updateTracker(currentCell);
if (result == false) {
done = true;
return null;
}
byte[] nextRowKey = tracker.nextRow();
-// We need to compare nextRowKey with currentCell
-int compareResult =
-Bytes.compareTo(nextRowKey, 0, nextRowKey.length,
currentCell.getRowArray(),
- currentCell.getRowOffset(), currentCell.getRowLength());
-if ((reversed && compareResult > 0) || (!reversed && compareResult < 0)) {
- // This can happen when we have multilpe filters and some other filter
- // returns next row with hint which is larger (smaller for reverse)
- // than the current (really?)
- result = tracker.updateTracker(currentCell);
- if (result == false) {
-done = true;
-return null;
- } else {
-nextRowKey = tracker.nextRow();
- }
-}
return KeyValueUtil.createFirstOnRow(nextRowKey);
}
/**
- * If we have multiple fuzzy keys, row tracker should improve overall
performance It calculates
- * all next rows (one per every fuzzy key), sort them accordingly (ascending
for regular and
- * descending for reverse). Next time getNextCellHint is called we check row
tracker first and
- * return next row from the tracker if it exists, if there are no rows in
the tracker we update
- * tracker with a current cell and return first row.
+ * If we have multiple fuzzy keys, row tracker should improve overall
performance. It calculates
+ * all next rows (one per every fuzzy key) and put them (the fuzzy key is
bundled) into a priority
+ * queue so that the smallest row key always appears at queue head, which
helps to decide the
+ * "Next Cell Hint". As scanning going on, the number of candidate rows in
the RowTracker will
+ * remain the size of fuzzy keys until some of the fuzzy keys won't possibly
have matches any
+ * more.
*/
private class RowTracker {
-private final List nextRows;
-private int next = -1;
+private final PriorityQueue>> nextRows;
+private boolean initialized = false;
RowTracker() {
- nextRows = new ArrayList();
-}
-
-boolean needsUpdate() {
- return next == -1 || next == nextRows.size();
+ nextRows =
+ new PriorityQueue>>(fuzzyKeysData.size(),
+ new Comparator>>() {
+@Override
+public int compare(Pair> o1,
+Pair> o2) {
+ int compare = Bytes.compareTo(o1.getFirst(), o2.getFirst());
+ if (!isReversed()) {
+
hbase git commit: HBASE-14269 FuzzyRowFilter omits certain rows when multiple fuzzy keys exist (hongbin ma)
Repository: hbase
Updated Branches:
refs/heads/branch-1.0 ea018af2e -> e7c7570d5
HBASE-14269 FuzzyRowFilter omits certain rows when multiple fuzzy keys exist
(hongbin ma)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e7c7570d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e7c7570d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e7c7570d
Branch: refs/heads/branch-1.0
Commit: e7c7570d5adf1b094d7fc7339e30291f75366b54
Parents: ea018af
Author: tedyu
Authored: Wed Aug 26 20:55:14 2015 -0700
Committer: tedyu
Committed: Wed Aug 26 20:55:14 2015 -0700
--
.../hadoop/hbase/filter/FuzzyRowFilter.java | 120 +--
.../filter/TestFuzzyRowFilterEndToEnd.java | 97 ++-
2 files changed, 129 insertions(+), 88 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/e7c7570d/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
index f112b2e..1f125e5 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
@@ -19,9 +19,9 @@ package org.apache.hadoop.hbase.filter;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.PriorityQueue;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValueUtil;
@@ -94,7 +94,7 @@ public class FuzzyRowFilter extends FilterBase {
private void preprocessSearchKey(Pair p) {
if (UnsafeAccess.isAvailable() == false) {
- return;
+ return;
}
byte[] key = p.getFirst();
byte[] mask = p.getSecond();
@@ -157,83 +157,83 @@ public class FuzzyRowFilter extends FilterBase {
@Override
public Cell getNextCellHint(Cell currentCell) {
-boolean result = true;
-if (tracker.needsUpdate()) {
- result = tracker.updateTracker(currentCell);
-}
+boolean result = tracker.updateTracker(currentCell);
if (result == false) {
done = true;
return null;
}
byte[] nextRowKey = tracker.nextRow();
-// We need to compare nextRowKey with currentCell
-int compareResult =
-Bytes.compareTo(nextRowKey, 0, nextRowKey.length,
currentCell.getRowArray(),
- currentCell.getRowOffset(), currentCell.getRowLength());
-if ((reversed && compareResult > 0) || (!reversed && compareResult < 0)) {
- // This can happen when we have multilpe filters and some other filter
- // returns next row with hint which is larger (smaller for reverse)
- // than the current (really?)
- result = tracker.updateTracker(currentCell);
- if (result == false) {
-done = true;
-return null;
- } else {
-nextRowKey = tracker.nextRow();
- }
-}
return KeyValueUtil.createFirstOnRow(nextRowKey);
}
/**
- * If we have multiple fuzzy keys, row tracker should improve overall
performance It calculates
- * all next rows (one per every fuzzy key), sort them accordingly (ascending
for regular and
- * descending for reverse). Next time getNextCellHint is called we check row
tracker first and
- * return next row from the tracker if it exists, if there are no rows in
the tracker we update
- * tracker with a current cell and return first row.
+ * If we have multiple fuzzy keys, row tracker should improve overall
performance. It calculates
+ * all next rows (one per every fuzzy key) and put them (the fuzzy key is
bundled) into a priority
+ * queue so that the smallest row key always appears at queue head, which
helps to decide the
+ * "Next Cell Hint". As scanning going on, the number of candidate rows in
the RowTracker will
+ * remain the size of fuzzy keys until some of the fuzzy keys won't possibly
have matches any
+ * more.
*/
private class RowTracker {
-private final List nextRows;
-private int next = -1;
+private final PriorityQueue>> nextRows;
+private boolean initialized = false;
RowTracker() {
- nextRows = new ArrayList();
-}
-
-boolean needsUpdate() {
- return next == -1 || next == nextRows.size();
+ nextRows =
+ new PriorityQueue>>(fuzzyKeysData.size(),
+ new Comparator>>() {
+@Override
+public int compare(Pair> o1,
+Pair> o2) {
+ int compare = Bytes.compareTo(o1.getFirst(), o2.getFirst());
+ if (!isReversed()) {
+
svn commit: r10313 - in /dev/hbase/hbase-1.1.2RC2: ./ hbase-1.1.2-bin.tar.gz hbase-1.1.2-bin.tar.gz.asc hbase-1.1.2-bin.tar.gz.mds hbase-1.1.2-src.tar.gz hbase-1.1.2-src.tar.gz.asc hbase-1.1.2-src.tar
Author: ndimiduk Date: Thu Aug 27 03:57:20 2015 New Revision: 10313 Log: HBase-1.1.2 RC2 artifacts Added: dev/hbase/hbase-1.1.2RC2/ dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-bin.tar.gz (with props) dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-bin.tar.gz.asc dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-bin.tar.gz.mds dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-src.tar.gz (with props) dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-src.tar.gz.asc dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-src.tar.gz.mds Added: dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-bin.tar.gz == Binary file - no diff available. Propchange: dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-bin.tar.gz -- svn:mime-type = application/octet-stream Added: dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-bin.tar.gz.asc == --- dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-bin.tar.gz.asc (added) +++ dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-bin.tar.gz.asc Thu Aug 27 03:57:20 2015 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIcBAABCgAGBQJV3ogmAAoJEK2QOQccNIm9EdcP/A8FaTKrVhOF6hua4jxXbKww +G8oObYexDttnm18G9JAcGRgxtR7UWvztyMr8nMwhM9igJur2MWWegG5GU+rtRjMu +9qONtvUR+9w6bqb0niqdIgMcB1LRa9Sa1fN7OVodvJYHlrGEKWYyy9igKgvIGYT1 +fMmO8hRjwelKKISzpYtOKYPeBaZhu60JRzcv3Tc0NGJfgLe5TSWCFflw6kNH7pWP ++KF7vAT55E+buEfyRC8R4pJwYk67HWb4nN5UKHhJkbUm8p7OXF2JtDrUugCu915B +Kh67K96+aOk6no8LK2qSN7HG7ViAU9ReQxtxiD32kiF2dehlUoDXgzC4kvoABg5y +TF06rB/ePQEZw2TWseztCFJlf0gWjvLXEJ47qZppjlqGoSH1GKM3A9PIh9hxfkct +4yEfI/YnGohl1MF0dINWnk5TcZHLghKGEJZtspotNcYx7ZNctB/9Np7ZYIBlYLWt +1HC+BcPWo9dL8plnA4+03ij5CwQsukbmwV1yiv4jNSAmr1WviXq+lN33AGGenrGZ +6oCVsqzXCYEdP1C9rsAVSOA4notMBhnlNQxjcs4PazwZzCwpAJNd4DjumF+yG20T +Ayp3j36m1iAMfJK3iMParrdjjVfqpPvhuJQ8lG2gCmRkHj1EKbkozyCMr+4xoLAM +gaDxiZHJWCBjU64CnkCk +=O4X/ +-END PGP SIGNATURE- Added: dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-bin.tar.gz.mds == --- dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-bin.tar.gz.mds (added) +++ dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-bin.tar.gz.mds Thu Aug 27 03:57:20 2015 @@ -0,0 +1,17 @@ +hbase-1.1.2-bin.tar.gz:MD5 = 9F 8C A7 EC 80 6F CB BA 33 AC F1 C5 61 5B 44 + 15 +hbase-1.1.2-bin.tar.gz: SHA1 = DC62 A7BB 102C B5C7 096E 74FE 8826 99A2 DDDE + A8A4 +hbase-1.1.2-bin.tar.gz: RMD160 = 1DB0 167A E9C3 10F9 98EB C165 E6C2 522C 1C12 + 1A3D +hbase-1.1.2-bin.tar.gz: SHA224 = 4E553467 50D54477 1A08CF9C E4FB8D60 255A8997 + 7A60CDC0 E778B337 +hbase-1.1.2-bin.tar.gz: SHA256 = 8CA5BF02 03CEF86B 4A0ACBBA 89AFCD59 77488EBC + 73EEC097 E93C592B 16F8BEDE +hbase-1.1.2-bin.tar.gz: SHA384 = 5B0C4354 68AA74CC 57286F6F 13093E86 E304FB5E + 9D3D21DE A6C66525 BA6F825D 060F338C D6255F72 + 8E719CC1 B1FA366B +hbase-1.1.2-bin.tar.gz: SHA512 = 1348B258 A19EFA6B 0D3832F4 D2457750 CD70B5DF + F17DC472 20B46CDE BE720B79 34F4C2EA 1B75B701 + 2ED90F55 2D4D56F7 0EA125B0 A1CADC9B CD2C17E1 + D528A6D4 Added: dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-src.tar.gz == Binary file - no diff available. Propchange: dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-src.tar.gz -- svn:mime-type = application/octet-stream Added: dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-src.tar.gz.asc == --- dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-src.tar.gz.asc (added) +++ dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-src.tar.gz.asc Thu Aug 27 03:57:20 2015 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIcBAABCgAGBQJV3ogrAAoJEK2QOQccNIm9dVgP/3jItpvuhg39ZBTlR9cQoa4y +7rgqfFXOKAzoHgsbyXhVi6PiWLEiejzfkJf3lbYbudBaqd45/DJz4jFtEfWBuUxL +3qxBGFn0ybIESCWlWJM2CKbWtp/Ge+3abmodsWhkkqnc1xVCFC9iOVQ5p5NLwMTc +Zi0vPR7oi1ubkHcqtG4jh6lqlNw3vL0k5LUsaOl3G6mlUSHXG+qZzb+ahRqNZd/8 +72OjlzfUko8bldfEX6cugGdVkwKz+gB9jeWFQYcg30EafXjmMzpzuoPBeIA5WQg4 +mT2oHkmyUS2HlDDxDUaOlieR3kvLW3xV4ksofyDNv6Rap83/N3Quv6P3ZRFC4FrO +FrZzqNc9iN3LceIEI0GJvGUV6lrv7fDnMNAB8mYbw/fv+kc7itkNZjXkQpa+z1Vf +cXywULlUOJoVp8surW++C7TRIJ7+qkzjmg4ee6g4euJnkDOUoBddBUBIIZneI+24 +uOlRda06X9nIn3l6JtgTJM2fAEVEVX04zwoapyvgIjhq2/dj76J+krUMZsb7Dn09 +tJKqtgRvud9W6A1vqBUkZOPcEWMnAYCNImxxmxU//i+h+ZdgP1TIvNOWzRwvjDJ6 +R6SfWZNqRN+psMnPqG2+1kk2Gma3QPG+7rpTfO6f2AeGoRFa81nPfdN4MAgZAm8C +opKRJKhZBy02B7IhVIzz +=Jepv +-END PGP SIGNATURE- Added: dev/hbase/hbase-1.1.2RC2/hbase-1.1.2-src.tar.gz.mds == --- d
Git Push Summary
Repository: hbase Updated Tags: refs/tags/1.1.2RC2 [created] b54078ddc
hbase git commit: HBASE-14269 FuzzyRowFilter omits certain rows when multiple fuzzy keys exist (hongbin ma)
Repository: hbase
Updated Branches:
refs/heads/0.98 4e4aabb93 -> 3fe903cb7
HBASE-14269 FuzzyRowFilter omits certain rows when multiple fuzzy keys exist
(hongbin ma)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3fe903cb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3fe903cb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3fe903cb
Branch: refs/heads/0.98
Commit: 3fe903cb77c09eb6905dc235b7e7109e3135bbc6
Parents: 4e4aabb
Author: tedyu
Authored: Wed Aug 26 21:02:51 2015 -0700
Committer: tedyu
Committed: Wed Aug 26 21:02:51 2015 -0700
--
.../hadoop/hbase/filter/FuzzyRowFilter.java | 118 +--
.../hadoop/hbase/filter/TestFuzzyRowFilter.java | 1 -
.../filter/TestFuzzyRowFilterEndToEnd.java | 94 ++-
3 files changed, 126 insertions(+), 87 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/3fe903cb/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
index 7a3f1c7..d6ad3a0 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/FuzzyRowFilter.java
@@ -19,9 +19,9 @@ package org.apache.hadoop.hbase.filter;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.Comparator;
import java.util.List;
+import java.util.PriorityQueue;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
@@ -157,83 +157,83 @@ public class FuzzyRowFilter extends FilterBase {
@Override
public Cell getNextCellHint(Cell currentCell) {
-boolean result = true;
-if (tracker.needsUpdate()) {
- result = tracker.updateTracker(currentCell);
-}
+boolean result = tracker.updateTracker(currentCell);
if (result == false) {
done = true;
return null;
}
byte[] nextRowKey = tracker.nextRow();
-// We need to compare nextRowKey with currentCell
-int compareResult =
-Bytes.compareTo(nextRowKey, 0, nextRowKey.length,
currentCell.getRowArray(),
- currentCell.getRowOffset(), currentCell.getRowLength());
-if ((reversed && compareResult > 0) || (!reversed && compareResult < 0)) {
- // This can happen when we have multilpe filters and some other filter
- // returns next row with hint which is larger (smaller for reverse)
- // than the current (really?)
- result = tracker.updateTracker(currentCell);
- if (result == false) {
-done = true;
-return null;
- } else {
-nextRowKey = tracker.nextRow();
- }
-}
return KeyValue.createFirstOnRow(nextRowKey);
}
/**
- * If we have multiple fuzzy keys, row tracker should improve overall
performance It calculates
- * all next rows (one per every fuzzy key), sort them accordingly (ascending
for regular and
- * descending for reverse). Next time getNextCellHint is called we check row
tracker first and
- * return next row from the tracker if it exists, if there are no rows in
the tracker we update
- * tracker with a current cell and return first row.
+ * If we have multiple fuzzy keys, row tracker should improve overall
performance. It calculates
+ * all next rows (one per every fuzzy key) and put them (the fuzzy key is
bundled) into a priority
+ * queue so that the smallest row key always appears at queue head, which
helps to decide the
+ * "Next Cell Hint". As scanning going on, the number of candidate rows in
the RowTracker will
+ * remain the size of fuzzy keys until some of the fuzzy keys won't possibly
have matches any
+ * more.
*/
private class RowTracker {
-private final List nextRows;
-private int next = -1;
+private final PriorityQueue>> nextRows;
+private boolean initialized = false;
RowTracker() {
- nextRows = new ArrayList();
-}
-
-boolean needsUpdate() {
- return next == -1 || next == nextRows.size();
+ nextRows =
+ new PriorityQueue>>(fuzzyKeysData.size(),
+ new Comparator>>() {
+@Override
+public int compare(Pair> o1,
+Pair> o2) {
+ int compare = Bytes.compareTo(o1.getFirst(), o2.getFirst());
+ if (!isReversed()) {
+return compare;
+ } else {
+return -compare;
+ }
+}
+ });
}
byte[] nextRow() {
- if (next < 0 || next == nextR
