Repository: hbase Updated Branches: refs/heads/branch-1.2 96ed17eaf -> 0ba68fce1
http://git-wip-us.apache.org/repos/asf/hbase/blob/0ba68fce/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RegionNormalizerProtos.java ---------------------------------------------------------------------- diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RegionNormalizerProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RegionNormalizerProtos.java new file mode 100644 index 0000000..204c9cd --- /dev/null +++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RegionNormalizerProtos.java @@ -0,0 +1,485 @@ +// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: RegionNormalizer.proto + +package org.apache.hadoop.hbase.protobuf.generated; + +public final class RegionNormalizerProtos { + private RegionNormalizerProtos() {} + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + } + public interface RegionNormalizerStateOrBuilder + extends com.google.protobuf.MessageOrBuilder { + + // optional bool normalizer_on = 1; + /** + * <code>optional bool normalizer_on = 1;</code> + */ + boolean hasNormalizerOn(); + /** + * <code>optional bool normalizer_on = 1;</code> + */ + boolean getNormalizerOn(); + } + /** + * Protobuf type {@code RegionNormalizerState} + */ + public static final class RegionNormalizerState extends + com.google.protobuf.GeneratedMessage + implements RegionNormalizerStateOrBuilder { + // Use RegionNormalizerState.newBuilder() to construct. + private RegionNormalizerState(com.google.protobuf.GeneratedMessage.Builder<?> builder) { + super(builder); + this.unknownFields = builder.getUnknownFields(); + } + private RegionNormalizerState(boolean noInit) { this.unknownFields = com.google.protobuf.UnknownFieldSet.getDefaultInstance(); } + + private static final RegionNormalizerState defaultInstance; + public static RegionNormalizerState getDefaultInstance() { + return defaultInstance; + } + + public RegionNormalizerState getDefaultInstanceForType() { + return defaultInstance; + } + + private final com.google.protobuf.UnknownFieldSet unknownFields; + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private RegionNormalizerState( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + initFields(); + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownField(input, unknownFields, + extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + bitField0_ |= 0x00000001; + normalizerOn_ = input.readBool(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e.getMessage()).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.internal_static_RegionNormalizerState_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.internal_static_RegionNormalizerState_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState.class, org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState.Builder.class); + } + + public static com.google.protobuf.Parser<RegionNormalizerState> PARSER = + new com.google.protobuf.AbstractParser<RegionNormalizerState>() { + public RegionNormalizerState parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new RegionNormalizerState(input, extensionRegistry); + } + }; + + @java.lang.Override + public com.google.protobuf.Parser<RegionNormalizerState> getParserForType() { + return PARSER; + } + + private int bitField0_; + // optional bool normalizer_on = 1; + public static final int NORMALIZER_ON_FIELD_NUMBER = 1; + private boolean normalizerOn_; + /** + * <code>optional bool normalizer_on = 1;</code> + */ + public boolean hasNormalizerOn() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * <code>optional bool normalizer_on = 1;</code> + */ + public boolean getNormalizerOn() { + return normalizerOn_; + } + + private void initFields() { + normalizerOn_ = false; + } + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized != -1) return isInitialized == 1; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + getSerializedSize(); + if (((bitField0_ & 0x00000001) == 0x00000001)) { + output.writeBool(1, normalizerOn_); + } + getUnknownFields().writeTo(output); + } + + private int memoizedSerializedSize = -1; + public int getSerializedSize() { + int size = memoizedSerializedSize; + if (size != -1) return size; + + size = 0; + if (((bitField0_ & 0x00000001) == 0x00000001)) { + size += com.google.protobuf.CodedOutputStream + .computeBoolSize(1, normalizerOn_); + } + size += getUnknownFields().getSerializedSize(); + memoizedSerializedSize = size; + return size; + } + + private static final long serialVersionUID = 0L; + @java.lang.Override + protected java.lang.Object writeReplace() + throws java.io.ObjectStreamException { + return super.writeReplace(); + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState)) { + return super.equals(obj); + } + org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState other = (org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState) obj; + + boolean result = true; + result = result && (hasNormalizerOn() == other.hasNormalizerOn()); + if (hasNormalizerOn()) { + result = result && (getNormalizerOn() + == other.getNormalizerOn()); + } + result = result && + getUnknownFields().equals(other.getUnknownFields()); + return result; + } + + private int memoizedHashCode = 0; + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptorForType().hashCode(); + if (hasNormalizerOn()) { + hash = (37 * hash) + NORMALIZER_ON_FIELD_NUMBER; + hash = (53 * hash) + hashBoolean(getNormalizerOn()); + } + hash = (29 * hash) + getUnknownFields().hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState parseFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseDelimitedFrom(input, extensionRegistry); + } + public static org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return PARSER.parseFrom(input); + } + public static org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return PARSER.parseFrom(input, extensionRegistry); + } + + public static Builder newBuilder() { return Builder.create(); } + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder(org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState prototype) { + return newBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { return newBuilder(this); } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code RegionNormalizerState} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessage.Builder<Builder> + implements org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerStateOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.internal_static_RegionNormalizerState_descriptor; + } + + protected com.google.protobuf.GeneratedMessage.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.internal_static_RegionNormalizerState_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState.class, org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState.Builder.class); + } + + // Construct using org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessage.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessage.alwaysUseFieldBuilders) { + } + } + private static Builder create() { + return new Builder(); + } + + public Builder clear() { + super.clear(); + normalizerOn_ = false; + bitField0_ = (bitField0_ & ~0x00000001); + return this; + } + + public Builder clone() { + return create().mergeFrom(buildPartial()); + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.internal_static_RegionNormalizerState_descriptor; + } + + public org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState getDefaultInstanceForType() { + return org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState.getDefaultInstance(); + } + + public org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState build() { + org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState buildPartial() { + org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState result = new org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState(this); + int from_bitField0_ = bitField0_; + int to_bitField0_ = 0; + if (((from_bitField0_ & 0x00000001) == 0x00000001)) { + to_bitField0_ |= 0x00000001; + } + result.normalizerOn_ = normalizerOn_; + result.bitField0_ = to_bitField0_; + onBuilt(); + return result; + } + + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState) { + return mergeFrom((org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState other) { + if (other == org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState.getDefaultInstance()) return this; + if (other.hasNormalizerOn()) { + setNormalizerOn(other.getNormalizerOn()); + } + this.mergeUnknownFields(other.getUnknownFields()); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos.RegionNormalizerState) e.getUnfinishedMessage(); + throw e; + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + private int bitField0_; + + // optional bool normalizer_on = 1; + private boolean normalizerOn_ ; + /** + * <code>optional bool normalizer_on = 1;</code> + */ + public boolean hasNormalizerOn() { + return ((bitField0_ & 0x00000001) == 0x00000001); + } + /** + * <code>optional bool normalizer_on = 1;</code> + */ + public boolean getNormalizerOn() { + return normalizerOn_; + } + /** + * <code>optional bool normalizer_on = 1;</code> + */ + public Builder setNormalizerOn(boolean value) { + bitField0_ |= 0x00000001; + normalizerOn_ = value; + onChanged(); + return this; + } + /** + * <code>optional bool normalizer_on = 1;</code> + */ + public Builder clearNormalizerOn() { + bitField0_ = (bitField0_ & ~0x00000001); + normalizerOn_ = false; + onChanged(); + return this; + } + + // @@protoc_insertion_point(builder_scope:RegionNormalizerState) + } + + static { + defaultInstance = new RegionNormalizerState(true); + defaultInstance.initFields(); + } + + // @@protoc_insertion_point(class_scope:RegionNormalizerState) + } + + private static com.google.protobuf.Descriptors.Descriptor + internal_static_RegionNormalizerState_descriptor; + private static + com.google.protobuf.GeneratedMessage.FieldAccessorTable + internal_static_RegionNormalizerState_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + static { + java.lang.String[] descriptorData = { + "\n\026RegionNormalizer.proto\".\n\025RegionNormal" + + "izerState\022\025\n\rnormalizer_on\030\001 \001(\010BI\n*org." + + "apache.hadoop.hbase.protobuf.generatedB\026" + + "RegionNormalizerProtosH\001\240\001\001" + }; + com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = + new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() { + public com.google.protobuf.ExtensionRegistry assignDescriptors( + com.google.protobuf.Descriptors.FileDescriptor root) { + descriptor = root; + internal_static_RegionNormalizerState_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_RegionNormalizerState_fieldAccessorTable = new + com.google.protobuf.GeneratedMessage.FieldAccessorTable( + internal_static_RegionNormalizerState_descriptor, + new java.lang.String[] { "NormalizerOn", }); + return null; + } + }; + com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + }, assigner); + } + + // @@protoc_insertion_point(outer_class_scope) +} http://git-wip-us.apache.org/repos/asf/hbase/blob/0ba68fce/hbase-protocol/src/main/protobuf/Master.proto ---------------------------------------------------------------------- diff --git a/hbase-protocol/src/main/protobuf/Master.proto b/hbase-protocol/src/main/protobuf/Master.proto index ee2fc38..bb9077a 100644 --- a/hbase-protocol/src/main/protobuf/Master.proto +++ b/hbase-protocol/src/main/protobuf/Master.proto @@ -262,6 +262,28 @@ message IsBalancerEnabledResponse { required bool enabled = 1; } +message NormalizeRequest { +} + +message NormalizeResponse { + required bool normalizer_ran = 1; +} + +message SetNormalizerRunningRequest { + required bool on = 1; +} + +message SetNormalizerRunningResponse { + optional bool prev_normalizer_value = 1; +} + +message IsNormalizerEnabledRequest { +} + +message IsNormalizerEnabledResponse { + required bool enabled = 1; +} + message RunCatalogScanRequest { } @@ -569,6 +591,24 @@ service MasterService { rpc IsBalancerEnabled(IsBalancerEnabledRequest) returns(IsBalancerEnabledResponse); + /** + * Run region normalizer. Can NOT run for various reasons. Check logs. + */ + rpc Normalize(NormalizeRequest) + returns(NormalizeResponse); + + /** + * Turn region normalizer on or off. + */ + rpc SetNormalizerRunning(SetNormalizerRunningRequest) + returns(SetNormalizerRunningResponse); + + /** + * Query whether region normalizer is enabled. + */ + rpc IsNormalizerEnabled(IsNormalizerEnabledRequest) + returns(IsNormalizerEnabledResponse); + /** Get a run of the catalog janitor */ rpc RunCatalogScan(RunCatalogScanRequest) returns(RunCatalogScanResponse); http://git-wip-us.apache.org/repos/asf/hbase/blob/0ba68fce/hbase-protocol/src/main/protobuf/RegionNormalizer.proto ---------------------------------------------------------------------- diff --git a/hbase-protocol/src/main/protobuf/RegionNormalizer.proto b/hbase-protocol/src/main/protobuf/RegionNormalizer.proto new file mode 100644 index 0000000..e5305d6 --- /dev/null +++ b/hbase-protocol/src/main/protobuf/RegionNormalizer.proto @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This file contains protocol buffers to represent the state of the load balancer. + +option java_package = "org.apache.hadoop.hbase.protobuf.generated"; +option java_outer_classname = "RegionNormalizerProtos"; +option java_generate_equals_and_hash = true; +option optimize_for = SPEED; + +message RegionNormalizerState { + optional bool normalizer_on = 1; +} http://git-wip-us.apache.org/repos/asf/hbase/blob/0ba68fce/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 8dbc321..b3fd70a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -142,6 +142,7 @@ import org.apache.hadoop.hbase.zookeeper.DrainingServerTracker; import org.apache.hadoop.hbase.zookeeper.LoadBalancerTracker; import org.apache.hadoop.hbase.zookeeper.MasterAddressTracker; import org.apache.hadoop.hbase.zookeeper.MetaTableLocator; +import org.apache.hadoop.hbase.zookeeper.RegionNormalizerTracker; import org.apache.hadoop.hbase.zookeeper.RegionServerTracker; import org.apache.hadoop.hbase.zookeeper.ZKClusterId; import org.apache.hadoop.hbase.zookeeper.ZKUtil; @@ -241,6 +242,9 @@ public class HMaster extends HRegionServer implements MasterServices, Server { // Tracker for load balancer state LoadBalancerTracker loadBalancerTracker; + // Tracker for region normalizer state + private RegionNormalizerTracker regionNormalizerTracker; + /** Namespace stuff */ private TableNamespaceManager tableNamespaceManager; @@ -275,8 +279,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { private volatile boolean serverCrashProcessingEnabled = false; LoadBalancer balancer; - RegionNormalizer normalizer; - private boolean normalizerEnabled = false; + private RegionNormalizer normalizer; private BalancerChore balancerChore; private RegionNormalizerChore normalizerChore; private ClusterStatusChore clusterStatusChore; @@ -551,9 +554,10 @@ public class HMaster extends HRegionServer implements MasterServices, Server { this.balancer = LoadBalancerFactory.getLoadBalancer(conf); this.normalizer = RegionNormalizerFactory.getRegionNormalizer(conf); this.normalizer.setMasterServices(this); - this.normalizerEnabled = conf.getBoolean(HConstants.HBASE_NORMALIZER_ENABLED, false); this.loadBalancerTracker = new LoadBalancerTracker(zooKeeper, this); this.loadBalancerTracker.start(); + this.regionNormalizerTracker = new RegionNormalizerTracker(zooKeeper, this); + this.regionNormalizerTracker.start(); this.assignmentManager = new AssignmentManager(this, serverManager, this.balancer, this.service, this.metricsMaster, this.tableLockManager); @@ -1309,7 +1313,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server { return false; } - if (!this.normalizerEnabled) { + if (!this.regionNormalizerTracker.isNormalizerOn()) { LOG.debug("Region normalization is disabled, don't run region normalizer."); return false; } @@ -2588,6 +2592,17 @@ public class HMaster extends HRegionServer implements MasterServices, Server { } /** + * Queries the state of the {@link RegionNormalizerTracker}. If it's not initialized, + * false is returned. + */ + public boolean isNormalizerOn() { + if (null == regionNormalizerTracker) { + return false; + } + return regionNormalizerTracker.isNormalizerOn(); + } + + /** * Fetch the configured {@link LoadBalancer} class name. If none is set, a default is returned. * * @return The name of the {@link LoadBalancer} in use. @@ -2596,4 +2611,11 @@ public class HMaster extends HRegionServer implements MasterServices, Server { return conf.get(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, LoadBalancerFactory .getDefaultLoadBalancerClass().getName()); } + + /** + * @return RegionNormalizerTracker instance + */ + public RegionNormalizerTracker getRegionNormalizerTracker() { + return regionNormalizerTracker; + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/0ba68fce/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index 1ea102c..8b0d587 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -27,6 +27,7 @@ import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CoordinatedStateException; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; @@ -106,6 +107,8 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCatalogJanitorE import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsCatalogJanitorEnabledResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsMasterRunningResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsNormalizerEnabledRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsNormalizerEnabledResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsProcedureDoneResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsRestoreSnapshotDoneRequest; @@ -130,6 +133,8 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableReques import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.NormalizeRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.NormalizeResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.OfflineRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.OfflineRegionResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotRequest; @@ -141,6 +146,8 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SecurityCapabilit import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse.Capability; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningResponse; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetNormalizerRunningRequest; +import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetNormalizerRunningResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetQuotaRequest; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetQuotaResponse; import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ShutdownRequest; @@ -269,6 +276,25 @@ public class MasterRpcServices extends RSRpcServices } /** + * Sets normalizer on/off flag in ZK. + */ + public boolean normalizerSwitch(boolean on) { + boolean oldValue = master.getRegionNormalizerTracker().isNormalizerOn(); + boolean newValue = on; + try { + try { + master.getRegionNormalizerTracker().setNormalizerOn(newValue); + } catch (KeeperException ke) { + throw new IOException(ke); + } + LOG.info(master.getClientIdAuditPrefix() + " set normalizerSwitch=" + newValue); + } catch (IOException ioe) { + LOG.warn("Error flipping normalizer switch", ioe); + } + return oldValue; + } + + /** * @return list of blocking services and their security info classes that this server supports */ protected List<BlockingServiceAndInterface> getServices() { @@ -1390,6 +1416,36 @@ public class MasterRpcServices extends RSRpcServices } @Override + public NormalizeResponse normalize(RpcController controller, + NormalizeRequest request) throws ServiceException { + try { + return NormalizeResponse.newBuilder().setNormalizerRan(master.normalizeRegions()).build(); + } catch (IOException | CoordinatedStateException ex) { + throw new ServiceException(ex); + } + } + + @Override + public SetNormalizerRunningResponse setNormalizerRunning(RpcController controller, + SetNormalizerRunningRequest request) throws ServiceException { + try { + master.checkInitialized(); + boolean prevValue = normalizerSwitch(request.getOn()); + return SetNormalizerRunningResponse.newBuilder().setPrevNormalizerValue(prevValue).build(); + } catch (IOException ioe) { + throw new ServiceException(ioe); + } + } + + @Override + public IsNormalizerEnabledResponse isNormalizerEnabled(RpcController controller, + IsNormalizerEnabledRequest request) throws ServiceException { + IsNormalizerEnabledResponse.Builder response = IsNormalizerEnabledResponse.newBuilder(); + response.setEnabled(master.isNormalizerOn()); + return response.build(); + } + + @Override public SetQuotaResponse setQuota(RpcController c, SetQuotaRequest req) throws ServiceException { try { master.checkInitialized(); http://git-wip-us.apache.org/repos/asf/hbase/blob/0ba68fce/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionNormalizerTracker.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionNormalizerTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionNormalizerTracker.java new file mode 100644 index 0000000..d1f49b3 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/zookeeper/RegionNormalizerTracker.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.zookeeper; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.Abortable; +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.protobuf.generated.RegionNormalizerProtos; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.zookeeper.KeeperException; + +import java.io.IOException; + +/** + * Tracks region normalizer state up in ZK + */ +public class RegionNormalizerTracker extends ZooKeeperNodeTracker { + private static final Log LOG = LogFactory.getLog(RegionNormalizerTracker.class); + + public RegionNormalizerTracker(ZooKeeperWatcher watcher, + Abortable abortable) { + super(watcher, watcher.getRegionNormalizerZNode(), abortable); + } + + /** + * Return true if region normalizer is on, false otherwise + */ + public boolean isNormalizerOn() { + byte [] upData = super.getData(false); + try { + // if data in ZK is null, use default of on. + return upData == null || parseFrom(upData).getNormalizerOn(); + } catch (DeserializationException dex) { + LOG.error("ZK state for RegionNormalizer could not be parsed " + + Bytes.toStringBinary(upData)); + // return false to be safe. + return false; + } + } + + /** + * Set region normalizer on/off + * @param normalizerOn whether normalizer should be on or off + * @throws KeeperException + */ + public void setNormalizerOn(boolean normalizerOn) throws KeeperException { + byte [] upData = toByteArray(normalizerOn); + try { + ZKUtil.setData(watcher, watcher.getRegionNormalizerZNode(), upData); + } catch(KeeperException.NoNodeException nne) { + ZKUtil.createAndWatch(watcher, watcher.getRegionNormalizerZNode(), upData); + } + super.nodeDataChanged(watcher.getRegionNormalizerZNode()); + } + + private byte [] toByteArray(boolean isNormalizerOn) { + RegionNormalizerProtos.RegionNormalizerState.Builder builder = + RegionNormalizerProtos.RegionNormalizerState.newBuilder(); + builder.setNormalizerOn(isNormalizerOn); + return ProtobufUtil.prependPBMagic(builder.build().toByteArray()); + } + + private RegionNormalizerProtos.RegionNormalizerState parseFrom(byte [] pbBytes) + throws DeserializationException { + ProtobufUtil.expectPBMagicPrefix(pbBytes); + RegionNormalizerProtos.RegionNormalizerState.Builder builder = + RegionNormalizerProtos.RegionNormalizerState.newBuilder(); + try { + int magicLen = ProtobufUtil.lengthOfPBMagic(); + ProtobufUtil.mergeFrom(builder, pbBytes, magicLen, pbBytes.length - magicLen); + } catch (IOException e) { + throw new DeserializationException(e); + } + return builder.build(); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/0ba68fce/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java index e5d0d96..44f6f50 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin2.java @@ -752,4 +752,26 @@ public class TestAdmin2 { // Current state should be the original state again assertEquals(initialState, admin.isBalancerEnabled()); } + + @Test(timeout = 30000) + public void testRegionNormalizer() throws Exception { + boolean initialState = admin.isNormalizerEnabled(); + + // flip state + boolean prevState = admin.setNormalizerRunning(!initialState); + + // The previous state should be the original state we observed + assertEquals(initialState, prevState); + + // Current state should be opposite of the original + assertEquals(!initialState, admin.isNormalizerEnabled()); + + // Reset it back to what it was + prevState = admin.setNormalizerRunning(initialState); + + // The previous state should be the opposite of the initial state + assertEquals(!initialState, prevState); + // Current state should be the original state again + assertEquals(initialState, admin.isNormalizerEnabled()); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/0ba68fce/hbase-shell/src/main/ruby/hbase/admin.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb index 5d1addd..7e78b90 100644 --- a/hbase-shell/src/main/ruby/hbase/admin.rb +++ b/hbase-shell/src/main/ruby/hbase/admin.rb @@ -122,6 +122,27 @@ module Hbase end #---------------------------------------------------------------------------------------------- + # Requests region normalization for all configured tables in the cluster + # Returns true if normalizer ran successfully + def normalize() + @admin.normalize() + end + + #---------------------------------------------------------------------------------------------- + # Enable/disable region normalizer + # Returns previous normalizer switch setting. + def normalizer_switch(enableDisable) + @admin.setNormalizerRunning(java.lang.Boolean::valueOf(enableDisable)) + end + + #---------------------------------------------------------------------------------------------- + # Query the current state of region normalizer. + # Returns the state of region normalizer (true is enabled). + def normalizer_enabled?() + @admin.isNormalizerEnabled() + end + + #---------------------------------------------------------------------------------------------- # Request a scan of the catalog table (for garbage collection) # Returns an int signifying the number of entries cleaned def catalogjanitor_run() @@ -559,6 +580,8 @@ module Hbase htd.setMaxFileSize(JLong.valueOf(arg.delete(MAX_FILESIZE))) if arg[MAX_FILESIZE] htd.setReadOnly(JBoolean.valueOf(arg.delete(READONLY))) if arg[READONLY] htd.setCompactionEnabled(JBoolean.valueOf(arg[COMPACTION_ENABLED])) if arg[COMPACTION_ENABLED] + htd.setNormalizationEnabled( + JBoolean.valueOf(arg[NORMALIZATION_ENABLED])) if arg[NORMALIZATION_ENABLED] htd.setMemStoreFlushSize(JLong.valueOf(arg.delete(MEMSTORE_FLUSHSIZE))) if arg[MEMSTORE_FLUSHSIZE] # DEFERRED_LOG_FLUSH is deprecated and was replaced by DURABILITY. To keep backward compatible, it still exists. # However, it has to be set before DURABILITY so that DURABILITY could overwrite if both args are set http://git-wip-us.apache.org/repos/asf/hbase/blob/0ba68fce/hbase-shell/src/main/ruby/shell.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index 704eb1c..eb9fe89 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -314,6 +314,9 @@ Shell.load_command_group( balancer balance_switch balancer_enabled + normalize + normalizer_switch + normalizer_enabled close_region compact flush http://git-wip-us.apache.org/repos/asf/hbase/blob/0ba68fce/hbase-shell/src/main/ruby/shell/commands/normalize.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/normalize.rb b/hbase-shell/src/main/ruby/shell/commands/normalize.rb new file mode 100644 index 0000000..7e6302c --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/normalize.rb @@ -0,0 +1,45 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class Normalize < Command + def help + return <<-EOF +Trigger region normalizer for all tables which have NORMALIZATION_ENABLED flag set. Returns true + if normalizer ran successfully, false otherwise. Note that this command has no effect + if region normalizer is disabled (make sure it's turned on using 'normalizer_switch' command). + + Examples: + + hbase> normalize +EOF + end + + def command() + format_simple_command do + formatter.row([ + admin.normalize()? "true": "false" + ]) + end + end + end + end +end + http://git-wip-us.apache.org/repos/asf/hbase/blob/0ba68fce/hbase-shell/src/main/ruby/shell/commands/normalizer_enabled.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/normalizer_enabled.rb b/hbase-shell/src/main/ruby/shell/commands/normalizer_enabled.rb new file mode 100644 index 0000000..1121b25 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/normalizer_enabled.rb @@ -0,0 +1,41 @@ +#!/usr/bin/env hbase-jruby +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Prints current region normalizer status + +module Shell + module Commands + class NormalizerEnabled < Command + def help + return <<-EOF +Query the state of region normalizer. +Examples: + + hbase> normalizer_enabled +EOF + end + + def command() + format_simple_command do + formatter.row([ + admin.normalizer_enabled?.to_s + ]) + end + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/0ba68fce/hbase-shell/src/main/ruby/shell/commands/normalizer_switch.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/normalizer_switch.rb b/hbase-shell/src/main/ruby/shell/commands/normalizer_switch.rb new file mode 100644 index 0000000..6d959c4 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/normalizer_switch.rb @@ -0,0 +1,43 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class NormalizerSwitch < Command + def help + return <<-EOF +Enable/Disable region normalizer. Returns previous normalizer state. +When normalizer is enabled, it handles all tables with 'NORMALIZATION_ENABLED' => true. +Examples: + + hbase> normalizer_switch true + hbase> normalizer_switch false +EOF + end + + def command(enableDisable) + format_simple_command do + formatter.row([ + admin.normalizer_switch(enableDisable)? "true" : "false" + ]) + end + end + end + end +end
