http://git-wip-us.apache.org/repos/asf/hbase/blob/341cf7e1/hbase-protocol-shaded/src/main/protobuf/Admin.proto ---------------------------------------------------------------------- diff --git a/hbase-protocol-shaded/src/main/protobuf/Admin.proto b/hbase-protocol-shaded/src/main/protobuf/Admin.proto index 338c80b..1a085e6 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Admin.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Admin.proto @@ -28,6 +28,7 @@ option optimize_for = SPEED; import "ClusterStatus.proto"; import "HBase.proto"; import "WAL.proto"; +import "Quota.proto"; message GetRegionInfoRequest { required RegionSpecifier region = 1; @@ -314,4 +315,12 @@ service AdminService { rpc GetRegionLoad(GetRegionLoadRequest) returns(GetRegionLoadResponse); + + /** Fetches the RegionServer's view of space quotas */ + rpc GetSpaceQuotaSnapshots(GetSpaceQuotaSnapshotsRequest) + returns(GetSpaceQuotaSnapshotsResponse); + + /** Fetches the RegionServer's space quota active enforcements */ + rpc GetSpaceQuotaEnforcements(GetSpaceQuotaEnforcementsRequest) + returns(GetSpaceQuotaEnforcementsResponse); }
http://git-wip-us.apache.org/repos/asf/hbase/blob/341cf7e1/hbase-protocol-shaded/src/main/protobuf/Master.proto ---------------------------------------------------------------------- diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto index 4e856c8..58e6f77 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Master.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto @@ -929,4 +929,8 @@ service MasterService { /** Unmark a list of ServerNames marked as draining. */ rpc removeDrainFromRegionServers(RemoveDrainFromRegionServersRequest) returns(RemoveDrainFromRegionServersResponse); + + /** Fetches the Master's view of space quotas */ + rpc GetSpaceQuotaRegionSizes(GetSpaceQuotaRegionSizesRequest) + returns(GetSpaceQuotaRegionSizesResponse); } http://git-wip-us.apache.org/repos/asf/hbase/blob/341cf7e1/hbase-protocol-shaded/src/main/protobuf/Quota.proto ---------------------------------------------------------------------- diff --git a/hbase-protocol-shaded/src/main/protobuf/Quota.proto b/hbase-protocol-shaded/src/main/protobuf/Quota.proto index 597b059..2d7e5f5 100644 --- a/hbase-protocol-shaded/src/main/protobuf/Quota.proto +++ b/hbase-protocol-shaded/src/main/protobuf/Quota.proto @@ -111,3 +111,38 @@ message SpaceQuotaSnapshot { optional uint64 usage = 2; optional uint64 limit = 3; } + +message GetSpaceQuotaRegionSizesRequest { +} + +message GetSpaceQuotaRegionSizesResponse { + message RegionSizes { + optional TableName table_name = 1; + optional uint64 size = 2; + } + repeated RegionSizes sizes = 1; +} + +message GetSpaceQuotaSnapshotsRequest { +} + +message GetSpaceQuotaSnapshotsResponse { + // Cannot use TableName as a map key, do the repeated nested message by hand. + message TableQuotaSnapshot { + optional TableName table_name = 1; + optional SpaceQuotaSnapshot snapshot = 2; + } + repeated TableQuotaSnapshot snapshots = 1; +} + +message GetSpaceQuotaEnforcementsRequest { +} + +message GetSpaceQuotaEnforcementsResponse { + // Cannot use TableName as a map key, do the repeated nested message by hand. + message TableViolationPolicy { + optional TableName table_name = 1; + optional SpaceViolationPolicy violation_policy = 2; + } + repeated TableViolationPolicy violation_policies = 1; +} http://git-wip-us.apache.org/repos/asf/hbase/blob/341cf7e1/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java index a7a2f94..53560e9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java @@ -21,8 +21,11 @@ package org.apache.hadoop.hbase.master; import java.io.IOException; import java.net.InetAddress; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import org.apache.commons.logging.Log; @@ -211,6 +214,9 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTa import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse.RegionSizes; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.GetLastFlushedSequenceIdResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerReportRequest; @@ -2029,4 +2035,38 @@ public class MasterRpcServices extends RSRpcServices throw new ServiceException(e); } } + + @Override + public GetSpaceQuotaRegionSizesResponse getSpaceQuotaRegionSizes( + RpcController controller, GetSpaceQuotaRegionSizesRequest request) throws ServiceException { + try { + master.checkInitialized(); + MasterQuotaManager quotaManager = this.master.getMasterQuotaManager(); + GetSpaceQuotaRegionSizesResponse.Builder builder = + GetSpaceQuotaRegionSizesResponse.newBuilder(); + if (null != quotaManager) { + Map<HRegionInfo,Long> regionSizes = quotaManager.snapshotRegionSizes(); + Map<TableName,Long> regionSizesByTable = new HashMap<>(); + // Translate hregioninfo+long -> tablename+long + for (Entry<HRegionInfo,Long> entry : regionSizes.entrySet()) { + final TableName tableName = entry.getKey().getTable(); + Long prevSize = regionSizesByTable.get(tableName); + if (null == prevSize) { + prevSize = 0L; + } + regionSizesByTable.put(tableName, prevSize + entry.getValue()); + } + // Serialize them into the protobuf + for (Entry<TableName,Long> tableSize : regionSizesByTable.entrySet()) { + builder.addSizes(RegionSizes.newBuilder() + .setTableName(ProtobufUtil.toProtoTableName(tableSize.getKey())) + .setSize(tableSize.getValue()).build()); + } + return builder.build(); + } + return builder.build(); + } catch (Exception e) { + throw new ServiceException(e); + } + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/341cf7e1/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ActivePolicyEnforcement.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ActivePolicyEnforcement.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ActivePolicyEnforcement.java index 9408e6c..a313fa1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ActivePolicyEnforcement.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/ActivePolicyEnforcement.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.hbase.quotas; +import java.util.Collections; import java.util.Map; import java.util.Objects; @@ -79,6 +80,13 @@ public class ActivePolicyEnforcement { return policy; } + /** + * Returns an unmodifiable version of the active {@link SpaceViolationPolicyEnforcement}s. + */ + public Map<TableName,SpaceViolationPolicyEnforcement> getPolicies() { + return Collections.unmodifiableMap(activePolicies); + } + @Override public String toString() { return getClass().getSimpleName() + ": " + activePolicies; http://git-wip-us.apache.org/repos/asf/hbase/blob/341cf7e1/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index fad21b0..89366a1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -99,6 +99,7 @@ import org.apache.hadoop.hbase.quotas.OperationQuota; import org.apache.hadoop.hbase.quotas.QuotaUtil; import org.apache.hadoop.hbase.quotas.RegionServerRpcQuotaManager; import org.apache.hadoop.hbase.quotas.RegionServerSpaceQuotaManager; +import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot; import org.apache.hadoop.hbase.quotas.SpaceViolationPolicyEnforcement; import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl; import org.apache.hadoop.hbase.regionserver.Leases.Lease; @@ -192,6 +193,13 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier; import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType; import org.apache.hadoop.hbase.shaded.protobuf.generated.MapReduceProtos.ScanMetrics; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse.TableQuotaSnapshot; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceViolationPolicy; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse.TableViolationPolicy; import org.apache.hadoop.hbase.shaded.protobuf.generated.RPCProtos.RequestHeader; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.BulkLoadDescriptor; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; @@ -3227,4 +3235,53 @@ public class RSRpcServices implements HBaseRPCErrorHandler, return UpdateConfigurationResponse.getDefaultInstance(); } + @Override + public GetSpaceQuotaSnapshotsResponse getSpaceQuotaSnapshots( + RpcController controller, GetSpaceQuotaSnapshotsRequest request) throws ServiceException { + try { + final RegionServerSpaceQuotaManager manager = + regionServer.getRegionServerSpaceQuotaManager(); + final GetSpaceQuotaSnapshotsResponse.Builder builder = + GetSpaceQuotaSnapshotsResponse.newBuilder(); + if (null != manager) { + final Map<TableName,SpaceQuotaSnapshot> snapshots = manager.copyQuotaSnapshots(); + for (Entry<TableName,SpaceQuotaSnapshot> snapshot : snapshots.entrySet()) { + builder.addSnapshots(TableQuotaSnapshot.newBuilder() + .setTableName(ProtobufUtil.toProtoTableName(snapshot.getKey())) + .setSnapshot(SpaceQuotaSnapshot.toProtoSnapshot(snapshot.getValue())) + .build()); + } + } + return builder.build(); + } catch (Exception e) { + throw new ServiceException(e); + } + } + + @Override + public GetSpaceQuotaEnforcementsResponse getSpaceQuotaEnforcements( + RpcController controller, GetSpaceQuotaEnforcementsRequest request) + throws ServiceException { + try { + final RegionServerSpaceQuotaManager manager = + regionServer.getRegionServerSpaceQuotaManager(); + final GetSpaceQuotaEnforcementsResponse.Builder builder = + GetSpaceQuotaEnforcementsResponse.newBuilder(); + if (null != manager) { + ActivePolicyEnforcement enforcements = manager.getActiveEnforcements(); + for (Entry<TableName,SpaceViolationPolicyEnforcement> enforcement + : enforcements.getPolicies().entrySet()) { + SpaceViolationPolicy pbPolicy = SpaceViolationPolicy.valueOf( + enforcement.getValue().getPolicyName()); + builder.addViolationPolicies(TableViolationPolicy.newBuilder() + .setTableName(ProtobufUtil.toProtoTableName(enforcement.getKey())) + .setViolationPolicy(pbPolicy).build()); + } + } + return builder.build(); + } catch (Exception e) { + throw new ServiceException(e); + } + } + } http://git-wip-us.apache.org/repos/asf/hbase/blob/341cf7e1/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java index f133afc..e52114e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java @@ -100,6 +100,10 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.PrepareBul import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.PrepareBulkLoadResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanRequest; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ScanResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaEnforcementsResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaSnapshotsResponse; import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; import org.apache.hadoop.hbase.quotas.RegionServerRpcQuotaManager; import org.apache.hadoop.hbase.quotas.RegionServerSpaceQuotaManager; @@ -726,4 +730,18 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices { public RegionServerSpaceQuotaManager getRegionServerSpaceQuotaManager() { return null; } + + @Override + public GetSpaceQuotaSnapshotsResponse getSpaceQuotaSnapshots( + RpcController controller, GetSpaceQuotaSnapshotsRequest request) + throws ServiceException { + return null; + } + + @Override + public GetSpaceQuotaEnforcementsResponse getSpaceQuotaEnforcements( + RpcController controller, GetSpaceQuotaEnforcementsRequest request) + throws ServiceException { + return null; + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/341cf7e1/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java new file mode 100644 index 0000000..d42f3d2 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaStatusRPCs.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.Waiter.Predicate; +import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus; +import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; + +/** + * Test class for the quota status RPCs in the master and regionserver. + */ +@Category({MediumTests.class}) +public class TestQuotaStatusRPCs { + private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final AtomicLong COUNTER = new AtomicLong(0); + + @Rule + public TestName testName = new TestName(); + private SpaceQuotaHelperForTests helper; + + @BeforeClass + public static void setUp() throws Exception { + Configuration conf = TEST_UTIL.getConfiguration(); + // Increase the frequency of some of the chores for responsiveness of the test + conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY, 1000); + conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY, 1000); + conf.setInt(QuotaObserverChore.QUOTA_OBSERVER_CHORE_DELAY_KEY, 1000); + conf.setInt(QuotaObserverChore.QUOTA_OBSERVER_CHORE_PERIOD_KEY, 1000); + conf.setInt(SpaceQuotaRefresherChore.POLICY_REFRESHER_CHORE_DELAY_KEY, 1000); + conf.setInt(SpaceQuotaRefresherChore.POLICY_REFRESHER_CHORE_PERIOD_KEY, 1000); + conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + TEST_UTIL.startMiniCluster(1); + } + + @AfterClass + public static void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void setupForTest() throws Exception { + helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER); + } + + @Test + public void testRegionSizesFromMaster() throws Exception { + final long tableSize = 1024L * 10L; // 10KB + final int numRegions = 10; + final TableName tn = helper.createTableWithRegions(numRegions); + // Will write at least `tableSize` data + helper.writeData(tn, tableSize); + + final HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster(); + final MasterQuotaManager quotaManager = master.getMasterQuotaManager(); + // Make sure the master has all of the reports + Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() { + @Override + public boolean evaluate() throws Exception { + return numRegions == countRegionsForTable(tn, quotaManager.snapshotRegionSizes()); + } + }); + + Map<TableName,Long> sizes = QuotaTableUtil.getMasterReportedTableSizes(TEST_UTIL.getConnection()); + Long size = sizes.get(tn); + assertNotNull("No reported size for " + tn, size); + assertTrue("Reported table size was " + size, size.longValue() >= tableSize); + } + + @Test + public void testQuotaSnapshotsFromRS() throws Exception { + final long sizeLimit = 1024L * 1024L; // 1MB + final long tableSize = 1024L * 10L; // 10KB + final int numRegions = 10; + final TableName tn = helper.createTableWithRegions(numRegions); + + // Define the quota + QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( + tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS); + TEST_UTIL.getAdmin().setQuota(settings); + + // Write at least `tableSize` data + helper.writeData(tn, tableSize); + + final HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); + final RegionServerSpaceQuotaManager manager = rs.getRegionServerSpaceQuotaManager(); + Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() { + @Override + public boolean evaluate() throws Exception { + SpaceQuotaSnapshot snapshot = manager.copyQuotaSnapshots().get(tn); + if (null == snapshot) { + return false; + } + return snapshot.getUsage() >= tableSize; + } + }); + + Map<TableName, SpaceQuotaSnapshot> snapshots = QuotaTableUtil.getRegionServerQuotaSnapshots( + TEST_UTIL.getConnection(), rs.getServerName()); + SpaceQuotaSnapshot snapshot = snapshots.get(tn); + assertNotNull("Did not find snapshot for " + tn, snapshot); + assertTrue( + "Observed table usage was " + snapshot.getUsage(), + snapshot.getUsage() >= tableSize); + assertEquals(snapshot.getLimit(), sizeLimit); + SpaceQuotaStatus pbStatus = snapshot.getQuotaStatus(); + assertFalse(pbStatus.isInViolation()); + } + + @Test + public void testQuotaEnforcementsFromRS() throws Exception { + final long sizeLimit = 1024L * 8L; // 8KB + final long tableSize = 1024L * 10L; // 10KB + final int numRegions = 10; + final TableName tn = helper.createTableWithRegions(numRegions); + + // Define the quota + QuotaSettings settings = QuotaSettingsFactory.limitTableSpace( + tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS); + TEST_UTIL.getAdmin().setQuota(settings); + + // Write at least `tableSize` data + try { + helper.writeData(tn, tableSize); + } catch (SpaceLimitingException e) { + // Pass + } + + final HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0); + final RegionServerSpaceQuotaManager manager = rs.getRegionServerSpaceQuotaManager(); + Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() { + @Override + public boolean evaluate() throws Exception { + ActivePolicyEnforcement enforcements = manager.getActiveEnforcements(); + SpaceViolationPolicyEnforcement enforcement = enforcements.getPolicyEnforcement(tn); + return enforcement.getQuotaSnapshot().getQuotaStatus().isInViolation(); + } + }); + + Map<TableName,SpaceViolationPolicy> violations = + QuotaTableUtil.getRegionServerQuotaViolations( + TEST_UTIL.getConnection(), rs.getServerName()); + SpaceViolationPolicy policy = violations.get(tn); + assertNotNull("Did not find policy for " + tn, policy); + assertEquals(SpaceViolationPolicy.NO_INSERTS, policy); + } + + private int countRegionsForTable(TableName tn, Map<HRegionInfo,Long> regionSizes) { + int size = 0; + for (HRegionInfo regionInfo : regionSizes.keySet()) { + if (tn.equals(regionInfo.getTable())) { + size++; + } + } + return size; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/341cf7e1/hbase-shell/src/main/ruby/hbase/quotas.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/hbase/quotas.rb b/hbase-shell/src/main/ruby/hbase/quotas.rb index d99fe72..a2b21fa 100644 --- a/hbase-shell/src/main/ruby/hbase/quotas.rb +++ b/hbase-shell/src/main/ruby/hbase/quotas.rb @@ -20,10 +20,12 @@ include Java java_import java.util.concurrent.TimeUnit java_import org.apache.hadoop.hbase.TableName +java_import org.apache.hadoop.hbase.ServerName java_import org.apache.hadoop.hbase.quotas.ThrottleType java_import org.apache.hadoop.hbase.quotas.QuotaFilter java_import org.apache.hadoop.hbase.quotas.QuotaRetriever java_import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory +java_import org.apache.hadoop.hbase.quotas.QuotaTableUtil java_import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy module HBaseQuotasConstants @@ -163,6 +165,20 @@ module Hbase @admin.setQuota(settings) end + def get_master_table_sizes() + QuotaTableUtil.getMasterReportedTableSizes(@admin.getConnection()) + end + + def get_rs_quota_snapshots(rs) + QuotaTableUtil.getRegionServerQuotaSnapshots(@admin.getConnection(), + ServerName.valueOf(rs)) + end + + def get_rs_quota_violations(rs) + QuotaTableUtil.getRegionServerQuotaViolations(@admin.getConnection(), + ServerName.valueOf(rs)) + end + def set_global_bypass(bypass, args) raise(ArgumentError, "Arguments should be a Hash") unless args.kind_of?(Hash) http://git-wip-us.apache.org/repos/asf/hbase/blob/341cf7e1/hbase-shell/src/main/ruby/shell.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb index 66480f9..b203edc 100644 --- a/hbase-shell/src/main/ruby/shell.rb +++ b/hbase-shell/src/main/ruby/shell.rb @@ -419,6 +419,9 @@ Shell.load_command_group( :commands => %w[ set_quota list_quotas + list_quota_table_sizes + list_quota_violations + list_quota_snapshots ] ) http://git-wip-us.apache.org/repos/asf/hbase/blob/341cf7e1/hbase-shell/src/main/ruby/shell/commands/list_quota_snapshots.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/list_quota_snapshots.rb b/hbase-shell/src/main/ruby/shell/commands/list_quota_snapshots.rb new file mode 100644 index 0000000..c907762 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/list_quota_snapshots.rb @@ -0,0 +1,59 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListQuotaSnapshots < Command + def help + return <<-EOF +Lists the current snapshot of quotas on the given RegionServer. This +information filters to each RegionServer from the Master. For each +table, a snapshot includes the filesystem use, the filesystem limit, +and the policy to enact when the limit is exceeded. This command is +useful for debugging the running state of a cluster using filesystem quotas. + +For example: + + hbase> list_quota_snapshots 'regionserver1.domain,16020,1483482894742' +EOF + end + + def command(hostname, args = {}) + formatter.header(["TABLE", "USAGE", "LIMIT", "IN VIOLATION", "POLICY"]) + count = 0 + quotas_admin.get_rs_quota_snapshots(hostname).each do |tableName,snapshot| + status = snapshot.getQuotaStatus() + policy = get_policy(status) + formatter.row([tableName.to_s, snapshot.getUsage().to_s, snapshot.getLimit().to_s, status.isInViolation().to_s, policy]) + count += 1 + end + formatter.footer(count) + end + + def get_policy(status) + # Unwrap the violation policy if it exists + if status.isInViolation() + status.getPolicy().name() + else + "None" + end + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/341cf7e1/hbase-shell/src/main/ruby/shell/commands/list_quota_table_sizes.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/list_quota_table_sizes.rb b/hbase-shell/src/main/ruby/shell/commands/list_quota_table_sizes.rb new file mode 100644 index 0000000..9325477 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/list_quota_table_sizes.rb @@ -0,0 +1,47 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListQuotaTableSizes < Command + def help + return <<-EOF +Lists the sizes of the tables in HBase as collected +for the purpose of implementing filesystem utilization +quotas. This information is extracted from the HBase +Master and drives future quota actions in the cluster. + +For example: + + hbase> list_quota_table_sizes +EOF + end + + def command(args = {}) + formatter.header(["TABLE", "SIZE"]) + count = 0 + quotas_admin.get_master_table_sizes().each do |tableName,size| + formatter.row([tableName.to_s, size.to_s]) + count += 1 + end + formatter.footer(count) + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/341cf7e1/hbase-shell/src/main/ruby/shell/commands/list_quota_violations.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/main/ruby/shell/commands/list_quota_violations.rb b/hbase-shell/src/main/ruby/shell/commands/list_quota_violations.rb new file mode 100644 index 0000000..f1836a2 --- /dev/null +++ b/hbase-shell/src/main/ruby/shell/commands/list_quota_violations.rb @@ -0,0 +1,48 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Shell + module Commands + class ListQuotaViolations < Command + def help + return <<-EOF +Lists the current quota violations being enforced by a RegionServer. +Violations are enacted based on the quota snapshot information a RegionServer +holds (see list_quota_snapshots). Each violation contains the action the +RegionServer is taking on the table. This command is useful in debugging +the running state of a cluster using filesystem quotas. + +For example: + + hbase> list_quota_violations 'regionserver1.domain,16020,1483482894742' +EOF + end + + def command(hostname, args = {}) + formatter.header(["TABLE", "POLICY"]) + count = 0 + quotas_admin.get_rs_quota_violations(hostname).each do |tableName,policy| + formatter.row([tableName.to_s, policy.name]) + count += 1 + end + formatter.footer(count) + end + end + end +end http://git-wip-us.apache.org/repos/asf/hbase/blob/341cf7e1/hbase-shell/src/test/ruby/hbase/quotas_test.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/test/ruby/hbase/quotas_test.rb b/hbase-shell/src/test/ruby/hbase/quotas_test.rb index 78c889c..076eaed 100644 --- a/hbase-shell/src/test/ruby/hbase/quotas_test.rb +++ b/hbase-shell/src/test/ruby/hbase/quotas_test.rb @@ -96,30 +96,6 @@ module Hbase end end - define_test '_parse_size accepts various forms of byte shorthand' do - qa = quotas_admin() - KILO = 1024 - MEGA = KILO * KILO - GIGA = MEGA * KILO - TERA = GIGA * KILO - PETA = TERA * KILO - assert_equal(1, qa._parse_size("1")) - assert_equal(1, qa._parse_size("1b")) - assert_equal(1, qa._parse_size("1B")) - assert_equal(KILO * 2, qa._parse_size("2k")) - assert_equal(KILO * 2, qa._parse_size("2K")) - assert_equal(MEGA * 5, qa._parse_size("5m")) - assert_equal(MEGA * 5, qa._parse_size("5M")) - assert_equal(GIGA * 3, qa._parse_size("3g")) - assert_equal(GIGA * 3, qa._parse_size("3G")) - assert_equal(TERA * 4, qa._parse_size("4t")) - assert_equal(TERA * 4, qa._parse_size("4T")) - assert_equal(PETA * 32, qa._parse_size("32p")) - assert_equal(PETA * 32, qa._parse_size("32P")) - assert_equal(GIGA * 4, qa._parse_size("4096m")) - assert_equal(GIGA * 4, qa._parse_size("4096M")) - end - define_test 'can set and remove quota' do command(:set_quota, TYPE => SPACE, LIMIT => '1G', POLICY => NO_INSERTS, TABLE => @test_name) output = capture_stdout{ command(:list_quotas) } http://git-wip-us.apache.org/repos/asf/hbase/blob/341cf7e1/hbase-shell/src/test/ruby/hbase/quotas_test_no_cluster.rb ---------------------------------------------------------------------- diff --git a/hbase-shell/src/test/ruby/hbase/quotas_test_no_cluster.rb b/hbase-shell/src/test/ruby/hbase/quotas_test_no_cluster.rb new file mode 100644 index 0000000..7504488 --- /dev/null +++ b/hbase-shell/src/test/ruby/hbase/quotas_test_no_cluster.rb @@ -0,0 +1,69 @@ +# +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +require 'shell' +require 'stringio' +require 'hbase_constants' +require 'hbase/hbase' +require 'hbase/table' + +include HBaseConstants + +module Hbase + class NoClusterSpaceQuotasTest < Test::Unit::TestCase + include TestHelpers + + define_test '_parse_size accepts various forms of byte shorthand' do + qa = ::Hbase::QuotasAdmin.new(nil) + KILO = 1024 + MEGA = KILO * KILO + GIGA = MEGA * KILO + TERA = GIGA * KILO + PETA = TERA * KILO + assert_equal(1, qa._parse_size("1")) + assert_equal(1, qa._parse_size("1b")) + assert_equal(1, qa._parse_size("1B")) + assert_equal(KILO * 2, qa._parse_size("2k")) + assert_equal(KILO * 2, qa._parse_size("2K")) + assert_equal(MEGA * 5, qa._parse_size("5m")) + assert_equal(MEGA * 5, qa._parse_size("5M")) + assert_equal(GIGA * 3, qa._parse_size("3g")) + assert_equal(GIGA * 3, qa._parse_size("3G")) + assert_equal(TERA * 4, qa._parse_size("4t")) + assert_equal(TERA * 4, qa._parse_size("4T")) + assert_equal(PETA * 32, qa._parse_size("32p")) + assert_equal(PETA * 32, qa._parse_size("32P")) + assert_equal(GIGA * 4, qa._parse_size("4096m")) + assert_equal(GIGA * 4, qa._parse_size("4096M")) + end + + define_test 'get policy name for status not in violation' do + java_import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot + java_import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy + + okStatus = SpaceQuotaSnapshot::SpaceQuotaStatus::notInViolation() + # By default, statuses are in violation + violatedStatus = SpaceQuotaSnapshot::SpaceQuotaStatus.new(SpaceViolationPolicy::NO_INSERTS) + # Pass in nil for the Shell instance (that we don't care about) + quotaSnapshotCommand = ::Shell::Commands::ListQuotaSnapshots.new(nil) + assert_equal('None', quotaSnapshotCommand.get_policy(okStatus)) + assert_equal('NO_INSERTS', quotaSnapshotCommand.get_policy(violatedStatus)) + end + end +end
