This is an automated email from the ASF dual-hosted git repository. rmdmattingly pushed a commit to branch HBASE-30348-v2 in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a462723ba9a0daf3ff1a4781eafe0a5b4bab9f81 Author: Ray Mattingly <[email protected]> AuthorDate: Wed Sep 2 11:22:47 2026 -0400 HBASE-30348 Throttles should work for system tables (excluding quotas, meta, namespace) RegionServerRpcQuotaManager#getQuota returned NoopOperationQuota for any table where TableName#isSystemTable() is true, and QuotaCache#getUserLimiter returned NoopQuotaLimiter under the same condition. That covers every table in the hbase namespace and every table in the backup namespace, so no quota of any kind could throttle them: the quota was accepted, stored, and displayed by list_quotas, but the request path never consulted a limiter and nothing warned the operator. Only hbase:meta, hbase:quota and hbase:namespace are exempt now. All three are read while a master is still initializing, before quotas can be served, so throttling them could prevent a throttle from ever being lifted: quotas load from hbase:quota, clients resolve region locations through hbase:meta, and TableNamespaceManager reads hbase:namespace during initClusterSchemaService, which runs before initQuotaManager creates hbase:quota. An earlier attempt at this change omitted hbase:namespace and deadlocked master startup on branch-2.x. QuotaCache#ensureInitialized loaded the cache on the calling RPC handler, and that load both reads hbase:quota and asks the master for cluster metrics. A read of a table which is not exempt could therefore wait on a master that cannot answer until the read itself completes. The load now runs on a separate thread and the request waits for it with a timeout, configurable via hbase.quota.cache.initial.load.timeout.ms and defaulting to ten seconds. A request that waits out the timeout proceeds with default quota state, which does not throttle, and the load is picked up once it finishes. This keeps the load synchronous in effect, so a quota which was just set is still visible to the next lookup and cluster scope factors are still computed before the first cluster scope check. It also means requests no longer queue on the monitor that the refresh holds while it talks to the master. Adds TestQuotaUtil covering the exemption, including that hbase:namespace is exempt while the backup tables and the other hbase namespace tables are not. --- .../org/apache/hadoop/hbase/quotas/QuotaCache.java | 54 ++++++++++++++-- .../org/apache/hadoop/hbase/quotas/QuotaUtil.java | 11 ++++ .../hbase/quotas/RegionServerRpcQuotaManager.java | 2 +- .../apache/hadoop/hbase/quotas/TestQuotaUtil.java | 74 ++++++++++++++++++++++ 4 files changed, 133 insertions(+), 8 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java index e6144de2c77..f2bf59f626f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java @@ -23,7 +23,12 @@ import java.util.EnumSet; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.ClusterMetrics; import org.apache.hadoop.hbase.ClusterMetrics.Option; @@ -35,6 +40,7 @@ import org.apache.hadoop.hbase.ipc.RpcCall; import org.apache.hadoop.hbase.ipc.RpcServer; import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; import org.apache.hadoop.security.UserGroupInformation; import org.apache.yetus.audience.InterfaceAudience; import org.apache.yetus.audience.InterfaceStability; @@ -44,6 +50,7 @@ import org.slf4j.LoggerFactory; import org.apache.hbase.thirdparty.com.google.common.cache.CacheBuilder; import org.apache.hbase.thirdparty.com.google.common.cache.CacheLoader; import org.apache.hbase.thirdparty.com.google.common.cache.LoadingCache; +import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; /** * Cache that keeps track of the quota settings for the users and tables that are interacting with @@ -66,6 +73,9 @@ public class QuotaCache implements Stoppable { "hbase.quota.user.override.key"; private static final int REFRESH_DEFAULT_PERIOD = 43_200_000; // 12 hours + public static final String INITIAL_LOAD_TIMEOUT_MS = "hbase.quota.cache.initial.load.timeout.ms"; + private static final long INITIAL_LOAD_TIMEOUT_MS_DEFAULT = 10_000; + private final Object initializerLock = new Object(); private volatile boolean initialized = false; @@ -85,6 +95,15 @@ public class QuotaCache implements Stoppable { private QuotaRefresherChore refreshChore; private boolean stopped = true; + // The initial load runs here rather than on the requesting thread, so that a request can put a + // bound on how long it waits for it. The load reads hbase:quota and asks the master for cluster + // metrics, neither of which is necessarily available yet when the first request arrives. + private final ExecutorService initialLoadExecutor = Executors + .newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat("quota-cache-initial-load-%d") + .setDaemon(true).setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build()); + private Future<?> initialLoad; + private long initialLoadTimeoutMs; + public QuotaCache(final RegionServerServices rsServices) { this.rsServices = rsServices; this.userOverrideRequestAttributeKey = @@ -99,6 +118,7 @@ public class QuotaCache implements Stoppable { // configuration reload is triggered. Periodic reloads are kept to a minimum to avoid // flooding the RegionServer holding the hbase:quota table with requests. int period = conf.getInt(REFRESH_CONF_KEY, REFRESH_DEFAULT_PERIOD); + initialLoadTimeoutMs = conf.getLong(INITIAL_LOAD_TIMEOUT_MS, INITIAL_LOAD_TIMEOUT_MS_DEFAULT); refreshChore = new QuotaRefresherChore(conf, period, this); rsServices.getChoreService().scheduleChore(refreshChore); } @@ -109,6 +129,7 @@ public class QuotaCache implements Stoppable { LOG.debug("Stopping QuotaRefresherChore chore."); refreshChore.shutdown(true); } + initialLoadExecutor.shutdownNow(); stopped = true; } @@ -118,12 +139,31 @@ public class QuotaCache implements Stoppable { } private void ensureInitialized() { - if (!initialized) { - synchronized (initializerLock) { - if (!initialized) { - refreshChore.chore(); - initialized = true; - } + if (initialized) { + return; + } + synchronized (initializerLock) { + if (initialized) { + return; + } + if (initialLoad == null) { + initialLoad = initialLoadExecutor.submit(() -> refreshChore.chore()); + } + try { + initialLoad.get(initialLoadTimeoutMs, TimeUnit.MILLISECONDS); + initialized = true; + } catch (TimeoutException e) { + // The load is still running, and will mark the cache initialized once a later request + // observes that it finished. Until then requests get default quota state, which does not + // throttle. Waiting any longer would risk stalling the RPC handler behind a master that + // may not have finished initializing. + LOG.warn("Quota cache not loaded within {}ms, serving default quotas for now", + initialLoadTimeoutMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + LOG.warn("Failed to load quota cache", e.getCause()); + initialLoad = null; } } } @@ -155,7 +195,7 @@ public class QuotaCache implements Stoppable { * @return the limiter associated to the specified user/table */ public QuotaLimiter getUserLimiter(final UserGroupInformation ugi, final TableName table) { - if (table.isSystemTable()) { + if (QuotaUtil.isThrottleExempt(table)) { return NoopQuotaLimiter.get(); } return getUserQuotaState(ugi).getTableLimiter(table); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java index 8497f861f70..80d5e5ee796 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java @@ -122,6 +122,17 @@ public class QuotaUtil extends QuotaTableUtil { return conf.getBoolean(QUOTA_CONF_KEY, QUOTA_ENABLED_DEFAULT); } + /** + * Returns true if the given table can never be throttled. These tables are read while a master is + * still initializing, before quotas can be served, so throttling them could prevent a throttle + * from ever being lifted. + */ + @SuppressWarnings("deprecation") // hbase:namespace still exists on pre-3.x clusters + public static boolean isThrottleExempt(final TableName tableName) { + return TableName.META_TABLE_NAME.equals(tableName) || QUOTA_TABLE_NAME.equals(tableName) + || TableName.NAMESPACE_TABLE_NAME.equals(tableName); + } + /* * ========================================================================= Quota "settings" * helpers diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java index 34fc57cb081..a7167c3821a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/RegionServerRpcQuotaManager.java @@ -137,7 +137,7 @@ public class RegionServerRpcQuotaManager implements RpcQuotaManager, Configurati */ public OperationQuota getQuota(final UserGroupInformation ugi, final TableName table, final int blockSizeBytes) { - if (isQuotaEnabled() && !table.isSystemTable() && isRpcThrottleEnabled()) { + if (isQuotaEnabled() && !QuotaUtil.isThrottleExempt(table) && isRpcThrottleEnabled()) { UserQuotaState userQuotaState = quotaCache.getUserQuotaState(ugi); QuotaLimiter userLimiter = userQuotaState.getTableLimiter(table); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaUtil.java new file mode 100644 index 00000000000..386604229d8 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaUtil.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.quotas; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.testclassification.RegionServerTests; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag(RegionServerTests.TAG) +@Tag(SmallTests.TAG) +public class TestQuotaUtil { + + @Test + public void testMetaTableIsThrottleExempt() { + assertTrue(QuotaUtil.isThrottleExempt(TableName.META_TABLE_NAME)); + } + + @Test + public void testQuotaTableIsThrottleExempt() { + assertTrue(QuotaUtil.isThrottleExempt(QuotaTableUtil.QUOTA_TABLE_NAME)); + } + + @Test + public void testNamespaceTableIsThrottleExempt() { + // TableNamespaceManager reads hbase:namespace during initClusterSchemaService, which runs + // before initQuotaManager creates hbase:quota. Throttling it deadlocks master startup. + assertTrue(QuotaUtil.isThrottleExempt(TableName.valueOf("hbase:namespace"))); + } + + @Test + public void testBackupTablesAreNotThrottleExempt() { + // Backup tables live in a system namespace, so they were previously exempt. They can + // accumulate very large cells, which makes them a meaningful source of IO pressure, so + // operators need to be able to throttle them. + assertFalse(QuotaUtil.isThrottleExempt(TableName.valueOf("backup:system"))); + assertFalse(QuotaUtil.isThrottleExempt(TableName.valueOf("backup:system_bulk"))); + } + + @Test + public void testOtherSystemTablesAreNotThrottleExempt() { + // These are only read after the master is initialized, so a throttle on them cannot prevent + // itself from being lifted. + assertFalse(QuotaUtil.isThrottleExempt(TableName.valueOf("hbase:acl"))); + assertFalse(QuotaUtil.isThrottleExempt(TableName.valueOf("hbase:labels"))); + assertFalse(QuotaUtil.isThrottleExempt(TableName.valueOf("hbase:rsgroup"))); + assertFalse(QuotaUtil.isThrottleExempt(TableName.valueOf("hbase:replication"))); + } + + @Test + public void testUserTablesAreNotThrottleExempt() { + assertFalse(QuotaUtil.isThrottleExempt(TableName.valueOf("my_table"))); + assertFalse(QuotaUtil.isThrottleExempt(TableName.valueOf("my_ns:my_table"))); + } +}
