HBASE-16996 Implement storage/retrieval of filesystem-use quotas into quota
table (Josh Elser)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a29abe64
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a29abe64
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a29abe64
Branch: refs/heads/HBASE-16961
Commit: a29abe646e2b1cd75ee1d0f186cc7486fe78b79e
Parents: 988a23e
Author: tedyu
Authored: Sat Dec 3 14:30:48 2016 -0800
Committer: Josh Elser
Committed: Mon Apr 17 15:35:31 2017 -0400
--
.../hadoop/hbase/quotas/QuotaTableUtil.java | 13 +-
.../hadoop/hbase/quotas/MasterQuotaManager.java | 30 +
.../hadoop/hbase/quotas/TestQuotaAdmin.java | 125 ++-
3 files changed, 165 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a29abe64/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index c44090f..8ef4f08 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -53,7 +53,9 @@ import org.apache.hadoop.hbase.util.Strings;
*
* ROW-KEY FAM/QUALDATA
* n. q:s
+ * n. u:du
* t. q:s
+ * t. u:du
* u. q:s
* u. q:s.
* u. q:s.:
@@ -72,6 +74,7 @@ public class QuotaTableUtil {
protected static final byte[] QUOTA_FAMILY_USAGE = Bytes.toBytes("u");
protected static final byte[] QUOTA_QUALIFIER_SETTINGS = Bytes.toBytes("s");
protected static final byte[] QUOTA_QUALIFIER_SETTINGS_PREFIX =
Bytes.toBytes("s.");
+ protected static final byte[] QUOTA_QUALIFIER_DISKUSAGE =
Bytes.toBytes("du");
protected static final byte[] QUOTA_USER_ROW_KEY_PREFIX =
Bytes.toBytes("u.");
protected static final byte[] QUOTA_TABLE_ROW_KEY_PREFIX =
Bytes.toBytes("t.");
protected static final byte[] QUOTA_NAMESPACE_ROW_KEY_PREFIX =
Bytes.toBytes("n.");
@@ -330,11 +333,16 @@ public class QuotaTableUtil {
* Quotas protobuf helpers
*/
protected static Quotas quotasFromData(final byte[] data) throws IOException
{
+return quotasFromData(data, 0, data.length);
+ }
+
+ protected static Quotas quotasFromData(
+ final byte[] data, int offset, int length) throws IOException {
int magicLen = ProtobufMagic.lengthOfPBMagic();
-if (!ProtobufMagic.isPBMagicPrefix(data, 0, magicLen)) {
+if (!ProtobufMagic.isPBMagicPrefix(data, offset, magicLen)) {
throw new IOException("Missing pb magic prefix");
}
-return Quotas.parseFrom(new ByteArrayInputStream(data, magicLen,
data.length - magicLen));
+return Quotas.parseFrom(new ByteArrayInputStream(data, offset + magicLen,
length - magicLen));
}
protected static byte[] quotasToData(final Quotas data) throws IOException {
@@ -348,6 +356,7 @@ public class QuotaTableUtil {
boolean hasSettings = false;
hasSettings |= quotas.hasThrottle();
hasSettings |= quotas.hasBypassGlobals();
+hasSettings |= quotas.hasSpace();
return !hasSettings;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a29abe64/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
index 5dab2e3..1469268 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.Quota
HBASE-16438 Create a cell type so that chunk id is embedded in it (Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c2c2178b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c2c2178b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c2c2178b
Branch: refs/heads/HBASE-16961
Commit: c2c2178b2eebe4439eadec6b37fae2566944c16b
Parents: c8cd921
Author: Ramkrishna
Authored: Mon Apr 17 09:10:59 2017 +0530
Committer: Ramkrishna
Committed: Mon Apr 17 09:28:24 2017 +0530
--
.../java/org/apache/hadoop/hbase/CellUtil.java | 24 --
.../org/apache/hadoop/hbase/ExtendedCell.java | 10 +
.../org/apache/hadoop/hbase/master/HMaster.java | 2 +
.../hbase/regionserver/ByteBufferChunkCell.java | 48 +++
.../apache/hadoop/hbase/regionserver/Chunk.java | 60 ++-
.../hadoop/hbase/regionserver/ChunkCreator.java | 404 +++
.../hbase/regionserver/HRegionServer.java | 14 +-
.../hbase/regionserver/MemStoreChunkPool.java | 265
.../hadoop/hbase/regionserver/MemStoreLAB.java | 4 +-
.../hbase/regionserver/MemStoreLABImpl.java | 171
.../regionserver/NoTagByteBufferChunkCell.java | 48 +++
.../hadoop/hbase/regionserver/OffheapChunk.java | 31 +-
.../hadoop/hbase/regionserver/OnheapChunk.java | 32 +-
.../hadoop/hbase/HBaseTestingUtility.java | 3 +
.../coprocessor/TestCoprocessorInterface.java | 4 +
.../TestRegionObserverScannerOpenHook.java | 3 +
.../coprocessor/TestRegionObserverStacking.java | 3 +
.../io/hfile/TestScannerFromBucketCache.java| 3 +
.../hadoop/hbase/master/TestCatalogJanitor.java | 7 +
.../hadoop/hbase/regionserver/TestBulkLoad.java | 2 +-
.../hbase/regionserver/TestCellFlatSet.java | 2 +-
.../regionserver/TestCompactingMemStore.java| 37 +-
.../TestCompactingToCellArrayMapMemStore.java | 16 +-
.../TestCompactionArchiveConcurrentClose.java | 1 +
.../TestCompactionArchiveIOException.java | 1 +
.../regionserver/TestCompactionPolicy.java | 1 +
.../hbase/regionserver/TestDefaultMemStore.java | 14 +-
.../regionserver/TestFailedAppendAndSync.java | 1 +
.../hbase/regionserver/TestHMobStore.java | 2 +-
.../hadoop/hbase/regionserver/TestHRegion.java | 2 +
.../regionserver/TestHRegionReplayEvents.java | 2 +-
.../regionserver/TestMemStoreChunkPool.java | 48 +--
.../hbase/regionserver/TestMemStoreLAB.java | 27 +-
.../TestMemstoreLABWithoutPool.java | 168
.../hbase/regionserver/TestRecoveredEdits.java | 1 +
.../hbase/regionserver/TestRegionIncrement.java | 1 +
.../hadoop/hbase/regionserver/TestStore.java| 1 +
.../TestStoreFileRefresherChore.java| 1 +
.../hbase/regionserver/TestWALLockup.java | 1 +
.../TestWALMonotonicallyIncreasingSeqId.java| 1 +
.../hbase/regionserver/wal/TestDurability.java | 3 +
41 files changed, 990 insertions(+), 479 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/c2c2178b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
index e1bc969..56de21b 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
@@ -3135,28 +3135,4 @@ public final class CellUtil {
return Type.DeleteFamily.getCode();
}
}
-
- /**
- * Clone the passed cell by copying its data into the passed buf.
- */
- public static Cell copyCellTo(Cell cell, ByteBuffer buf, int offset, int
len) {
-int tagsLen = cell.getTagsLength();
-if (cell instanceof ExtendedCell) {
- ((ExtendedCell) cell).write(buf, offset);
-} else {
- // Normally all Cell impls within Server will be of type ExtendedCell.
Just considering the
- // other case also. The data fragments within Cell is copied into buf as
in KeyValue
- // serialization format only.
- KeyValueUtil.appendTo(cell, buf, offset, true);
-}
-if (tagsLen == 0) {
- // When tagsLen is 0, make a NoTagsByteBufferKeyValue version. This is
an optimized class
- // which directly return tagsLen as 0. So we avoid parsing many length
components in
- // reading the tagLength stored in the backing buffer. The Memstore
addition of every Cell
- // call getTagsLength().
- return new NoTagsByteBufferKeyValue(buf, offset, len,
cell.getSequenceId());
-} else {
- return new ByteBufferKeyValue(buf, offset, len, cell.getSequenceId());
-}
- }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c2c2178b/hbase-common/src
HBASE-16995 Build client Java API and client protobuf messages - addendum fixes
white spaces (Josh Elser)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/eaeef44e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/eaeef44e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/eaeef44e
Branch: refs/heads/HBASE-16961
Commit: eaeef44e2fbd4f13a7f2d8dc5934eda3c54c529f
Parents: 990062a
Author: tedyu
Authored: Thu Nov 17 10:42:18 2016 -0800
Committer: Josh Elser
Committed: Mon Apr 17 15:35:31 2017 -0400
--
.../hbase/quotas/TestQuotaSettingsFactory.java| 2 +-
.../shaded/protobuf/generated/MasterProtos.java | 2 +-
.../shaded/protobuf/generated/QuotaProtos.java| 18 +-
.../hbase/protobuf/generated/QuotaProtos.java | 4 ++--
4 files changed, 13 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/eaeef44e/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
--
diff --git
a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
index 17015d6..e0012a7 100644
---
a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
+++
b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
@@ -44,7 +44,7 @@ import org.junit.experimental.categories.Category;
*/
@Category(SmallTests.class)
public class TestQuotaSettingsFactory {
-
+
@Test
public void testAllQuotasAddedToList() {
final SpaceQuota spaceQuota = SpaceQuota.newBuilder()
http://git-wip-us.apache.org/repos/asf/hbase/blob/eaeef44e/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
index 0c3248c..bbc6d1d 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
@@ -63752,7 +63752,7 @@ public final class MasterProtos {
* optional .hbase.pb.SpaceLimitRequest space_limit = 8;
*/
private
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequestOrBuilder>
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequestOrBuilder>
getSpaceLimitFieldBuilder() {
if (spaceLimitBuilder_ == null) {
spaceLimitBuilder_ = new
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
http://git-wip-us.apache.org/repos/asf/hbase/blob/eaeef44e/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index e3c6bfd..0ab2576 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -4362,7 +4362,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuota space = 3;
*/
private
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
org.apache.hadoop.hbase.shaded.protobuf.gene
HBASE-17903 Corrected the alias for the link of HBASE-6580
Signed-off-by: CHIA-PING TSAI
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/918aa465
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/918aa465
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/918aa465
Branch: refs/heads/HBASE-16961
Commit: 918aa4655c4109159f27b6d78460bd3681c11f06
Parents: 8db9760
Author: Jan Hentschel
Authored: Sun Apr 16 17:02:47 2017 +0200
Committer: CHIA-PING TSAI
Committed: Mon Apr 17 10:22:25 2017 +0800
--
src/main/asciidoc/_chapters/architecture.adoc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/918aa465/src/main/asciidoc/_chapters/architecture.adoc
--
diff --git a/src/main/asciidoc/_chapters/architecture.adoc
b/src/main/asciidoc/_chapters/architecture.adoc
index 773d237..27aebd9 100644
--- a/src/main/asciidoc/_chapters/architecture.adoc
+++ b/src/main/asciidoc/_chapters/architecture.adoc
@@ -230,7 +230,7 @@ try (Connection connection =
ConnectionFactory.createConnection(conf)) {
.`HTablePool` is Deprecated
[WARNING]
-Previous versions of this guide discussed `HTablePool`, which was deprecated
in HBase 0.94, 0.95, and 0.96, and removed in 0.98.1, by
link:https://issues.apache.org/jira/browse/HBASE-6580[HBASE-6500], or
`HConnection`, which is deprecated in HBase 1.0 by `Connection`.
+Previous versions of this guide discussed `HTablePool`, which was deprecated
in HBase 0.94, 0.95, and 0.96, and removed in 0.98.1, by
link:https://issues.apache.org/jira/browse/HBASE-6580[HBASE-6580], or
`HConnection`, which is deprecated in HBase 1.0 by `Connection`.
Please use
link:http://hbase.apache.org/apidocs/org/apache/hadoop/hbase/client/Connection.html[Connection]
instead.
http://git-wip-us.apache.org/repos/asf/hbase/blob/bcf6da40/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
new file mode 100644
index 000..98236c2
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -0,0 +1,596 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.NamespaceNotFoundException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore.TablesWithQuotas;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+
+/**
+ * Test class for {@link QuotaObserverChore} that uses a live HBase cluster.
+ */
+@Category(LargeTests.class)
+public class TestQuotaObserverChoreWithMiniCluster {
+ private static final Log LOG =
LogFactory.getLog(TestQuotaObserverChoreWithMiniCluster.class);
+ private static final int SIZE_PER_VALUE = 256;
+ private static final String F1 = "f1";
+ private static final HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
+ private static final AtomicLong COUNTER = new AtomicLong(0);
+ private static final long ONE_MEGABYTE = 1024L * 1024L;
+ private static final long DEFAULT_WAIT_MILLIS = 500;
+
+ @Rule
+ public TestName testName = new TestName();
+
+ private HMaster master;
+ private QuotaObserverChore chore;
+ private SpaceQuotaViolationNotifierForTest violationNotifier;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+Configuration conf = TEST_UTIL.getConfiguration();
+conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY,
1000);
+conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY,
1000);
+conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_DELAY_KEY, 1000);
+conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_PERIOD_KEY, 1000);
+conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
+TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void removeAllQuotas() throws Exception {
+final Connection conn = TEST_UTIL.getConnection();
+
HBASE-17794 Swap "violation" for "snapshot" where appropriate
A couple of variables and comments in which violation is incorrectly
used to describe what the code is doing. This was a hold over from early
implementation -- need to scrub these out for clarity.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/abe1c065
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/abe1c065
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/abe1c065
Branch: refs/heads/HBASE-16961
Commit: abe1c065ab803ba60fa28b17432746e756c31ed7
Parents: 5b3926b
Author: Josh Elser
Authored: Thu Mar 16 19:26:14 2017 -0400
Committer: Josh Elser
Committed: Mon Apr 17 15:47:49 2017 -0400
--
.../java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java| 4 ++--
hbase-protocol-shaded/src/main/protobuf/Quota.proto| 2 +-
.../org/apache/hadoop/hbase/quotas/QuotaObserverChore.java | 6 +++---
.../apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java| 2 +-
4 files changed, 7 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/abe1c065/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index ad59517..c008702 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -228,7 +228,7 @@ public class QuotaTableUtil {
}
/**
- * Creates a {@link Scan} which returns only quota violations from the quota
table.
+ * Creates a {@link Scan} which returns only quota snapshots from the quota
table.
*/
public static Scan makeQuotaSnapshotScan() {
Scan s = new Scan();
@@ -246,7 +246,7 @@ public class QuotaTableUtil {
* will throw an {@link IllegalArgumentException}.
*
* @param result A row from the quota table.
- * @param snapshots A map of violations to add the result of this method
into.
+ * @param snapshots A map of snapshots to add the result of this method into.
*/
public static void extractQuotaSnapshot(
Result result, Map snapshots) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/abe1c065/hbase-protocol-shaded/src/main/protobuf/Quota.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
index 1a6d5ed..364c58b 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
@@ -98,7 +98,7 @@ message SpaceLimitRequest {
}
// Represents the state of a quota on a table. Either the quota is not in
violation
-// or it is in violatino there is a violation policy which should be in effect.
+// or it is in violation there is a violation policy which should be in effect.
message SpaceQuotaStatus {
optional SpaceViolationPolicy policy = 1;
optional bool in_violation = 2;
http://git-wip-us.apache.org/repos/asf/hbase/blob/abe1c065/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index 94c5c87..254f2a1 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -532,9 +532,9 @@ public class QuotaObserverChore extends ScheduledChore {
}
/**
- * Stores the quota violation state for the given table.
+ * Stores the quota state for the given table.
*/
- void setTableQuotaViolation(TableName table, SpaceQuotaSnapshot snapshot) {
+ void setTableQuotaSnapshot(TableName table, SpaceQuotaSnapshot snapshot) {
this.tableQuotaSnapshots.put(table, snapshot);
}
@@ -552,7 +552,7 @@ public class QuotaObserverChore extends ScheduledChore {
}
/**
- * Stores the quota violation state for the given namespace.
+ * Stores the quota state for the given namespace.
*/
void setNamespaceQuotaSnapshot(String namespace, SpaceQuotaSnapshot
snapshot) {
this.namespaceQuotaSnapshots.put(namespace, snapshot);
http://git-wip-us.apache.org/repos/asf/hbase/blob/abe1c065/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java
---
http://git-wip-us.apache.org/repos/asf/hbase/blob/ecdfb823/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index 095f4bd..d56d6ec 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -116,7 +116,6 @@ import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
-import org.apache.hadoop.hbase.filter.FilterAllFilter;
import org.apache.hadoop.hbase.filter.FilterBase;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.NullComparator;
@@ -4932,7 +4931,6 @@ public class TestHRegion {
String callingMethod, Configuration conf, boolean isReadOnly, byte[]...
families)
throws IOException {
Path logDir = TEST_UTIL.getDataTestDirOnTestFS(callingMethod + ".log");
-ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0,
0, null);
HRegionInfo hri = new HRegionInfo(tableName, startKey, stopKey);
final WAL wal = HBaseTestingUtility.createWal(conf, logDir, hri);
return initHRegion(tableName, startKey, stopKey, isReadOnly,
http://git-wip-us.apache.org/repos/asf/hbase/blob/ecdfb823/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
index 6eed7df..0054642 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegionReplayEvents.java
@@ -153,7 +153,7 @@ public class TestHRegionReplayEvents {
}
time = System.currentTimeMillis();
-ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0,
0, null);
+
primaryHri = new HRegionInfo(htd.getTableName(),
HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
false, time, 0);
http://git-wip-us.apache.org/repos/asf/hbase/blob/ecdfb823/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
index 1768801..37a7664 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreChunkPool.java
@@ -48,30 +48,30 @@ import static org.junit.Assert.assertTrue;
@Category({RegionServerTests.class, SmallTests.class})
public class TestMemStoreChunkPool {
private final static Configuration conf = new Configuration();
- private static ChunkCreator chunkCreator;
+ private static MemStoreChunkPool chunkPool;
private static boolean chunkPoolDisabledBeforeTest;
@BeforeClass
public static void setUpBeforeClass() throws Exception {
conf.setBoolean(MemStoreLAB.USEMSLAB_KEY, true);
conf.setFloat(MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY, 0.2f);
-chunkPoolDisabledBeforeTest = ChunkCreator.chunkPoolDisabled;
-ChunkCreator.chunkPoolDisabled = false;
+chunkPoolDisabledBeforeTest = MemStoreChunkPool.chunkPoolDisabled;
+MemStoreChunkPool.chunkPoolDisabled = false;
long globalMemStoreLimit = (long)
(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()
.getMax() * MemorySizeUtil.getGlobalMemStoreHeapPercent(conf, false));
-chunkCreator = ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT,
false,
- globalMemStoreLimit, 0.2f, MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT, null);
-assertTrue(chunkCreator != null);
+chunkPool = MemStoreChunkPool.initialize(globalMemStoreLimit, 0.2f,
+MemStoreLAB.POOL_INITIAL_SIZE_DEFAULT,
MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false);
+assertTrue(chunkPool != null);
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
-ChunkCreator.chunkPoolDisabled = chunkPoolDisabledBeforeTest;
+MemStoreChunkPool.chunkPoolDisabled = chunkPoolDisabledBeforeTest;
}
@Before
public void tearDown() throws Exception {
-chunkCreator.clearChunksInPool();
+chunkPool.clearChunks();
}
@Test
@
http://git-wip-us.apache.org/repos/asf/hbase/blob/2dea6764/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
index d7d4db0..e90c934 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
@@ -10164,6 +10164,1912 @@ public final class RegionServerStatusProtos {
}
+ public interface RegionSpaceUseOrBuilder extends
+ // @@protoc_insertion_point(interface_extends:hbase.pb.RegionSpaceUse)
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
+
+/**
+ *
+ * A region identifier
+ *
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+boolean hasRegion();
+/**
+ *
+ * A region identifier
+ *
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo
getRegion();
+/**
+ *
+ * A region identifier
+ *
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfoOrBuilder
getRegionOrBuilder();
+
+/**
+ *
+ * The size in bytes of the region
+ *
+ *
+ * optional uint64 size = 2;
+ */
+boolean hasSize();
+/**
+ *
+ * The size in bytes of the region
+ *
+ *
+ * optional uint64 size = 2;
+ */
+long getSize();
+ }
+ /**
+ * Protobuf type {@code hbase.pb.RegionSpaceUse}
+ */
+ public static final class RegionSpaceUse extends
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3
implements
+ // @@protoc_insertion_point(message_implements:hbase.pb.RegionSpaceUse)
+ RegionSpaceUseOrBuilder {
+// Use RegionSpaceUse.newBuilder() to construct.
+private
RegionSpaceUse(org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.Builder
builder) {
+ super(builder);
+}
+private RegionSpaceUse() {
+ size_ = 0L;
+}
+
+@java.lang.Override
+public final
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+ return this.unknownFields;
+}
+private RegionSpaceUse(
+org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream
input,
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite
extensionRegistry)
+throws
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
{
+ this();
+ int mutable_bitField0_ = 0;
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.Builder
unknownFields =
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+boolean done = false;
+while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+case 0:
+ done = true;
+ break;
+default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+ }
+ break;
+}
+case 10: {
+
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.Builder
subBuilder = null;
+ if (((bitField0_ & 0x0001) == 0x0001)) {
+subBuilder = region_.toBuilder();
+ }
+ region_ =
input.readMessage(org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.PARSER,
extensionRegistry);
+ if (subBuilder != null) {
+subBuilder.mergeFrom(region_);
+region_ = subBuilder.buildPartial();
+ }
+ bitField0_ |= 0x0001;
+ break;
+}
+case 16: {
+ bitField0_ |= 0x0002;
+ size_ = input.readUInt64();
+ break;
+}
+ }
+}
+ } catch
(org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
e) {
+throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+throw new
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException(
+e).setUnfinishedMessage(this);
+ } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/0d76d667/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
index cc40536..d466e59 100644
---
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
@@ -5778,6 +5778,1284 @@ public final class QuotaProtos {
// @@protoc_insertion_point(class_scope:hbase.pb.SpaceLimitRequest)
}
+ public interface SpaceQuotaStatusOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+// optional .hbase.pb.SpaceViolationPolicy policy = 1;
+/**
+ * optional .hbase.pb.SpaceViolationPolicy policy = 1;
+ */
+boolean hasPolicy();
+/**
+ * optional .hbase.pb.SpaceViolationPolicy policy = 1;
+ */
+
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy
getPolicy();
+
+// optional bool in_violation = 2;
+/**
+ * optional bool in_violation = 2;
+ */
+boolean hasInViolation();
+/**
+ * optional bool in_violation = 2;
+ */
+boolean getInViolation();
+ }
+ /**
+ * Protobuf type {@code hbase.pb.SpaceQuotaStatus}
+ *
+ *
+ * Represents the state of a quota on a table. Either the quota is not in
violation
+ * or it is in violatino there is a violation policy which should be in
effect.
+ *
+ */
+ public static final class SpaceQuotaStatus extends
+ com.google.protobuf.GeneratedMessage
+ implements SpaceQuotaStatusOrBuilder {
+// Use SpaceQuotaStatus.newBuilder() to construct.
+private SpaceQuotaStatus(com.google.protobuf.GeneratedMessage.Builder
builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+}
+private SpaceQuotaStatus(boolean noInit) { this.unknownFields =
com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+private static final SpaceQuotaStatus defaultInstance;
+public static SpaceQuotaStatus getDefaultInstance() {
+ return defaultInstance;
+}
+
+public SpaceQuotaStatus getDefaultInstanceForType() {
+ return defaultInstance;
+}
+
+private final com.google.protobuf.UnknownFieldSet unknownFields;
+@java.lang.Override
+public final com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+ return this.unknownFields;
+}
+private SpaceQuotaStatus(
+com.google.protobuf.CodedInputStream input,
+com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+boolean done = false;
+while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+case 0:
+ done = true;
+ break;
+default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+ }
+ break;
+}
+case 8: {
+ int rawValue = input.readEnum();
+
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy
value =
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy.valueOf(rawValue);
+ if (value == null) {
+unknownFields.mergeVarintField(1, rawValue);
+ } else {
+bitField0_ |= 0x0001;
+policy_ = value;
+ }
+ break;
+}
+case 16: {
+ bitField0_ |= 0x0002;
+ inViolation_ = input.readBool();
+ break;
+}
+ }
+}
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+throw new com.google.protobuf.InvalidProtocolBufferException(
+e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+ }
+}
+public static final com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+ return
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.internal_static_hbase_pb_SpaceQuotaStatus_descriptor;
+}
+
+protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+internalGetFieldAccessorTable() {
+ return
org.apache
http://git-wip-us.apache.org/repos/asf/hbase/blob/095fabf1/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index a4c6095..d56def5 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -4362,7 +4362,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuota space = 3;
*/
private
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
getSpaceFieldBuilder() {
if (spaceBuilder_ == null) {
spaceBuilder_ = new
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
@@ -6077,7 +6077,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuota quota = 1;
*/
private
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
getQuotaFieldBuilder() {
if (quotaBuilder_ == null) {
quotaBuilder_ = new
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
@@ -6351,7 +6351,7 @@ public final class QuotaProtos {
return memoizedHashCode;
}
int hash = 41;
- hash = (19 * hash) + getDescriptorForType().hashCode();
+ hash = (19 * hash) + getDescriptor().hashCode();
if (hasPolicy()) {
hash = (37 * hash) + POLICY_FIELD_NUMBER;
hash = (53 * hash) + policy_;
@@ -6978,7 +6978,7 @@ public final class QuotaProtos {
return memoizedHashCode;
}
int hash = 41;
- hash = (19 * hash) + getDescriptorForType().hashCode();
+ hash = (19 * hash) + getDescriptor().hashCode();
if (hasStatus()) {
hash = (37 * hash) + STATUS_FIELD_NUMBER;
hash = (53 * hash) + getStatus().hashCode();
@@ -7351,7 +7351,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuotaStatus status = 1;
*/
private
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatus,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatus.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatusOrBuilder>
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatus,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatus.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaStatusOrBuilder>
getStatusFieldBuilder() {
if (statusBuilder_ == null) {
statusBuilder_ = new
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
@@ -7476,163 +7476,5829 @@ public final class QuotaProtos {
}
- private static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
-internal_static_hbase_pb_TimedQuota_descriptor;
- private static final
-
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
- internal_static_hbase_pb_TimedQuota_fieldAccessorTable;
- private static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
-internal_static_hbase_pb_Throttle_descriptor;
- private static final
-
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
- internal_static_hbase_pb_Throttle_fieldAccessorTable;
- private static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/2dea6764/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
index 1c373ee..23ddd43 100644
--- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
@@ -141,6 +141,22 @@ message SplitTableRegionResponse {
optional uint64 proc_id = 1;
}
+message RegionSpaceUse {
+ optional RegionInfo region = 1; // A region identifier
+ optional uint64 size = 2; // The size in bytes of the region
+}
+
+/**
+ * Reports filesystem usage for regions.
+ */
+message RegionSpaceUseReportRequest {
+ repeated RegionSpaceUse space_use = 1;
+}
+
+message RegionSpaceUseReportResponse {
+
+}
+
service RegionServerStatusService {
/** Called when a region server first starts. */
rpc RegionServerStartup(RegionServerStartupRequest)
@@ -182,4 +198,10 @@ service RegionServerStatusService {
*/
rpc getProcedureResult(GetProcedureResultRequest)
returns(GetProcedureResultResponse);
+
+ /**
+ * Reports Region filesystem space use
+ */
+ rpc ReportRegionSpaceUse(RegionSpaceUseReportRequest)
+returns(RegionSpaceUseReportResponse);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2dea6764/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 9af8f45..f454248 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
+import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -216,6 +217,9 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProto
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStatusService;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorResponse;
@@ -2006,4 +2010,19 @@ public class MasterRpcServices extends RSRpcServices
throw new ServiceException(e);
}
}
+
+ @Override
+ public RegionSpaceUseReportResponse reportRegionSpaceUse(RpcController
controller,
+ RegionSpaceUseReportRequest request) throws ServiceException {
+try {
+ master.checkInitialized();
+ MasterQuotaManager quotaManager = this.master.getMasterQuotaManager();
+ for (RegionSpaceUse report : request.getSpaceUseList()) {
+quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()),
report.getSize());
+ }
+ return RegionSpaceUseReportResponse.newBuilder().build();
+} catch (Exception e) {
+ throw new ServiceException(e);
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2dea6764/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
new file mode 100644
index 000..01540eb
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or mor
HBASE-17888: Added generic methods for updating metrics on submit and finish of
a procedure execution
Signed-off-by: Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c8461456
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c8461456
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c8461456
Branch: refs/heads/HBASE-16961
Commit: c8461456d0ae81b90d67d36e1e077ae1d01102e5
Parents: e2a7461
Author: Umesh Agashe
Authored: Mon Apr 10 15:32:43 2017 -0700
Committer: Michael Stack
Committed: Fri Apr 14 11:51:08 2017 -0700
--
.../apache/hadoop/hbase/client/HBaseAdmin.java | 2 +-
.../org/apache/hadoop/hbase/ProcedureInfo.java | 20 +-
.../master/MetricsAssignmentManagerSource.java | 9 +-
.../MetricsAssignmentManagerSourceImpl.java | 9 +-
.../hadoop/hbase/procedure2/Procedure.java | 41 +-
.../hbase/procedure2/ProcedureExecutor.java | 11 +
.../hadoop/hbase/procedure2/ProcedureUtil.java | 10 +-
.../hbase/procedure2/TestProcedureMetrics.java | 254 ++
.../procedure2/TestStateMachineProcedure.java | 1 -
.../shaded/protobuf/generated/MasterProtos.java | 490 +--
.../protobuf/generated/ProcedureProtos.java | 146 +++---
.../src/main/protobuf/Master.proto | 2 +-
.../src/main/protobuf/Procedure.proto | 2 +-
.../hadoop/hbase/master/MasterRpcServices.java | 4 +-
.../master/procedure/ServerCrashProcedure.java | 2 +-
.../hbase-webapps/master/procedures.jsp | 2 +-
.../main/ruby/shell/commands/list_procedures.rb | 6 +-
17 files changed, 652 insertions(+), 359 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/c8461456/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 155a272..cadd6cc 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -2114,7 +2114,7 @@ public class HBaseAdmin implements Admin {
procedureState, procProto.hasParentId() ? procProto.getParentId() :
-1, nonceKey,
procProto.hasException()?
ForeignExceptionUtil.toIOException(procProto.getException()):
null,
-procProto.getLastUpdate(), procProto.getStartTime(),
+procProto.getLastUpdate(), procProto.getSubmittedTime(),
procProto.hasResult()? procProto.getResult().toByteArray() : null);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c8461456/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureInfo.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureInfo.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureInfo.java
index bb8bb08..6104c22 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureInfo.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureInfo.java
@@ -39,7 +39,7 @@ public class ProcedureInfo implements Cloneable {
private final NonceKey nonceKey;
private final IOException exception;
private final long lastUpdate;
- private final long startTime;
+ private final long submittedTime;
private final byte[] result;
private long clientAckTime = -1;
@@ -54,7 +54,7 @@ public class ProcedureInfo implements Cloneable {
final NonceKey nonceKey,
final IOException exception,
final long lastUpdate,
- final long startTime,
+ final long submittedTime,
final byte[] result) {
this.procId = procId;
this.procName = procName;
@@ -63,7 +63,7 @@ public class ProcedureInfo implements Cloneable {
this.parentId = parentId;
this.nonceKey = nonceKey;
this.lastUpdate = lastUpdate;
-this.startTime = startTime;
+this.submittedTime = submittedTime;
// If the procedure is completed, we should treat exception and result
differently
this.exception = exception;
@@ -74,7 +74,7 @@ public class ProcedureInfo implements Cloneable {
justification="Intentional; calling super class clone doesn't make sense
here.")
public ProcedureInfo clone() {
return new ProcedureInfo(procId, procName, procOwner, procState, parentId,
nonceKey,
- exception, lastUpdate, startTime, result);
+ exception, lastUpdate, submittedTime, result);
}
@Override
@@ -96,10 +96,10 @@ public class ProcedureInfo implements Cloneable {
sb.append(procState);
long now = EnvironmentEdgeManager.currentTime();
-
HBASE-15535 Correct link to Trafodion
Signed-off-by: CHIA-PING TSAI
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/363f6275
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/363f6275
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/363f6275
Branch: refs/heads/HBASE-16961
Commit: 363f62751c760cc8056a2b1be40a410281e634f7
Parents: 918aa46
Author: Gábor Lipták
Authored: Sat Apr 15 11:43:38 2017 -0400
Committer: CHIA-PING TSAI
Committed: Mon Apr 17 10:26:28 2017 +0800
--
src/main/asciidoc/_chapters/sql.adoc | 2 +-
src/main/site/xdoc/supportingprojects.xml | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/363f6275/src/main/asciidoc/_chapters/sql.adoc
--
diff --git a/src/main/asciidoc/_chapters/sql.adoc
b/src/main/asciidoc/_chapters/sql.adoc
index b47104c..b1ad063 100644
--- a/src/main/asciidoc/_chapters/sql.adoc
+++ b/src/main/asciidoc/_chapters/sql.adoc
@@ -37,6 +37,6 @@ link:http://phoenix.apache.org[Apache Phoenix]
=== Trafodion
-link:https://wiki.trafodion.org/[Trafodion: Transactional SQL-on-HBase]
+link:http://trafodion.incubator.apache.org/[Trafodion: Transactional
SQL-on-HBase]
:numbered:
http://git-wip-us.apache.org/repos/asf/hbase/blob/363f6275/src/main/site/xdoc/supportingprojects.xml
--
diff --git a/src/main/site/xdoc/supportingprojects.xml
b/src/main/site/xdoc/supportingprojects.xml
index f349c7f..f949a57 100644
--- a/src/main/site/xdoc/supportingprojects.xml
+++ b/src/main/site/xdoc/supportingprojects.xml
@@ -46,9 +46,9 @@ under the License.
for HBase.
https://github.com/juwi/HBase-TAggregator";>HBase
TAggregator
An HBase coprocessor for timeseries-based aggregations.
- http://www.trafodion.org";>Trafodion
- Trafodion is an HP-sponsored Apache-licensed open source SQL on HBase
-DBMS with full-ACID distributed transaction support.
+ http://trafodion.incubator.apache.org/";>Apache
Trafodion
+ Apache Trafodion is a webscale SQL-on-Hadoop solution enabling
+transactional or operational workloads on Hadoop.
http://phoenix.apache.org/";>Apache Phoenix
Apache Phoenix is a relational database layer over HBase delivered as a
client-embedded JDBC driver targeting low latency queries over HBase
data.
HBASE-17428 Implement informational RPCs for space quotas
Create some RPCs that can expose the in-memory state that the
RegionServers and Master hold to drive the space quota "state machine".
Then, create some hbase shell commands to interact with those.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/095fabf1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/095fabf1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/095fabf1
Branch: refs/heads/HBASE-16961
Commit: 095fabf16d91ce0e5c042b657a9f07a548d49a49
Parents: efd6edc
Author: Josh Elser
Authored: Tue Feb 21 15:36:39 2017 -0500
Committer: Josh Elser
Committed: Mon Apr 17 15:44:00 2017 -0400
--
.../hbase/client/ConnectionImplementation.java |9 +
.../hadoop/hbase/client/QuotaStatusCalls.java | 125 +
.../client/ShortCircuitMasterConnection.java|7 +
.../hadoop/hbase/quotas/QuotaTableUtil.java | 77 +
.../hbase/shaded/protobuf/RequestConverter.java | 33 +
.../shaded/protobuf/generated/AdminProtos.java | 394 +-
.../shaded/protobuf/generated/MasterProtos.java | 92 +-
.../shaded/protobuf/generated/QuotaProtos.java | 5986 +-
.../generated/RegionServerStatusProtos.java | 28 +-
.../src/main/protobuf/Admin.proto |9 +
.../src/main/protobuf/Master.proto |4 +
.../src/main/protobuf/Quota.proto | 35 +
.../hbase/protobuf/generated/QuotaProtos.java |6 +-
.../hadoop/hbase/master/MasterRpcServices.java | 40 +
.../hbase/quotas/ActivePolicyEnforcement.java |8 +
.../hbase/regionserver/RSRpcServices.java | 57 +
.../hadoop/hbase/master/MockRegionServer.java | 18 +
.../hbase/quotas/TestQuotaStatusRPCs.java | 192 +
hbase-shell/src/main/ruby/hbase/quotas.rb | 16 +
hbase-shell/src/main/ruby/shell.rb |3 +
.../ruby/shell/commands/list_quota_snapshots.rb | 59 +
.../shell/commands/list_quota_table_sizes.rb| 47 +
.../shell/commands/list_quota_violations.rb | 48 +
hbase-shell/src/test/ruby/hbase/quotas_test.rb | 24 -
.../test/ruby/hbase/quotas_test_no_cluster.rb | 69 +
25 files changed, 7066 insertions(+), 320 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/095fabf1/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index 99feb14..3f27e1c 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -94,6 +94,8 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCa
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.DisableReplicationPeerRequest;
@@ -1731,6 +1733,13 @@ class ConnectionImplementation implements
ClusterConnection, Closeable {
ListReplicationPeersRequest request) throws ServiceException {
return stub.listReplicationPeers(controller, request);
}
+
+ @Override
+ public GetSpaceQuotaRegionSizesResponse getSpaceQuotaRegionSizes(
+ RpcController controller, GetSpaceQuotaRegionSizesRequest request)
+ throws ServiceException {
+return stub.getSpaceQuotaRegionSizes(controller, request);
+ }
};
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/095fabf1/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
new file mode 100644
index 000..f0f385d
--- /dev/null
+++
b/hbase-client/src/main/java/
HBASE-17000 Implement computation of online region sizes and report to the
Master
Includes a trivial implementation of the Master-side collection to
avoid. Only enough to write a test to verify RS collection.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2dea6764
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2dea6764
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2dea6764
Branch: refs/heads/HBASE-16961
Commit: 2dea67644982aa18d174de32f454f526354eea5c
Parents: a29abe6
Author: Josh Elser
Authored: Mon Nov 7 13:46:42 2016 -0500
Committer: Josh Elser
Committed: Mon Apr 17 15:35:31 2017 -0400
--
.../generated/RegionServerStatusProtos.java | 2071 +-
.../src/main/protobuf/RegionServerStatus.proto | 22 +
.../hadoop/hbase/master/MasterRpcServices.java | 19 +
.../quotas/FileSystemUtilizationChore.java | 205 ++
.../hadoop/hbase/quotas/MasterQuotaManager.java | 15 +
.../hbase/regionserver/HRegionServer.java | 72 +
.../quotas/TestFileSystemUtilizationChore.java | 357 +++
.../hadoop/hbase/quotas/TestRegionSizeUse.java | 194 ++
.../TestRegionServerRegionSpaceUseReport.java | 99 +
9 files changed, 3032 insertions(+), 22 deletions(-)
--
HBASE-17906 When a huge amount of data writing to hbase through thrift2, there
will be a deadlock error. (Albert Lee)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9dd5cda0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9dd5cda0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9dd5cda0
Branch: refs/heads/HBASE-16961
Commit: 9dd5cda01747ffb91ac084792fa4a8670859e810
Parents: da5fb27
Author: Michael Stack
Authored: Thu Apr 13 21:59:11 2017 -0700
Committer: Michael Stack
Committed: Thu Apr 13 21:59:11 2017 -0700
--
.../main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java | 3 ---
1 file changed, 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/9dd5cda0/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
--
diff --git
a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
index 560ae64..8f56b10 100644
---
a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
+++
b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftServer.java
@@ -432,9 +432,6 @@ public class ThriftServer extends Configured implements
Tool {
throw new RuntimeException("Could not parse the value provided for the
port option", e);
}
-// Thrift's implementation uses '0' as a placeholder for 'use the default.'
-int backlog = conf.getInt(BACKLOG_CONF_KEY, 0);
-
// Local hostname and user name,
// used only if QOP is configured.
String host = null;
HBASE-17557 HRegionServer#reportRegionSizesForQuotas() should respond to
UnsupportedOperationException
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b8c5387f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b8c5387f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b8c5387f
Branch: refs/heads/HBASE-16961
Commit: b8c5387fa27e7f6816bacac8e8826085d04bd1a3
Parents: 2a21313
Author: tedyu
Authored: Mon Jan 30 07:47:40 2017 -0800
Committer: Josh Elser
Committed: Tue Apr 11 13:10:02 2017 -0400
--
.../quotas/FileSystemUtilizationChore.java | 20 +---
.../hbase/regionserver/HRegionServer.java | 24
2 files changed, 36 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/b8c5387f/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
index 01540eb..efc17ff 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
@@ -53,6 +53,9 @@ public class FileSystemUtilizationChore extends
ScheduledChore {
static final String FS_UTILIZATION_MAX_ITERATION_DURATION_KEY =
"hbase.regionserver.quotas.fs.utilization.chore.max.iteration.millis";
static final long FS_UTILIZATION_MAX_ITERATION_DURATION_DEFAULT = 5000L;
+ private int numberOfCyclesToSkip = 0, prevNumberOfCyclesToSkip = 0;
+ private static final int CYCLE_UPPER_BOUND = 32;
+
private final HRegionServer rs;
private final long maxIterationMillis;
private Iterator leftoverRegions;
@@ -67,6 +70,10 @@ public class FileSystemUtilizationChore extends
ScheduledChore {
@Override
protected void chore() {
+if (numberOfCyclesToSkip > 0) {
+ numberOfCyclesToSkip--;
+ return;
+}
final Map onlineRegionSizes = new HashMap<>();
final Set onlineRegions = new HashSet<>(rs.getOnlineRegions());
// Process the regions from the last run if we have any. If we are somehow
having difficulty
@@ -126,7 +133,14 @@ public class FileSystemUtilizationChore extends
ScheduledChore {
+ skippedSplitParents + " regions due to being the parent of a
split, and"
+ skippedRegionReplicas + " regions due to being region replicas.");
}
-reportRegionSizesToMaster(onlineRegionSizes);
+if (!reportRegionSizesToMaster(onlineRegionSizes)) {
+ // backoff reporting
+ numberOfCyclesToSkip = prevNumberOfCyclesToSkip > 0 ? 2 *
prevNumberOfCyclesToSkip : 1;
+ if (numberOfCyclesToSkip > CYCLE_UPPER_BOUND) {
+numberOfCyclesToSkip = CYCLE_UPPER_BOUND;
+ }
+ prevNumberOfCyclesToSkip = numberOfCyclesToSkip;
+}
}
/**
@@ -166,8 +180,8 @@ public class FileSystemUtilizationChore extends
ScheduledChore {
*
* @param onlineRegionSizes The computed region sizes to report.
*/
- void reportRegionSizesToMaster(Map onlineRegionSizes) {
-this.rs.reportRegionSizesForQuotas(onlineRegionSizes);
+ boolean reportRegionSizesToMaster(Map onlineRegionSizes) {
+return this.rs.reportRegionSizesForQuotas(onlineRegionSizes);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/b8c5387f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 9be4131..053e4ac 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.hbase.ChoreService;
import org.apache.hadoop.hbase.ClockOutOfSyncException;
import org.apache.hadoop.hbase.CoordinatedStateManager;
import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
+import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseInterfaceAudience;
import org.apache.hadoop.hbase.HConstants;
@@ -1248,13 +1249,14 @@ public class HRegionServer extends HasThread implements
* Reports the given map of Regions and their size on the filesystem to the
active Master.
*
* @param onlineRegionSizes A map of region info to size in bytes
+ * @return false if FileSystem
HBASE-17836 CellUtil#estimatedSerializedSizeOf is slow when input is
ByteBufferCell
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1a701ce4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1a701ce4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1a701ce4
Branch: refs/heads/HBASE-16961
Commit: 1a701ce44484f45a8a07ea9826b84f0df6f1518e
Parents: 48b2502
Author: Chia-Ping Tsai
Authored: Sat Apr 1 13:50:01 2017 +0800
Committer: Chia-Ping Tsai
Committed: Fri Apr 7 09:30:15 2017 +0800
--
.../src/main/java/org/apache/hadoop/hbase/CellUtil.java | 8 +++-
.../src/main/java/org/apache/hadoop/hbase/SplitLogTask.java | 2 +-
2 files changed, 4 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/1a701ce4/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
index 6585173..e1bc969 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
@@ -1385,12 +1385,10 @@ public final class CellUtil {
* @return Estimate of the cell size in bytes.
*/
public static int estimatedSerializedSizeOf(final Cell cell) {
-// If a KeyValue, we can give a good estimate of size.
-if (cell instanceof KeyValue) {
- return ((KeyValue)cell).getLength() + Bytes.SIZEOF_INT;
+if (cell instanceof ExtendedCell) {
+ return ((ExtendedCell) cell).getSerializedSize(true) + Bytes.SIZEOF_INT;
}
-// TODO: Should we add to Cell a sizeOf? Would it help? Does it make
sense if Cell is
-// prefix encoded or compressed?
+
return getSumOfCellElementLengths(cell) +
// Use the KeyValue's infrastructure size presuming that another
implementation would have
// same basic cost.
http://git-wip-us.apache.org/repos/asf/hbase/blob/1a701ce4/hbase-server/src/main/java/org/apache/hadoop/hbase/SplitLogTask.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/SplitLogTask.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/SplitLogTask.java
index 03d5108..3ecaa86 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/SplitLogTask.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/SplitLogTask.java
@@ -86,7 +86,7 @@ public class SplitLogTask {
public ServerName getServerName() {
return this.originServer;
}
-
+
public ZooKeeperProtos.SplitLogTask.RecoveryMode getMode() {
return this.mode;
}
HBASE-17447 Implement a MasterObserver for automatically deleting space quotas
When a table or namespace is deleted, it would be nice to automatically
delete the quota on said table/NS. It's possible that not all people
would want this functionality so we can leave it up to the user to
configure this Observer.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f46c16aa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f46c16aa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f46c16aa
Branch: refs/heads/HBASE-16961
Commit: f46c16aa2f03ba07c2585e012e176852bd95
Parents: 59148f6
Author: Josh Elser
Authored: Thu Mar 16 18:54:01 2017 -0400
Committer: Josh Elser
Committed: Tue Apr 11 13:10:03 2017 -0400
--
.../hbase/quotas/MasterSpaceQuotaObserver.java | 85 ++
.../quotas/TestMasterSpaceQuotaObserver.java| 169 +++
src/main/asciidoc/_chapters/ops_mgt.adoc| 17 ++
3 files changed, 271 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/f46c16aa/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
new file mode 100644
index 000..a3abf32
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.MasterObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+
+/**
+ * An observer to automatically delete space quotas when a table/namespace
+ * are deleted.
+ */
+@InterfaceAudience.Private
+public class MasterSpaceQuotaObserver implements MasterObserver {
+ private CoprocessorEnvironment cpEnv;
+ private Configuration conf;
+ private boolean quotasEnabled = false;
+
+ @Override
+ public void start(CoprocessorEnvironment ctx) throws IOException {
+this.cpEnv = ctx;
+this.conf = cpEnv.getConfiguration();
+this.quotasEnabled = QuotaUtil.isQuotaEnabled(conf);
+ }
+
+ @Override
+ public void postDeleteTable(
+ ObserverContext ctx, TableName tableName)
throws IOException {
+// Do nothing if quotas aren't enabled
+if (!quotasEnabled) {
+ return;
+}
+final MasterServices master = ctx.getEnvironment().getMasterServices();
+final Connection conn = master.getConnection();
+Quotas quotas = QuotaUtil.getTableQuota(master.getConnection(), tableName);
+if (null != quotas && quotas.hasSpace()) {
+ QuotaSettings settings =
QuotaSettingsFactory.removeTableSpaceLimit(tableName);
+ try (Admin admin = conn.getAdmin()) {
+admin.setQuota(settings);
+ }
+}
+ }
+
+ @Override
+ public void postDeleteNamespace(
+ ObserverContext ctx, String namespace)
throws IOException {
+// Do nothing if quotas aren't enabled
+if (!quotasEnabled) {
+ return;
+}
+final MasterServices master = ctx.getEnvironment().getMasterServices();
+final Connection conn = master.getConnection();
+Quotas quotas = QuotaUtil.getNamespaceQuota(master.getConnection(),
namespace);
+if (null != quotas && quotas.hasSpace()) {
+ QuotaSettings settings =
QuotaSettingsFactory
HBASE-17025 Add shell commands for space quotas
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bae29b64
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bae29b64
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bae29b64
Branch: refs/heads/HBASE-16961
Commit: bae29b64a8542f16f392b166ae78ebf3026698dd
Parents: 79bf188
Author: Josh Elser
Authored: Wed Jan 11 11:55:29 2017 -0500
Committer: Josh Elser
Committed: Tue Apr 11 13:10:02 2017 -0400
--
hbase-shell/src/main/ruby/hbase/quotas.rb | 62 -
hbase-shell/src/main/ruby/hbase_constants.rb| 1 +
.../src/main/ruby/shell/commands/set_quota.rb | 45 +-
.../hadoop/hbase/client/AbstractTestShell.java | 1 +
hbase-shell/src/test/ruby/hbase/quotas_test.rb | 137 +++
hbase-shell/src/test/ruby/tests_runner.rb | 1 +
6 files changed, 242 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/bae29b64/hbase-shell/src/main/ruby/hbase/quotas.rb
--
diff --git a/hbase-shell/src/main/ruby/hbase/quotas.rb
b/hbase-shell/src/main/ruby/hbase/quotas.rb
index bf2dc63..d99fe72 100644
--- a/hbase-shell/src/main/ruby/hbase/quotas.rb
+++ b/hbase-shell/src/main/ruby/hbase/quotas.rb
@@ -24,14 +24,22 @@ java_import org.apache.hadoop.hbase.quotas.ThrottleType
java_import org.apache.hadoop.hbase.quotas.QuotaFilter
java_import org.apache.hadoop.hbase.quotas.QuotaRetriever
java_import org.apache.hadoop.hbase.quotas.QuotaSettingsFactory
+java_import org.apache.hadoop.hbase.quotas.SpaceViolationPolicy
module HBaseQuotasConstants
+ # RPC Quota constants
GLOBAL_BYPASS = 'GLOBAL_BYPASS'
THROTTLE_TYPE = 'THROTTLE_TYPE'
THROTTLE = 'THROTTLE'
REQUEST = 'REQUEST'
WRITE = 'WRITE'
READ = 'READ'
+ # Space quota constants
+ SPACE = 'SPACE'
+ NO_INSERTS = 'NO_INSERTS'
+ NO_WRITES = 'NO_WRITES'
+ NO_WRITES_COMPACTIONS = 'NO_WRITES_COMPACTIONS'
+ DISABLE = 'DISABLE'
end
module Hbase
@@ -107,6 +115,54 @@ module Hbase
@admin.setQuota(settings)
end
+def limit_space(args)
+ raise(ArgumentError, 'Argument should be a Hash') unless (not args.nil?
and args.kind_of?(Hash))
+ # Let the user provide a raw number
+ if args[LIMIT].is_a?(Numeric)
+limit = args[LIMIT]
+ else
+# Parse a string a 1K, 2G, etc.
+limit = _parse_size(args[LIMIT])
+ end
+ # Extract the policy, failing if something bogus was provided
+ policy = SpaceViolationPolicy.valueOf(args[POLICY])
+ # Create a table or namespace quota
+ if args.key?(TABLE)
+if args.key?(NAMESPACE)
+ raise(ArgumentError, "Only one of TABLE or NAMESPACE can be
specified.")
+end
+settings =
QuotaSettingsFactory.limitTableSpace(TableName.valueOf(args.delete(TABLE)),
limit, policy)
+ elsif args.key?(NAMESPACE)
+if args.key?(TABLE)
+ raise(ArgumentError, "Only one of TABLE or NAMESPACE can be
specified.")
+end
+settings =
QuotaSettingsFactory.limitNamespaceSpace(args.delete(NAMESPACE), limit, policy)
+ else
+raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.')
+ end
+ # Apply the quota
+ @admin.setQuota(settings)
+end
+
+def remove_space_limit(args)
+ raise(ArgumentError, 'Argument should be a Hash') unless (not args.nil?
and args.kind_of?(Hash))
+ if args.key?(TABLE)
+if args.key?(NAMESPACE)
+ raise(ArgumentError, "Only one of TABLE or NAMESPACE can be
specified.")
+end
+table = TableName.valueOf(args.delete(TABLE))
+settings = QuotaSettingsFactory.removeTableSpaceLimit(table)
+ elsif args.key?(NAMESPACE)
+if args.key?(TABLE)
+ raise(ArgumentError, "Only one of TABLE or NAMESPACE can be
specified.")
+end
+settings =
QuotaSettingsFactory.removeNamespaceSpaceLimit(args.delete(NAMESPACE))
+ else
+raise(ArgumentError, 'One of TABLE or NAMESPACE must be specified.')
+ end
+ @admin.setQuota(settings)
+end
+
def set_global_bypass(bypass, args)
raise(ArgumentError, "Arguments should be a Hash") unless
args.kind_of?(Hash)
@@ -171,7 +227,7 @@ module Hbase
return _size_from_str(match[1].to_i, match[2])
end
else
-raise "Invalid size limit syntax"
+raise(ArgumentError, "Invalid size limit syntax")
end
end
@@ -188,7 +244,7 @@ module Hbase
end
if limit <= 0
- raise "Invalid throttle limit, must be greater then 0"
+ raise(ArgumentError, "Invalid throttle limit, must be greater then
0")
end
case match[3]
@@ -200,7 +256,7 @@ modul
http://git-wip-us.apache.org/repos/asf/hbase/blob/2a21313a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
index 1c373ee..23ddd43 100644
--- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
@@ -141,6 +141,22 @@ message SplitTableRegionResponse {
optional uint64 proc_id = 1;
}
+message RegionSpaceUse {
+ optional RegionInfo region = 1; // A region identifier
+ optional uint64 size = 2; // The size in bytes of the region
+}
+
+/**
+ * Reports filesystem usage for regions.
+ */
+message RegionSpaceUseReportRequest {
+ repeated RegionSpaceUse space_use = 1;
+}
+
+message RegionSpaceUseReportResponse {
+
+}
+
service RegionServerStatusService {
/** Called when a region server first starts. */
rpc RegionServerStartup(RegionServerStartupRequest)
@@ -182,4 +198,10 @@ service RegionServerStatusService {
*/
rpc getProcedureResult(GetProcedureResultRequest)
returns(GetProcedureResultResponse);
+
+ /**
+ * Reports Region filesystem space use
+ */
+ rpc ReportRegionSpaceUse(RegionSpaceUseReportRequest)
+returns(RegionSpaceUseReportResponse);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2a21313a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index f86f800..f10d4bb 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
+import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -216,6 +217,9 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProto
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStatusService;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorResponse;
@@ -2006,4 +2010,19 @@ public class MasterRpcServices extends RSRpcServices
throw new ServiceException(e);
}
}
+
+ @Override
+ public RegionSpaceUseReportResponse reportRegionSpaceUse(RpcController
controller,
+ RegionSpaceUseReportRequest request) throws ServiceException {
+try {
+ master.checkInitialized();
+ MasterQuotaManager quotaManager = this.master.getMasterQuotaManager();
+ for (RegionSpaceUse report : request.getSpaceUseList()) {
+quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()),
report.getSize());
+ }
+ return RegionSpaceUseReportResponse.newBuilder().build();
+} catch (Exception e) {
+ throw new ServiceException(e);
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2a21313a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
new file mode 100644
index 000..01540eb
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or mor
http://git-wip-us.apache.org/repos/asf/hbase/blob/4a715450/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
new file mode 100644
index 000..4a7000c
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.Objects;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.ArgumentMatcher;
+
+/**
+ * Test case for {@link TableSpaceQuotaViolationNotifier}.
+ */
+@Category(SmallTests.class)
+public class TestTableSpaceQuotaViolationNotifier {
+
+ private TableSpaceQuotaViolationNotifier notifier;
+ private Connection conn;
+
+ @Before
+ public void setup() throws Exception {
+notifier = new TableSpaceQuotaViolationNotifier();
+conn = mock(Connection.class);
+notifier.initialize(conn);
+ }
+
+ @Test
+ public void testToViolation() throws Exception {
+final TableName tn = TableName.valueOf("inviolation");
+final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
+final Table quotaTable = mock(Table.class);
+
when(conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable);
+
+final Put expectedPut = new Put(Bytes.toBytes("t." +
tn.getNameAsString()));
+final SpaceQuota protoQuota = SpaceQuota.newBuilder()
+.setViolationPolicy(ProtobufUtil.toProtoViolationPolicy(policy))
+.build();
+expectedPut.addColumn(Bytes.toBytes("u"), Bytes.toBytes("v"),
protoQuota.toByteArray());
+
+notifier.transitionTableToViolation(tn, policy);
+
+verify(quotaTable).put(argThat(new SingleCellPutMatcher(expectedPut)));
+ }
+
+ @Test
+ public void testToObservance() throws Exception {
+final TableName tn = TableName.valueOf("notinviolation");
+final Table quotaTable = mock(Table.class);
+
when(conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable);
+
+final Delete expectedDelete = new Delete(Bytes.toBytes("t." +
tn.getNameAsString()));
+expectedDelete.addColumn(Bytes.toBytes("u"), Bytes.toBytes("v"));
+
+notifier.transitionTableToObservance(tn);
+
+verify(quotaTable).delete(argThat(new
SingleCellDeleteMatcher(expectedDelete)));
+ }
+
+ /**
+ * Parameterized for Puts.
+ */
+ private static class SingleCellPutMatcher extends
SingleCellMutationMatcher {
+private SingleCellPutMatcher(Put expected) {
+ super(expected);
+}
+ }
+
+ /**
+ * Parameterized for Deletes.
+ */
+ private static class SingleCellDeleteMatcher extends
SingleCellMutationMatcher {
+private SingleCellDeleteMatcher(Delete expected) {
+ super(expected);
+}
+ }
+
+ /**
+ * Quick hack to verify a Mutation with one column.
+ */
+ private static class SingleCellMutationMatcher extends ArgumentMatcher
{
+private final Mutation expected;
+
+private SingleCellMutationMatcher(Mutation expected) {
+ this.
HBASE-17259 API to remove space quotas on a table/namespace
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/79bf1888
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/79bf1888
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/79bf1888
Branch: refs/heads/HBASE-16961
Commit: 79bf188818ae32e309433a28f73048558fb3b2db
Parents: 7f42b49
Author: Josh Elser
Authored: Wed Jan 11 12:47:06 2017 -0500
Committer: Josh Elser
Committed: Tue Apr 11 13:10:02 2017 -0400
--
.../hbase/quotas/QuotaSettingsFactory.java | 22 +++
.../hadoop/hbase/quotas/QuotaTableUtil.java | 6 +-
.../hadoop/hbase/quotas/SpaceLimitSettings.java | 44 -
.../hbase/quotas/TestQuotaSettingsFactory.java | 20 +++
.../shaded/protobuf/generated/QuotaProtos.java | 157 +++---
.../src/main/protobuf/Quota.proto | 1 +
.../hbase/protobuf/generated/QuotaProtos.java | 159 ---
hbase-protocol/src/main/protobuf/Quota.proto| 1 +
.../hadoop/hbase/quotas/MasterQuotaManager.java | 9 +-
.../hadoop/hbase/quotas/TestQuotaAdmin.java | 49 +-
10 files changed, 423 insertions(+), 45 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/79bf1888/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
index 7f1c180..184277d 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
@@ -316,6 +316,17 @@ public class QuotaSettingsFactory {
}
/**
+ * Creates a {@link QuotaSettings} object to remove the FileSystem space
quota for the given
+ * table.
+ *
+ * @param tableName The name of the table to remove the quota for.
+ * @return A {@link QuotaSettings} object.
+ */
+ public static QuotaSettings removeTableSpaceLimit(TableName tableName) {
+return new SpaceLimitSettings(tableName, true);
+ }
+
+ /**
* Creates a {@link QuotaSettings} object to limit the FileSystem space
usage for the given
* namespace to the given size in bytes. When the space usage is exceeded by
all tables in the
* namespace, the provided {@link SpaceViolationPolicy} is enacted on all
tables in the namespace.
@@ -329,4 +340,15 @@ public class QuotaSettingsFactory {
final String namespace, long sizeLimit, final SpaceViolationPolicy
violationPolicy) {
return new SpaceLimitSettings(namespace, sizeLimit, violationPolicy);
}
+
+ /**
+ * Creates a {@link QuotaSettings} object to remove the FileSystem space
quota for the given
+* namespace.
+ *
+ * @param namespace The namespace to remove the quota on.
+ * @return A {@link QuotaSettings} object.
+ */
+ public static QuotaSettings removeNamespaceSpaceLimit(String namespace) {
+return new SpaceLimitSettings(namespace, true);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/79bf1888/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index 42da811..ca525f0 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -390,7 +390,11 @@ public class QuotaTableUtil {
boolean hasSettings = false;
hasSettings |= quotas.hasThrottle();
hasSettings |= quotas.hasBypassGlobals();
-hasSettings |= quotas.hasSpace();
+// Only when there is a space quota, make sure there's actually both
fields provided
+// Otherwise, it's a noop.
+if (quotas.hasSpace()) {
+ hasSettings |= (quotas.getSpace().hasSoftLimit() &&
quotas.getSpace().hasViolationPolicy());
+}
return !hasSettings;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/79bf1888/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
index e54882e..8ff7623 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/h
HBASE-17857 Remove IS annotations from IA.Public classes
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a66d4918
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a66d4918
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a66d4918
Branch: refs/heads/HBASE-16961
Commit: a66d491892514fd4a188d6ca87d6260d8ae46184
Parents: 910b680
Author: zhangduo
Authored: Tue Apr 4 20:30:10 2017 +0800
Committer: zhangduo
Committed: Wed Apr 5 15:34:06 2017 +0800
--
.../hbase/classification/InterfaceAudience.java | 1 -
.../classification/InterfaceStability.java | 1 -
.../hadoop/hbase/CallDroppedException.java | 4 +-
.../hadoop/hbase/CallQueueTooBigException.java | 4 +-
.../hadoop/hbase/ClockOutOfSyncException.java | 2 -
.../org/apache/hadoop/hbase/ClusterStatus.java | 2 -
.../hadoop/hbase/DoNotRetryIOException.java | 2 -
.../hadoop/hbase/DroppedSnapshotException.java | 2 -
.../apache/hadoop/hbase/HColumnDescriptor.java | 2 -
.../org/apache/hadoop/hbase/HRegionInfo.java| 2 -
.../apache/hadoop/hbase/HRegionLocation.java| 2 -
.../apache/hadoop/hbase/HTableDescriptor.java | 2 -
.../hbase/InvalidFamilyOperationException.java | 2 -
.../apache/hadoop/hbase/KeepDeletedCells.java | 2 -
.../hadoop/hbase/MasterNotRunningException.java | 2 -
.../hadoop/hbase/MemoryCompactionPolicy.java| 2 -
.../hadoop/hbase/MultiActionResultTooLarge.java | 2 -
.../hadoop/hbase/NamespaceExistException.java | 2 -
.../hbase/NamespaceNotFoundException.java | 2 -
.../hbase/NotAllMetaRegionsOnlineException.java | 2 -
.../hadoop/hbase/NotServingRegionException.java | 2 -
.../hadoop/hbase/PleaseHoldException.java | 2 -
.../apache/hadoop/hbase/RegionException.java| 2 -
.../org/apache/hadoop/hbase/RegionLoad.java | 2 -
.../hadoop/hbase/RegionTooBusyException.java| 2 -
.../hbase/ReplicationPeerNotFoundException.java | 4 +-
.../hadoop/hbase/RetryImmediatelyException.java | 2 -
.../org/apache/hadoop/hbase/ServerLoad.java | 2 -
.../hadoop/hbase/TableExistsException.java | 2 -
.../hadoop/hbase/TableInfoMissingException.java | 2 -
.../hadoop/hbase/TableNotDisabledException.java | 2 -
.../hadoop/hbase/TableNotEnabledException.java | 2 -
.../hadoop/hbase/TableNotFoundException.java| 2 -
.../hadoop/hbase/UnknownRegionException.java| 2 -
.../hadoop/hbase/UnknownScannerException.java | 2 -
.../hbase/ZooKeeperConnectionException.java | 2 -
.../org/apache/hadoop/hbase/client/Admin.java | 2 -
.../org/apache/hadoop/hbase/client/Append.java | 2 -
.../apache/hadoop/hbase/client/AsyncAdmin.java | 2 -
.../hadoop/hbase/client/AsyncConnection.java| 4 +-
.../apache/hadoop/hbase/client/AsyncTable.java | 2 -
.../hadoop/hbase/client/AsyncTableBase.java | 2 -
.../hadoop/hbase/client/AsyncTableBuilder.java | 2 -
.../hbase/client/AsyncTableRegionLocator.java | 2 -
.../apache/hadoop/hbase/client/Attributes.java | 2 -
.../hadoop/hbase/client/BufferedMutator.java| 3 -
.../hbase/client/BufferedMutatorParams.java | 2 -
.../apache/hadoop/hbase/client/CompactType.java | 4 +-
.../hadoop/hbase/client/CompactionState.java| 2 -
.../apache/hadoop/hbase/client/Connection.java | 2 -
.../hadoop/hbase/client/ConnectionFactory.java | 2 -
.../apache/hadoop/hbase/client/Consistency.java | 2 -
.../org/apache/hadoop/hbase/client/Delete.java | 2 -
.../hbase/client/DoNotRetryRegionException.java | 2 -
.../apache/hadoop/hbase/client/Durability.java | 2 -
.../org/apache/hadoop/hbase/client/Get.java | 2 -
.../hadoop/hbase/client/HTableMultiplexer.java | 3 -
.../apache/hadoop/hbase/client/Increment.java | 2 -
.../hadoop/hbase/client/IsolationLevel.java | 2 -
.../hadoop/hbase/client/MasterSwitchType.java | 4 +-
.../hbase/client/MobCompactPartitionPolicy.java | 2 -
.../apache/hadoop/hbase/client/Mutation.java| 2 -
.../client/NoServerForRegionException.java | 2 -
.../apache/hadoop/hbase/client/Operation.java | 2 -
.../hbase/client/OperationWithAttributes.java | 2 -
.../org/apache/hadoop/hbase/client/Put.java | 2 -
.../org/apache/hadoop/hbase/client/Query.java | 4 +-
.../hadoop/hbase/client/RawAsyncTable.java | 4 -
.../hbase/client/RawScanResultConsumer.java | 6 +-
.../hadoop/hbase/client/RegionLoadStats.java| 2 -
.../hadoop/hbase/client/RegionLocator.java | 2 -
.../hbase/client/RegionOfflineException.java| 2 -
.../hadoop/hbase/client/RequestController.java | 4 -
.../hbase/client/RequestControllerFactory.java | 2 -
.../org/apache/hadoop/hbase/client/Result.java | 2 -
.../hadoop/hbase/client/ResultScanner.java | 2 -
.../hbase/client/RetriesExhaustedException.java |
http://git-wip-us.apache.org/repos/asf/hbase/blob/5def44f5/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index 01ba8f6..71980b1 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -239,12 +239,20 @@ public final class QuotaProtos {
* THROTTLE = 1;
*/
THROTTLE(1),
+/**
+ * SPACE = 2;
+ */
+SPACE(2),
;
/**
* THROTTLE = 1;
*/
public static final int THROTTLE_VALUE = 1;
+/**
+ * SPACE = 2;
+ */
+public static final int SPACE_VALUE = 2;
public final int getNumber() {
@@ -262,6 +270,7 @@ public final class QuotaProtos {
public static QuotaType forNumber(int value) {
switch (value) {
case 1: return THROTTLE;
+case 2: return SPACE;
default: return null;
}
}
@@ -311,6 +320,150 @@ public final class QuotaProtos {
// @@protoc_insertion_point(enum_scope:hbase.pb.QuotaType)
}
+ /**
+ *
+ * Defines what action should be taken when the SpaceQuota is violated
+ *
+ *
+ * Protobuf enum {@code hbase.pb.SpaceViolationPolicy}
+ */
+ public enum SpaceViolationPolicy
+ implements
org.apache.hadoop.hbase.shaded.com.google.protobuf.ProtocolMessageEnum {
+/**
+ *
+ * Disable the table(s)
+ *
+ *
+ * DISABLE = 1;
+ */
+DISABLE(1),
+/**
+ *
+ * No writes, bulk-loads, or compactions
+ *
+ *
+ * NO_WRITES_COMPACTIONS = 2;
+ */
+NO_WRITES_COMPACTIONS(2),
+/**
+ *
+ * No writes or bulk-loads
+ *
+ *
+ * NO_WRITES = 3;
+ */
+NO_WRITES(3),
+/**
+ *
+ * No puts or bulk-loads, but deletes are allowed
+ *
+ *
+ * NO_INSERTS = 4;
+ */
+NO_INSERTS(4),
+;
+
+/**
+ *
+ * Disable the table(s)
+ *
+ *
+ * DISABLE = 1;
+ */
+public static final int DISABLE_VALUE = 1;
+/**
+ *
+ * No writes, bulk-loads, or compactions
+ *
+ *
+ * NO_WRITES_COMPACTIONS = 2;
+ */
+public static final int NO_WRITES_COMPACTIONS_VALUE = 2;
+/**
+ *
+ * No writes or bulk-loads
+ *
+ *
+ * NO_WRITES = 3;
+ */
+public static final int NO_WRITES_VALUE = 3;
+/**
+ *
+ * No puts or bulk-loads, but deletes are allowed
+ *
+ *
+ * NO_INSERTS = 4;
+ */
+public static final int NO_INSERTS_VALUE = 4;
+
+
+public final int getNumber() {
+ return value;
+}
+
+/**
+ * @deprecated Use {@link #forNumber(int)} instead.
+ */
+@java.lang.Deprecated
+public static SpaceViolationPolicy valueOf(int value) {
+ return forNumber(value);
+}
+
+public static SpaceViolationPolicy forNumber(int value) {
+ switch (value) {
+case 1: return DISABLE;
+case 2: return NO_WRITES_COMPACTIONS;
+case 3: return NO_WRITES;
+case 4: return NO_INSERTS;
+default: return null;
+ }
+}
+
+public static
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap
+internalGetValueMap() {
+ return internalValueMap;
+}
+private static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap<
+SpaceViolationPolicy> internalValueMap =
+ new
org.apache.hadoop.hbase.shaded.com.google.protobuf.Internal.EnumLiteMap()
{
+public SpaceViolationPolicy findValueByNumber(int number) {
+ return SpaceViolationPolicy.forNumber(number);
+}
+ };
+
+public final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumValueDescriptor
+getValueDescriptor() {
+ return getDescriptor().getValues().get(ordinal());
+}
+public final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptorForType() {
+ return getDescriptor();
+}
+public static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptor() {
+ return
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.getDescriptor().getEnumTypes().get(3);
+}
+
+private static final SpaceViolationPolicy[] VALUES = values();
+
+public static SpaceViolationPolicy valueOf(
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.EnumValueDescriptor
desc) {
+ if (desc.getType() != getDescriptor()) {
+throw new java.lang.IllegalAr
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotException.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotException.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotException.java
index 05f3556..f6817e7 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotException.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/ExportSnapshotException.java
@@ -18,13 +18,11 @@
package org.apache.hadoop.hbase.snapshot;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
/**
* Thrown when a snapshot could not be exported due to an error during the
operation.
*/
@InterfaceAudience.Public
-@InterfaceStability.Stable
@SuppressWarnings("serial")
public class ExportSnapshotException extends HBaseSnapshotException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java
index 2fe58ed..bd185a1 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/HBaseSnapshotException.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hbase.snapshot;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.client.SnapshotDescription;
/**
@@ -27,7 +26,6 @@ import org.apache.hadoop.hbase.client.SnapshotDescription;
*/
@SuppressWarnings("serial")
@InterfaceAudience.Public
-@InterfaceStability.Evolving
public class HBaseSnapshotException extends DoNotRetryIOException {
private SnapshotDescription description;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java
index 70e8d3b..de58077 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotException.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hbase.snapshot;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.client.SnapshotDescription;
/**
@@ -27,7 +26,6 @@ import org.apache.hadoop.hbase.client.SnapshotDescription;
*/
@SuppressWarnings("serial")
@InterfaceAudience.Public
-@InterfaceStability.Evolving
public class RestoreSnapshotException extends HBaseSnapshotException {
public RestoreSnapshotException(String msg, SnapshotDescription desc) {
super(msg, desc);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java
index 2738b3d..9cfe83a 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCreationException.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.snapshot;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.client.SnapshotDescription;
/**
@@ -27,7 +26,6 @@ import org.apache.hadoop.hbase.client.SnapshotDescription;
*/
@SuppressWarnings("serial")
@InterfaceAudience.Public
-@InterfaceStability.Evolving
public class SnapshotCreationException extends HBaseSnapshotException {
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-client/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotDoesNotExistException.java
--
diff --git
a/hbase-client/src/main/java/or
http://git-wip-us.apache.org/repos/asf/hbase/blob/a1ffeea4/hbase-protocol-shaded/src/main/protobuf/Admin.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/Admin.proto
b/hbase-protocol-shaded/src/main/protobuf/Admin.proto
index 338c80b..1a085e6 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Admin.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Admin.proto
@@ -28,6 +28,7 @@ option optimize_for = SPEED;
import "ClusterStatus.proto";
import "HBase.proto";
import "WAL.proto";
+import "Quota.proto";
message GetRegionInfoRequest {
required RegionSpecifier region = 1;
@@ -314,4 +315,12 @@ service AdminService {
rpc GetRegionLoad(GetRegionLoadRequest)
returns(GetRegionLoadResponse);
+
+ /** Fetches the RegionServer's view of space quotas */
+ rpc GetSpaceQuotaSnapshots(GetSpaceQuotaSnapshotsRequest)
+returns(GetSpaceQuotaSnapshotsResponse);
+
+ /** Fetches the RegionServer's space quota active enforcements */
+ rpc GetSpaceQuotaEnforcements(GetSpaceQuotaEnforcementsRequest)
+returns(GetSpaceQuotaEnforcementsResponse);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a1ffeea4/hbase-protocol-shaded/src/main/protobuf/Master.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto
b/hbase-protocol-shaded/src/main/protobuf/Master.proto
index 4e856c8..58e6f77 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Master.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto
@@ -929,4 +929,8 @@ service MasterService {
/** Unmark a list of ServerNames marked as draining. */
rpc removeDrainFromRegionServers(RemoveDrainFromRegionServersRequest)
returns(RemoveDrainFromRegionServersResponse);
+
+ /** Fetches the Master's view of space quotas */
+ rpc GetSpaceQuotaRegionSizes(GetSpaceQuotaRegionSizesRequest)
+returns(GetSpaceQuotaRegionSizesResponse);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a1ffeea4/hbase-protocol-shaded/src/main/protobuf/Quota.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
index 597b059..2d7e5f5 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
@@ -111,3 +111,38 @@ message SpaceQuotaSnapshot {
optional uint64 usage = 2;
optional uint64 limit = 3;
}
+
+message GetSpaceQuotaRegionSizesRequest {
+}
+
+message GetSpaceQuotaRegionSizesResponse {
+ message RegionSizes {
+optional TableName table_name = 1;
+optional uint64 size = 2;
+ }
+ repeated RegionSizes sizes = 1;
+}
+
+message GetSpaceQuotaSnapshotsRequest {
+}
+
+message GetSpaceQuotaSnapshotsResponse {
+ // Cannot use TableName as a map key, do the repeated nested message by hand.
+ message TableQuotaSnapshot {
+optional TableName table_name = 1;
+optional SpaceQuotaSnapshot snapshot = 2;
+ }
+ repeated TableQuotaSnapshot snapshots = 1;
+}
+
+message GetSpaceQuotaEnforcementsRequest {
+}
+
+message GetSpaceQuotaEnforcementsResponse {
+ // Cannot use TableName as a map key, do the repeated nested message by hand.
+ message TableViolationPolicy {
+optional TableName table_name = 1;
+optional SpaceViolationPolicy violation_policy = 2;
+ }
+ repeated TableViolationPolicy violation_policies = 1;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a1ffeea4/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index a7a2f94..53560e9 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -21,8 +21,11 @@ package org.apache.hadoop.hbase.master;
import java.io.IOException;
import java.net.InetAddress;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import org.apache.commons.logging.Log;
@@ -211,6 +214,9 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTa
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.gene
HBASE-16999 Implement master and regionserver synchronization of quota state
* Implement the RegionServer reading violation from the quota table
* Implement the Master reporting violations to the quota table
* RegionServers need to track its enforced policies
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4a715450
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4a715450
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4a715450
Branch: refs/heads/HBASE-16961
Commit: 4a7154502f8fd28af0fa4480aa3707a0a8459519
Parents: acf2153
Author: Josh Elser
Authored: Fri Nov 18 15:38:19 2016 -0500
Committer: Josh Elser
Committed: Tue Apr 11 13:10:02 2017 -0400
--
.../hadoop/hbase/quotas/QuotaTableUtil.java | 92 -
.../org/apache/hadoop/hbase/master/HMaster.java | 35 +++-
.../hadoop/hbase/quotas/QuotaObserverChore.java | 5 +-
.../hbase/quotas/RegionServerQuotaManager.java | 200 ---
.../quotas/RegionServerRpcQuotaManager.java | 200 +++
.../quotas/RegionServerSpaceQuotaManager.java | 169
.../quotas/SpaceQuotaViolationNotifier.java | 16 +-
.../SpaceQuotaViolationNotifierFactory.java | 62 ++
.../SpaceQuotaViolationNotifierForTest.java | 4 +
...SpaceQuotaViolationPolicyRefresherChore.java | 154 ++
.../TableSpaceQuotaViolationNotifier.java | 55 +
.../hbase/regionserver/HRegionServer.java | 21 +-
.../hbase/regionserver/RSRpcServices.java | 7 +-
.../regionserver/RegionServerServices.java | 12 +-
.../hadoop/hbase/MockRegionServerServices.java | 10 +-
.../hadoop/hbase/master/MockRegionServer.java | 10 +-
.../TestQuotaObserverChoreWithMiniCluster.java | 2 +
.../hadoop/hbase/quotas/TestQuotaTableUtil.java | 47 +
.../hadoop/hbase/quotas/TestQuotaThrottle.java | 4 +-
.../TestRegionServerSpaceQuotaManager.java | 127
...SpaceQuotaViolationPolicyRefresherChore.java | 131
.../TestTableSpaceQuotaViolationNotifier.java | 144 +
22 files changed, 1281 insertions(+), 226 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/4a715450/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index 1640ddc..505e94b 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -23,16 +23,20 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
@@ -43,7 +47,12 @@ import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.filter.RegexStringComparator;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.protobuf.ProtobufMagic;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString;
+import
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Strings;
@@ -52,9 +61,8 @@ import org.apache.hadoop.hbase.util.Strings;
*
* ROW-KEY FAM/QUALDATA
* n. q:s
- * n. u:du
* t. q:s
- * t. u:du
+ * t. u:v
* u. q:s
* u
HBASE-17003 Documentation updates for space quotas
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/95603567
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/95603567
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/95603567
Branch: refs/heads/HBASE-16961
Commit: 95603567a180f5c9a7d3e126435728cb1b561305
Parents: 51adf1e
Author: Josh Elser
Authored: Thu Mar 16 16:21:14 2017 -0400
Committer: Josh Elser
Committed: Tue Apr 11 13:10:03 2017 -0400
--
src/main/asciidoc/_chapters/ops_mgt.adoc | 64 ++-
1 file changed, 63 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/95603567/src/main/asciidoc/_chapters/ops_mgt.adoc
--
diff --git a/src/main/asciidoc/_chapters/ops_mgt.adoc
b/src/main/asciidoc/_chapters/ops_mgt.adoc
index e4c077f..f9009f3 100644
--- a/src/main/asciidoc/_chapters/ops_mgt.adoc
+++ b/src/main/asciidoc/_chapters/ops_mgt.adoc
@@ -1705,7 +1705,7 @@ handling multiple workloads:
[[quota]]
=== Quotas
-HBASE-11598 introduces quotas, which allow you to throttle requests based on
+HBASE-11598 introduces RPC quotas, which allow you to throttle requests based
on
the following limits:
. <>
@@ -1885,6 +1885,68 @@ at the same time and that fewer scans can be executed at
the same time. A value
`0.9` will give more queue/handlers to scans, so the number of scans executed
will
increase and the number of gets will decrease.
+[[space-quotas]]
+=== Space Quotas
+
+link:https://issues.apache.org/jira/browse/HBASE-16961[HBASE-16961] introduces
a new type of
+quotas for HBase to leverage: filesystem quotas. These "space" quotas limit
the amount of space
+on the filesystem that HBase namespaces and tables can consume. If a user,
malicious or ignorant,
+has the ability to write data into HBase, with enough time, that user can
effectively crash HBase
+(or worse HDFS) by consuming all available space. When there is no filesystem
space available,
+HBase crashes because it can no longer create/sync data to the write-ahead log.
+
+This feature allows a for a limit to be set on the size of a table or
namespace. When a space quota is set
+on a namespace, the quota's limit applies to the sum of usage of all tables in
that namespace.
+When a table with a quota exists in a namespace with a quota, the table quota
takes priority
+over the namespace quota. This allows for a scenario where a large limit can
be placed on
+a collection of tables, but a single table in that collection can have a
fine-grained limit set.
+
+The existing `set_quota` and `list_quota` HBase shell commands can be used to
interact with
+space quotas. Space quotas are quotas with a `TYPE` of `SPACE` and have
`LIMIT` and `POLICY`
+attributes. The `LIMIT` is a string that refers to the amount of space on the
filesystem
+that the quota subject (e.g. the table or namespace) may consume. For example,
valid values
+of `LIMIT` are `'10G'`, `'2T'`, or `'256M'`. The `POLICY` refers to the action
that HBase will
+take when the quota subject's usage exceeds the `LIMIT`. The following are
valid `POLICY` values.
+
+* `NO_INSERTS` - No new data may be written (e.g. `Put`, `Increment`,
`Append`).
+* `NO_WRITES` - Same as `NO_INSERTS` but `Deletes` are also disallowed.
+* `NO_WRITES_COMPACTIONS` - Same as `NO_WRITES` but compactions are also
disallowed.
+* `DISABLE` - The table(s) are disabled, preventing all read/write access.
+
+.Setting simple space quotas
+
+# Sets a quota on the table 't1' with a limit of 1GB, disallowing
Puts/Increments/Appends when the table exceeds 1GB
+hbase> set_quota TYPE => SPACE, TABLE => 't1', LIMIT => '1G', POLICY =>
NO_INSERTS
+
+# Sets a quota on the namespace 'ns1' with a limit of 50TB, disallowing
Puts/Increments/Appends/Deletes
+hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => '50T', POLICY =>
NO_WRITES
+
+# Sets a quota on the table 't3' with a limit of 2TB, disallowing any writes
and compactions when the table exceeds 2TB.
+hbase> set_quota TYPE => SPACE, TABLE => 't3', LIMIT => '2T', POLICY =>
NO_WRITES_COMPACTIONS
+
+# Sets a quota on the table 't2' with a limit of 50GB, disabling the table
when it exceeds 50GB
+hbase> set_quota TYPE => SPACE, TABLE => 't2', LIMIT => '50G', POLICY =>
DISABLE
+
+
+Consider the following scenario to set up quotas on a namespace, overriding
the quota on tables in that namespace
+
+.Table and Namespace space quotas
+
+hbase> create_namespace 'ns1'
+hbase> create 'ns1:t1'
+hbase> create 'ns1:t2'
+hbase> create 'ns1:t3'
+hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => '100T', POLICY =>
NO_INSERTS
+hbase> set_quota TYPE => SPACE, TABLE => 'ns1:t2', LIMIT => '200G', POLICY =>
NO_WRITES
+h
HBASE-17000 Implement computation of online region sizes and report to the
Master
Includes a trivial implementation of the Master-side collection to
avoid. Only enough to write a test to verify RS collection.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2a21313a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2a21313a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2a21313a
Branch: refs/heads/HBASE-16961
Commit: 2a21313aac9a5a9e25b454949b08973bea650291
Parents: 76f3e7f
Author: Josh Elser
Authored: Mon Nov 7 13:46:42 2016 -0500
Committer: Josh Elser
Committed: Tue Apr 11 13:10:02 2017 -0400
--
.../generated/RegionServerStatusProtos.java | 2071 +-
.../src/main/protobuf/RegionServerStatus.proto | 22 +
.../hadoop/hbase/master/MasterRpcServices.java | 19 +
.../quotas/FileSystemUtilizationChore.java | 205 ++
.../hadoop/hbase/quotas/MasterQuotaManager.java | 15 +
.../hbase/regionserver/HRegionServer.java | 72 +
.../quotas/TestFileSystemUtilizationChore.java | 357 +++
.../hadoop/hbase/quotas/TestRegionSizeUse.java | 194 ++
.../TestRegionServerRegionSpaceUseReport.java | 99 +
9 files changed, 3032 insertions(+), 22 deletions(-)
--
HBASE-16995 Build client Java API and client protobuf messages - addendum fixes
line lengths (Josh Elser)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/61301c47
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/61301c47
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/61301c47
Branch: refs/heads/HBASE-16961
Commit: 61301c47bc898650a48e7ac8e47930587f515367
Parents: 341b21a
Author: tedyu
Authored: Mon Nov 21 13:00:27 2016 -0800
Committer: Josh Elser
Committed: Tue Apr 11 13:10:02 2017 -0400
--
.../hbase/quotas/QuotaSettingsFactory.java | 20
.../hadoop/hbase/quotas/SpaceLimitSettings.java | 8
.../hbase/shaded/protobuf/ProtobufUtil.java | 9 +
3 files changed, 21 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/61301c47/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
index 8512e39..7f1c180 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
@@ -128,7 +128,8 @@ public class QuotaSettingsFactory {
static QuotaSettings fromSpace(TableName table, String namespace, SpaceQuota
protoQuota) {
if ((null == table && null == namespace) || (null != table && null !=
namespace)) {
- throw new IllegalArgumentException("Can only construct
SpaceLimitSettings for a table or namespace.");
+ throw new IllegalArgumentException(
+ "Can only construct SpaceLimitSettings for a table or namespace.");
}
if (null != table) {
return SpaceLimitSettings.fromSpaceQuota(table, protoQuota);
@@ -300,29 +301,32 @@ public class QuotaSettingsFactory {
*/
/**
- * Creates a {@link QuotaSettings} object to limit the FileSystem space
usage for the given table to the given size in bytes.
- * When the space usage is exceeded by the table, the provided {@link
SpaceViolationPolicy} is enacted on the table.
+ * Creates a {@link QuotaSettings} object to limit the FileSystem space
usage for the given table
+ * to the given size in bytes. When the space usage is exceeded by the
table, the provided
+ * {@link SpaceViolationPolicy} is enacted on the table.
*
* @param tableName The name of the table on which the quota should be
applied.
* @param sizeLimit The limit of a table's size in bytes.
* @param violationPolicy The action to take when the quota is exceeded.
* @return An {@link QuotaSettings} object.
*/
- public static QuotaSettings limitTableSpace(final TableName tableName, long
sizeLimit, final SpaceViolationPolicy violationPolicy) {
+ public static QuotaSettings limitTableSpace(
+ final TableName tableName, long sizeLimit, final SpaceViolationPolicy
violationPolicy) {
return new SpaceLimitSettings(tableName, sizeLimit, violationPolicy);
}
/**
- * Creates a {@link QuotaSettings} object to limit the FileSystem space
usage for the given namespace to the given size in bytes.
- * When the space usage is exceeded by all tables in the namespace, the
provided {@link SpaceViolationPolicy} is enacted on
- * all tables in the namespace.
+ * Creates a {@link QuotaSettings} object to limit the FileSystem space
usage for the given
+ * namespace to the given size in bytes. When the space usage is exceeded by
all tables in the
+ * namespace, the provided {@link SpaceViolationPolicy} is enacted on all
tables in the namespace.
*
* @param namespace The namespace on which the quota should be applied.
* @param sizeLimit The limit of the namespace's size in bytes.
* @param violationPolicy The action to take when the the quota is exceeded.
* @return An {@link QuotaSettings} object.
*/
- public static QuotaSettings limitNamespaceSpace(final String namespace, long
sizeLimit, final SpaceViolationPolicy violationPolicy) {
+ public static QuotaSettings limitNamespaceSpace(
+ final String namespace, long sizeLimit, final SpaceViolationPolicy
violationPolicy) {
return new SpaceLimitSettings(namespace, sizeLimit, violationPolicy);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/61301c47/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/SpaceLimitSettings.java
b/hbase-client/src/main/java/org/apa
HBASE-17816 HRegion#mutateRowWithLocks should update writeRequestCount metric
(Weizhan Zeng)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/48b2502a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/48b2502a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/48b2502a
Branch: refs/heads/HBASE-16961
Commit: 48b2502a5fcd4d3cd954c3abf6703422da7cdc2f
Parents: af604f0
Author: Jerry He
Authored: Thu Apr 6 16:45:45 2017 -0700
Committer: Jerry He
Committed: Thu Apr 6 16:45:45 2017 -0700
--
.../hadoop/hbase/regionserver/HRegion.java | 1 +
.../hadoop/hbase/regionserver/TestHRegion.java | 24
2 files changed, 25 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/48b2502a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 7f889ce..a87b679 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -6966,6 +6966,7 @@ public class HRegion implements HeapSize,
PropagatingConfigurationObserver, Regi
@Override
public void mutateRowsWithLocks(Collection mutations,
Collection rowsToLock, long nonceGroup, long nonce) throws
IOException {
+writeRequestsCount.add(mutations.size());
MultiRowMutationProcessor proc = new MultiRowMutationProcessor(mutations,
rowsToLock);
processRowsWithLocks(proc, -1, nonceGroup, nonce);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/48b2502a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
index eac3c77..d56d6ec 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
@@ -6391,4 +6391,28 @@ public class TestHRegion {
this.region = null;
}
}
+
+ @Test
+ public void testMutateRow_WriteRequestCount() throws Exception {
+byte[] row1 = Bytes.toBytes("row1");
+byte[] fam1 = Bytes.toBytes("fam1");
+byte[] qf1 = Bytes.toBytes("qualifier");
+byte[] val1 = Bytes.toBytes("value1");
+
+RowMutations rm = new RowMutations(row1);
+Put put = new Put(row1);
+put.addColumn(fam1, qf1, val1);
+rm.add(put);
+
+this.region = initHRegion(tableName, method, CONF, fam1);
+try {
+ long wrcBeforeMutate = this.region.writeRequestsCount.longValue();
+ this.region.mutateRow(rm);
+ long wrcAfterMutate = this.region.writeRequestsCount.longValue();
+ Assert.assertEquals(wrcBeforeMutate + rm.getMutations().size(),
wrcAfterMutate);
+} finally {
+ HBaseTestingUtility.closeRegionAndWAL(this.region);
+ this.region = null;
+}
+ }
}
HBASE-16996 Implement storage/retrieval of filesystem-use quotas into quota
table (Josh Elser)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/76f3e7fb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/76f3e7fb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/76f3e7fb
Branch: refs/heads/HBASE-16961
Commit: 76f3e7fbd07f9db5b352cee28f3b08670cbccc3c
Parents: 61301c4
Author: tedyu
Authored: Sat Dec 3 14:30:48 2016 -0800
Committer: Josh Elser
Committed: Tue Apr 11 13:10:02 2017 -0400
--
.../hadoop/hbase/quotas/QuotaTableUtil.java | 13 +-
.../hadoop/hbase/quotas/MasterQuotaManager.java | 30 +
.../hadoop/hbase/quotas/TestQuotaAdmin.java | 125 ++-
3 files changed, 165 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/76f3e7fb/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index 116dd0c..1640ddc 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -52,7 +52,9 @@ import org.apache.hadoop.hbase.util.Strings;
*
* ROW-KEY FAM/QUALDATA
* n. q:s
+ * n. u:du
* t. q:s
+ * t. u:du
* u. q:s
* u. q:s.
* u. q:s.:
@@ -71,6 +73,7 @@ public class QuotaTableUtil {
protected static final byte[] QUOTA_FAMILY_USAGE = Bytes.toBytes("u");
protected static final byte[] QUOTA_QUALIFIER_SETTINGS = Bytes.toBytes("s");
protected static final byte[] QUOTA_QUALIFIER_SETTINGS_PREFIX =
Bytes.toBytes("s.");
+ protected static final byte[] QUOTA_QUALIFIER_DISKUSAGE =
Bytes.toBytes("du");
protected static final byte[] QUOTA_USER_ROW_KEY_PREFIX =
Bytes.toBytes("u.");
protected static final byte[] QUOTA_TABLE_ROW_KEY_PREFIX =
Bytes.toBytes("t.");
protected static final byte[] QUOTA_NAMESPACE_ROW_KEY_PREFIX =
Bytes.toBytes("n.");
@@ -298,11 +301,16 @@ public class QuotaTableUtil {
* Quotas protobuf helpers
*/
protected static Quotas quotasFromData(final byte[] data) throws IOException
{
+return quotasFromData(data, 0, data.length);
+ }
+
+ protected static Quotas quotasFromData(
+ final byte[] data, int offset, int length) throws IOException {
int magicLen = ProtobufMagic.lengthOfPBMagic();
-if (!ProtobufMagic.isPBMagicPrefix(data, 0, magicLen)) {
+if (!ProtobufMagic.isPBMagicPrefix(data, offset, magicLen)) {
throw new IOException("Missing pb magic prefix");
}
-return Quotas.parseFrom(new ByteArrayInputStream(data, magicLen,
data.length - magicLen));
+return Quotas.parseFrom(new ByteArrayInputStream(data, offset + magicLen,
length - magicLen));
}
protected static byte[] quotasToData(final Quotas data) throws IOException {
@@ -316,6 +324,7 @@ public class QuotaTableUtil {
boolean hasSettings = false;
hasSettings |= quotas.hasThrottle();
hasSettings |= quotas.hasBypassGlobals();
+hasSettings |= quotas.hasSpace();
return !hasSettings;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/76f3e7fb/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
index 5dab2e3..1469268 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.Quota
Repository: hbase
Updated Branches:
refs/heads/HBASE-16961 8271fd53f -> f46c16aa2 (forced update)
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiHFileOutputFormat.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiHFileOutputFormat.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiHFileOutputFormat.java
index dc2fc0d..3c90b59 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiHFileOutputFormat.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiHFileOutputFormat.java
@@ -16,7 +16,6 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -44,7 +43,6 @@ import com.google.common.annotations.VisibleForTesting;
*
*/
@InterfaceAudience.Public
-@InterfaceStability.Evolving
@VisibleForTesting
public class MultiHFileOutputFormat extends
FileOutputFormat {
private static final Log LOG =
LogFactory.getLog(MultiHFileOutputFormat.class);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormat.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormat.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormat.java
index 3099c0d..a8e6837 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormat.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormat.java
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Scan;
@@ -55,7 +54,6 @@ import org.apache.hadoop.hbase.client.Scan;
*
*/
@InterfaceAudience.Public
-@InterfaceStability.Evolving
public class MultiTableInputFormat extends MultiTableInputFormatBase implements
Configurable {
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
index 25ea047..e18b3aa 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatBase.java
@@ -25,7 +25,6 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.TableName;
@@ -54,7 +53,6 @@ import java.util.Iterator;
* filters etc. Subclasses may use other TableRecordReader implementations.
*/
@InterfaceAudience.Public
-@InterfaceStability.Evolving
public abstract class MultiTableInputFormatBase extends
InputFormat {
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableOutputFormat.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableOutputFormat.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableOutputFormat.java
index 7feb7a9..4cc784f 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableOutputFormat.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MultiTableOutputFormat.java
@@ -25,7 +25,6 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
@@ -61,7 +
HBASE-17886 Fix compatibility of ServerSideScanMetrics
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d7e3116a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d7e3116a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d7e3116a
Branch: refs/heads/HBASE-16961
Commit: d7e3116a1744057359ca48d94aa50d7fdf0db974
Parents: 17737b2
Author: Yu Li
Authored: Thu Apr 6 17:29:22 2017 +0800
Committer: Yu Li
Committed: Thu Apr 6 17:29:22 2017 +0800
--
.../hadoop/hbase/client/metrics/ServerSideScanMetrics.java | 5 +
1 file changed, 5 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d7e3116a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
index 8a96aeb..03764ed 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/metrics/ServerSideScanMetrics.java
@@ -49,6 +49,11 @@ public class ServerSideScanMetrics {
public static final String COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME =
"ROWS_SCANNED";
public static final String COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME =
"ROWS_FILTERED";
+ /** @deprecated Use {@link #COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME} instead */
+ public static final String COUNT_OF_ROWS_SCANNED_KEY =
COUNT_OF_ROWS_SCANNED_KEY_METRIC_NAME;
+ /** @deprecated Use {@link #COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME} instead
*/
+ public static final String COUNT_OF_ROWS_FILTERED_KEY =
COUNT_OF_ROWS_FILTERED_KEY_METRIC_NAME;
+
/**
* number of rows filtered during scan RPC
*/
HBASE-17478 Avoid reporting FS use when quotas are disabled
Also, gracefully produce responses when quotas are disabled.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/376ac5d2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/376ac5d2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/376ac5d2
Branch: refs/heads/HBASE-16961
Commit: 376ac5d29b9ac1388256e5387fcbe30492fef13a
Parents: bae29b6
Author: Josh Elser
Authored: Tue Jan 17 14:41:45 2017 -0500
Committer: Josh Elser
Committed: Tue Apr 11 13:10:02 2017 -0400
--
.../hadoop/hbase/master/MasterRpcServices.java | 4 +++
.../hadoop/hbase/quotas/MasterQuotaManager.java | 13 +--
.../hbase/regionserver/HRegionServer.java | 5 ++-
.../hbase/quotas/TestMasterQuotaManager.java| 37
4 files changed, 56 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/376ac5d2/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index f10d4bb..a7a2f94 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -59,6 +59,7 @@ import
org.apache.hadoop.hbase.procedure.MasterProcedureManager;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
+import org.apache.hadoop.hbase.quotas.QuotaUtil;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -2016,6 +2017,9 @@ public class MasterRpcServices extends RSRpcServices
RegionSpaceUseReportRequest request) throws ServiceException {
try {
master.checkInitialized();
+ if (!QuotaUtil.isQuotaEnabled(master.getConfiguration())) {
+return RegionSpaceUseReportResponse.newBuilder().build();
+ }
MasterQuotaManager quotaManager = this.master.getMasterQuotaManager();
for (RegionSpaceUse report : request.getSpaceUseList()) {
quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()),
report.getSize());
http://git-wip-us.apache.org/repos/asf/hbase/blob/376ac5d2/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
index a5832f9..cb614ea 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hbase.quotas;
import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -58,6 +59,8 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.TimedQuota;
@InterfaceStability.Evolving
public class MasterQuotaManager implements RegionStateListener {
private static final Log LOG = LogFactory.getLog(MasterQuotaManager.class);
+ private static final Map EMPTY_MAP =
Collections.unmodifiableMap(
+ new HashMap<>());
private final MasterServices masterServices;
private NamedLock namespaceLocks;
@@ -529,13 +532,19 @@ public class MasterQuotaManager implements
RegionStateListener {
}
public void addRegionSize(HRegionInfo hri, long size) {
-// TODO Make proper API
+if (null == regionSizes) {
+ return;
+}
+// TODO Make proper API?
// TODO Prevent from growing indefinitely
regionSizes.put(hri, size);
}
public Map snapshotRegionSizes() {
-// TODO Make proper API
+if (null == regionSizes) {
+ return EMPTY_MAP;
+}
+// TODO Make proper API?
return new HashMap<>(regionSizes);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/376ac5d2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 79e2c46..81c58c3 100644
---
a/hbase-server/src/mai
HBASE-17881 Remove the ByteBufferCellImpl
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/18c5ecf6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/18c5ecf6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/18c5ecf6
Branch: refs/heads/HBASE-16961
Commit: 18c5ecf6ed57e80b32568ca1a1a12c7af36bab46
Parents: 1a701ce
Author: Chia-Ping Tsai
Authored: Wed Apr 5 21:11:29 2017 +0800
Committer: Chia-Ping Tsai
Committed: Fri Apr 7 21:14:19 2017 +0800
--
.../hadoop/hbase/filter/TestComparators.java| 14 +-
.../hadoop/hbase/filter/TestKeyOnlyFilter.java | 4 +-
.../apache/hadoop/hbase/TestCellComparator.java | 7 +-
.../org/apache/hadoop/hbase/TestCellUtil.java | 184 +--
.../filter/TestSingleColumnValueFilter.java | 36 ++--
5 files changed, 36 insertions(+), 209 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/18c5ecf6/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestComparators.java
--
diff --git
a/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestComparators.java
b/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestComparators.java
index d9e4033..0c69ece 100644
---
a/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestComparators.java
+++
b/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestComparators.java
@@ -21,11 +21,11 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.nio.ByteBuffer;
+import org.apache.hadoop.hbase.ByteBufferKeyValue;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.TestCellUtil.ByteBufferCellImpl;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -51,18 +51,18 @@ public class TestComparators {
// Row compare
KeyValue kv = new KeyValue(r1, f, q1, v1);
ByteBuffer buffer = ByteBuffer.wrap(kv.getBuffer());
-Cell bbCell = new ByteBufferCellImpl(buffer, 0, buffer.remaining());
+Cell bbCell = new ByteBufferKeyValue(buffer, 0, buffer.remaining());
ByteArrayComparable comparable = new BinaryComparator(r1);
assertEquals(0, CellComparator.compareRow(bbCell, comparable));
assertEquals(0, CellComparator.compareRow(kv, comparable));
kv = new KeyValue(r0, f, q1, v1);
buffer = ByteBuffer.wrap(kv.getBuffer());
-bbCell = new ByteBufferCellImpl(buffer, 0, buffer.remaining());
+bbCell = new ByteBufferKeyValue(buffer, 0, buffer.remaining());
assertTrue(CellComparator.compareRow(bbCell, comparable) > 0);
assertTrue(CellComparator.compareRow(kv, comparable) > 0);
kv = new KeyValue(r2, f, q1, v1);
buffer = ByteBuffer.wrap(kv.getBuffer());
-bbCell = new ByteBufferCellImpl(buffer, 0, buffer.remaining());
+bbCell = new ByteBufferKeyValue(buffer, 0, buffer.remaining());
assertTrue(CellComparator.compareRow(bbCell, comparable) < 0);
assertTrue(CellComparator.compareRow(kv, comparable) < 0);
// Qualifier compare
@@ -71,12 +71,12 @@ public class TestComparators {
assertEquals(0, CellComparator.compareQualifier(kv, comparable));
kv = new KeyValue(r2, f, q2, v1);
buffer = ByteBuffer.wrap(kv.getBuffer());
-bbCell = new ByteBufferCellImpl(buffer, 0, buffer.remaining());
+bbCell = new ByteBufferKeyValue(buffer, 0, buffer.remaining());
assertEquals(0, CellComparator.compareQualifier(bbCell, comparable));
assertEquals(0, CellComparator.compareQualifier(kv, comparable));
kv = new KeyValue(r2, f, q3, v1);
buffer = ByteBuffer.wrap(kv.getBuffer());
-bbCell = new ByteBufferCellImpl(buffer, 0, buffer.remaining());
+bbCell = new ByteBufferKeyValue(buffer, 0, buffer.remaining());
assertTrue(CellComparator.compareQualifier(bbCell, comparable) < 0);
assertTrue(CellComparator.compareQualifier(kv, comparable) < 0);
// Value compare
@@ -85,7 +85,7 @@ public class TestComparators {
assertEquals(0, CellComparator.compareValue(kv, comparable));
kv = new KeyValue(r1, f, q1, v2);
buffer = ByteBuffer.wrap(kv.getBuffer());
-bbCell = new ByteBufferCellImpl(buffer, 0, buffer.remaining());
+bbCell = new ByteBufferKeyValue(buffer, 0, buffer.remaining());
assertTrue(CellComparator.compareValue(bbCell, comparable) < 0);
assertTrue(CellComparator.compareValue(kv, comparable) < 0);
// Family compare
http://git-wip-us.apache.org/repos/asf/hbase/blob/18c5ecf6/hbase-client/src/test/java/org/apache/hadoop/hbase/filter/TestKeyOnlyFilter.java
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index 52e35ab..f71699d 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -13024,6 +13024,3031 @@ public final class QuotaProtos {
}
+ public interface GetQuotaStatesRequestOrBuilder extends
+ //
@@protoc_insertion_point(interface_extends:hbase.pb.GetQuotaStatesRequest)
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
+ }
+ /**
+ * Protobuf type {@code hbase.pb.GetQuotaStatesRequest}
+ */
+ public static final class GetQuotaStatesRequest extends
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3
implements
+ //
@@protoc_insertion_point(message_implements:hbase.pb.GetQuotaStatesRequest)
+ GetQuotaStatesRequestOrBuilder {
+// Use GetQuotaStatesRequest.newBuilder() to construct.
+private
GetQuotaStatesRequest(org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.Builder
builder) {
+ super(builder);
+}
+private GetQuotaStatesRequest() {
+}
+
+@java.lang.Override
+public final
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+ return this.unknownFields;
+}
+private GetQuotaStatesRequest(
+org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream
input,
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite
extensionRegistry)
+throws
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
{
+ this();
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.Builder
unknownFields =
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+boolean done = false;
+while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+case 0:
+ done = true;
+ break;
+default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+ }
+ break;
+}
+ }
+}
+ } catch
(org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
e) {
+throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+throw new
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException(
+e).setUnfinishedMessage(this);
+ } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+ }
+}
+public static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+ return
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.internal_static_hbase_pb_GetQuotaStatesRequest_descriptor;
+}
+
+protected
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+internalGetFieldAccessorTable() {
+ return
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.internal_static_hbase_pb_GetQuotaStatesRequest_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest.class,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest.Builder.class);
+}
+
+private byte memoizedIsInitialized = -1;
+public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized == 1) return true;
+ if (isInitialized == 0) return false;
+
+ memoizedIsInitialized = 1;
+ return true;
+}
+
+public void
writeTo(org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedOutputStream
output)
+throws java.io.IOException {
+ unknownFields.writeTo(output);
+}
+
+public int getSerializedSize() {
+ int size = memoizedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ size += unknownFields.getSerializedSize();
+ memoizedSize = size;
+ return size;
+}
+
+private static final long serialVersionUID = 0L;
+@java.lang.Override
+public boolean equals(final java.lang.Object obj) {
+ if (obj == this) {
+ return true;
+
HBASE-16995 Build client Java API and client protobuf messages - addendum fixes
white spaces (Josh Elser)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/341b21ad
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/341b21ad
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/341b21ad
Branch: refs/heads/HBASE-16961
Commit: 341b21ad7ba60e206868b40be8a73f217606420b
Parents: 5def44f
Author: tedyu
Authored: Thu Nov 17 10:42:18 2016 -0800
Committer: Josh Elser
Committed: Tue Apr 11 13:10:02 2017 -0400
--
.../hbase/quotas/TestQuotaSettingsFactory.java| 2 +-
.../shaded/protobuf/generated/MasterProtos.java | 2 +-
.../shaded/protobuf/generated/QuotaProtos.java| 18 +-
.../hbase/protobuf/generated/QuotaProtos.java | 4 ++--
4 files changed, 13 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/341b21ad/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
--
diff --git
a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
index 17015d6..e0012a7 100644
---
a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
+++
b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
@@ -44,7 +44,7 @@ import org.junit.experimental.categories.Category;
*/
@Category(SmallTests.class)
public class TestQuotaSettingsFactory {
-
+
@Test
public void testAllQuotasAddedToList() {
final SpaceQuota spaceQuota = SpaceQuota.newBuilder()
http://git-wip-us.apache.org/repos/asf/hbase/blob/341b21ad/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
index 45ff86d..3c78980 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
@@ -63752,7 +63752,7 @@ public final class MasterProtos {
* optional .hbase.pb.SpaceLimitRequest space_limit = 8;
*/
private
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequestOrBuilder>
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequestOrBuilder>
getSpaceLimitFieldBuilder() {
if (spaceLimitBuilder_ == null) {
spaceLimitBuilder_ = new
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
http://git-wip-us.apache.org/repos/asf/hbase/blob/341b21ad/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index 71980b1..4ceab07 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -4362,7 +4362,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuota space = 3;
*/
private
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
org.apache.hadoop.hbase.shaded.protobuf.gene
HBASE-17869 UnsafeAvailChecker wrongly returns false on ppc
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/af604f0c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/af604f0c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/af604f0c
Branch: refs/heads/HBASE-16961
Commit: af604f0c0cf3c40c56746150ffa860aad07f128a
Parents: 9109803
Author: Jerry He
Authored: Thu Apr 6 16:04:47 2017 -0700
Committer: Jerry He
Committed: Thu Apr 6 16:04:47 2017 -0700
--
.../hadoop/hbase/util/UnsafeAvailChecker.java | 24
1 file changed, 15 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/af604f0c/hbase-common/src/main/java/org/apache/hadoop/hbase/util/UnsafeAvailChecker.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/UnsafeAvailChecker.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/UnsafeAvailChecker.java
index 90e6ec8..886cb3c 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/UnsafeAvailChecker.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/UnsafeAvailChecker.java
@@ -51,15 +51,21 @@ public class UnsafeAvailChecker {
});
// When Unsafe itself is not available/accessible consider unaligned as
false.
if (avail) {
- try {
-// Using java.nio.Bits#unaligned() to check for unaligned-access
capability
-Class clazz = Class.forName("java.nio.Bits");
-Method m = clazz.getDeclaredMethod("unaligned");
-m.setAccessible(true);
-unaligned = (Boolean) m.invoke(null);
- } catch (Exception e) {
-LOG.warn("java.nio.Bits#unaligned() check failed."
-+ "Unsafe based read/write of primitive types won't be used", e);
+ String arch = System.getProperty("os.arch");
+ if ("ppc64".equals(arch) || "ppc64le".equals(arch)) {
+// java.nio.Bits.unaligned() wrongly returns false on ppc
(JDK-8165231),
+unaligned = true;
+ } else {
+try {
+ // Using java.nio.Bits#unaligned() to check for unaligned-access
capability
+ Class clazz = Class.forName("java.nio.Bits");
+ Method m = clazz.getDeclaredMethod("unaligned");
+ m.setAccessible(true);
+ unaligned = (Boolean) m.invoke(null);
+} catch (Exception e) {
+ LOG.warn("java.nio.Bits#unaligned() check failed."
+ + "Unsafe based read/write of primitive types won't be used", e);
+}
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f42b498/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
index c493b25..943c898 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -22,16 +22,12 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.IOException;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -40,20 +36,15 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.NamespaceNotFoundException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.quotas.QuotaObserverChore.TablesWithQuotas;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -62,7 +53,6 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
-import com.google.common.collect.HashMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
@@ -72,11 +62,8 @@ import com.google.common.collect.Multimap;
@Category(LargeTests.class)
public class TestQuotaObserverChoreWithMiniCluster {
private static final Log LOG =
LogFactory.getLog(TestQuotaObserverChoreWithMiniCluster.class);
- private static final int SIZE_PER_VALUE = 256;
- private static final String F1 = "f1";
private static final HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
private static final AtomicLong COUNTER = new AtomicLong(0);
- private static final long ONE_MEGABYTE = 1024L * 1024L;
private static final long DEFAULT_WAIT_MILLIS = 500;
@Rule
@@ -84,18 +71,19 @@ public class TestQuotaObserverChoreWithMiniCluster {
private HMaster master;
private QuotaObserverChore chore;
- private SpaceQuotaViolationNotifierForTest violationNotifier;
+ private SpaceQuotaSnapshotNotifierForTest snapshotNotifier;
+ private SpaceQuotaHelperForTests helper;
@BeforeClass
public static void setUp() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY,
1000);
conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY,
1000);
-conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_DELAY_KEY, 1000);
-conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_PERIOD_KEY, 1000);
+conf.setInt(QuotaObserverChore.QUOTA_OBSERVER_CHORE_DELAY_KEY, 1000);
+conf.setInt(QuotaObserverChore.QUOTA_OBSERVER_CHORE_PERIOD_KEY, 1000);
conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
-conf.setClass(SpaceQuotaViolationNotifierFactory.VIOLATION_NOTIFIER_KEY,
-SpaceQuotaViolationNotifierForTest.class,
SpaceQuotaViolationNotifier.class);
+conf.setClass(SpaceQuotaSnapshotNotifierFactory.SNAPSHOT_NOTIFIER_KEY,
+SpaceQuotaSnapshotNotifierForTest.class,
SpaceQuotaSnapshotNotifier.class);
TEST_UTIL.startMiniCluster(1);
}
@@ -131,40 +119,55 @@ public class TestQuotaObserverChoreWithMiniCluster {
}
master = TEST_UTIL.getMiniHBaseCluster().getMaster();
-violationNotifier =
-(SpaceQuotaViolationNotifierForTest)
master.getSpaceQuotaViolationNotifier();
-violationNotifier.clearTableViolations();
+snapshotNotifier =
+(SpaceQuotaSnapshotNotifierForTest)
master.getSpaceQuotaSnapshotNotifier();
+snapshotNotifier.clearSnapshots();
chore = master.getQuotaObserverChore();
+helpe
HBASE-17002 JMX metrics and some UI additions for space quotas
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/51adf1e8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/51adf1e8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/51adf1e8
Branch: refs/heads/HBASE-16961
Commit: 51adf1e81f56b2e0cf1a25922554cbcb0a9c67cf
Parents: 39d1eca
Author: Josh Elser
Authored: Wed Feb 15 14:24:57 2017 -0500
Committer: Josh Elser
Committed: Tue Apr 11 13:10:03 2017 -0400
--
.../hbase/client/ConnectionImplementation.java |8 +
.../hadoop/hbase/client/QuotaStatusCalls.java | 39 +-
.../client/ShortCircuitMasterConnection.java|8 +
.../hadoop/hbase/quotas/QuotaTableUtil.java | 41 +
.../hbase/shaded/protobuf/RequestConverter.java | 11 +
.../hbase/master/MetricsMasterQuotaSource.java | 75 +
.../master/MetricsMasterQuotaSourceFactory.java | 26 +
.../hbase/master/MetricsMasterWrapper.java | 13 +
.../MetricsRegionServerQuotaSource.java | 54 +
.../MetricsMasterQuotaSourceFactoryImpl.java| 36 +
.../master/MetricsMasterQuotaSourceImpl.java| 129 +
...hadoop.hbase.master.MetricsMasterQuotaSource | 18 +
...hbase.master.MetricsMasterQuotaSourceFactory | 18 +
.../shaded/protobuf/generated/MasterProtos.java | 93 +-
.../shaded/protobuf/generated/QuotaProtos.java | 3099 +-
.../src/main/protobuf/Master.proto |6 +-
.../src/main/protobuf/Quota.proto | 17 +
.../org/apache/hadoop/hbase/master/HMaster.java |2 +-
.../hadoop/hbase/master/MasterRpcServices.java | 38 +
.../hadoop/hbase/master/MetricsMaster.java | 42 +
.../hbase/master/MetricsMasterWrapperImpl.java | 42 +-
.../hadoop/hbase/quotas/QuotaObserverChore.java | 92 +-
.../resources/hbase-webapps/master/table.jsp| 59 +
.../hbase/master/TestMasterMetricsWrapper.java | 17 +
.../hbase/quotas/TestQuotaStatusRPCs.java | 83 +
25 files changed, 4032 insertions(+), 34 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index 3f27e1c..d9219f3 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -94,6 +94,8 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCa
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest;
@@ -1740,6 +1742,12 @@ class ConnectionImplementation implements
ClusterConnection, Closeable {
throws ServiceException {
return stub.getSpaceQuotaRegionSizes(controller, request);
}
+
+ @Override
+ public GetQuotaStatesResponse getQuotaStates(
+ RpcController controller, GetQuotaStatesRequest request) throws
ServiceException {
+return stub.getQuotaStates(controller, request);
+ }
};
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
index f0f385d..af36d1e 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.Re
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f42b498/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index 8b127d9..973ac8c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -37,9 +37,8 @@ import
org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.quotas.QuotaViolationStore.ViolationState;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
import com.google.common.annotations.VisibleForTesting;
@@ -54,51 +53,51 @@ import com.google.common.collect.Multimap;
@InterfaceAudience.Private
public class QuotaObserverChore extends ScheduledChore {
private static final Log LOG = LogFactory.getLog(QuotaObserverChore.class);
- static final String VIOLATION_OBSERVER_CHORE_PERIOD_KEY =
- "hbase.master.quotas.violation.observer.chore.period";
- static final int VIOLATION_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; //
5 minutes in millis
+ static final String QUOTA_OBSERVER_CHORE_PERIOD_KEY =
+ "hbase.master.quotas.observer.chore.period";
+ static final int QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 5
minutes in millis
- static final String VIOLATION_OBSERVER_CHORE_DELAY_KEY =
- "hbase.master.quotas.violation.observer.chore.delay";
- static final long VIOLATION_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1
minute
+ static final String QUOTA_OBSERVER_CHORE_DELAY_KEY =
+ "hbase.master.quotas.observer.chore.delay";
+ static final long QUOTA_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1
minute
- static final String VIOLATION_OBSERVER_CHORE_TIMEUNIT_KEY =
- "hbase.master.quotas.violation.observer.chore.timeunit";
- static final String VIOLATION_OBSERVER_CHORE_TIMEUNIT_DEFAULT =
TimeUnit.MILLISECONDS.name();
+ static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_KEY =
+ "hbase.master.quotas.observer.chore.timeunit";
+ static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_DEFAULT =
TimeUnit.MILLISECONDS.name();
- static final String VIOLATION_OBSERVER_CHORE_REPORT_PERCENT_KEY =
- "hbase.master.quotas.violation.observer.report.percent";
- static final double VIOLATION_OBSERVER_CHORE_REPORT_PERCENT_DEFAULT= 0.95;
+ static final String QUOTA_OBSERVER_CHORE_REPORT_PERCENT_KEY =
+ "hbase.master.quotas.observer.report.percent";
+ static final double QUOTA_OBSERVER_CHORE_REPORT_PERCENT_DEFAULT= 0.95;
private final Connection conn;
private final Configuration conf;
private final MasterQuotaManager quotaManager;
/*
- * Callback that changes in quota violation are passed to.
+ * Callback that changes in quota snapshots are passed to.
*/
- private final SpaceQuotaViolationNotifier violationNotifier;
+ private final SpaceQuotaSnapshotNotifier snapshotNotifier;
/*
- * Preserves the state of quota violations for tables and namespaces
+ * Preserves the state of quota snapshots for tables and namespaces
*/
- private final Map tableQuotaViolationStates;
- private final Map namespaceQuotaViolationStates;
+ private final Map tableQuotaSnapshots;
+ private final Map namespaceQuotaSnapshots;
/*
- * Encapsulates logic for moving tables/namespaces into or out of quota
violation
+ * Encapsulates logic for tracking the state of a table/namespace WRT space
quotas
*/
- private QuotaViolationStore tableViolationStore;
- private QuotaViolationStore namespaceViolationStore;
+ private QuotaSnapshotStore tableSnapshotStore;
+ private QuotaSnapshotStore namespaceSnapshotStore;
public QuotaObserverChore(HMaster master) {
this(
master.getConnection(), master.getConfiguration(),
-master.getSpaceQuotaViolationNotifier(),
master.getMasterQuotaManager(),
+master.getSpaceQuotaSnapshotNotifier(), master.getMasterQuotaManager(),
master);
}
QuotaObserverChore(
- Connection conn, Configuration conf, SpaceQuotaViolationNotifier
violationNotifier,
+ Connection conn, Configuration conf, SpaceQuotaSnapshotNotifier
snapshotNotifier,
MasterQuotaManager quotaManager, Stoppable stopper) {
super(
QuotaObserverChore.class.getSimpleName(), stopper
HBASE-17568 Better handle stale/missing region size reports
* Expire region reports in the master after a timeout.
* Move regions in violation out of violation when insufficient
region size reports are observed.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/39d1eca0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/39d1eca0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/39d1eca0
Branch: refs/heads/HBASE-16961
Commit: 39d1eca0bfc57e6ab720911748315c2a44d2cbac
Parents: 10b004b
Author: Josh Elser
Authored: Fri Feb 3 16:33:47 2017 -0500
Committer: Josh Elser
Committed: Tue Apr 11 13:10:03 2017 -0400
--
.../hadoop/hbase/master/MasterRpcServices.java | 4 +-
.../hadoop/hbase/quotas/MasterQuotaManager.java | 86 ++-
.../hadoop/hbase/quotas/QuotaObserverChore.java | 53 -
.../hbase/quotas/TestMasterQuotaManager.java| 48 +++-
.../TestQuotaObserverChoreRegionReports.java| 233 +++
5 files changed, 412 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/39d1eca0/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 53560e9..eb2711c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -251,6 +251,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.Updat
import org.apache.hadoop.hbase.snapshot.ClientSnapshotDescriptionUtils;
import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.ForeignExceptionUtil;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.zookeeper.KeeperException;
@@ -2027,8 +2028,9 @@ public class MasterRpcServices extends RSRpcServices
return RegionSpaceUseReportResponse.newBuilder().build();
}
MasterQuotaManager quotaManager = this.master.getMasterQuotaManager();
+ final long now = EnvironmentEdgeManager.currentTime();
for (RegionSpaceUse report : request.getSpaceUseList()) {
-quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()),
report.getSize());
+quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()),
report.getSize(), now);
}
return RegionSpaceUseReportResponse.newBuilder().build();
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/39d1eca0/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
index cb614ea..0622dba 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
@@ -22,9 +22,12 @@ import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.DoNotRetryIOException;
@@ -47,6 +50,8 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.ThrottleRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.TimedQuota;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Master Quota Manager.
* It is responsible for initialize the quota table on the first-run and
@@ -68,7 +73,7 @@ public class MasterQuotaManager implements
RegionStateListener {
private NamedLock userLocks;
private boolean enabled = false;
private NamespaceAuditor namespaceQuotaManager;
- private ConcurrentHashMap regionSizes;
+ private ConcurrentHashMap
regionSizes;
public MasterQuotaManager(final MasterServices masterServices) {
this.masterServices = masterServices;
@@ -531,21 +536,88 @@ public class MasterQuotaManager implements
RegionStateListener {
}
}
- public void addRegionSize(HRe
http://git-wip-us.apache.org/repos/asf/hbase/blob/a1ffeea4/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index 957d261..52e35ab 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -7476,6 +7476,5554 @@ public final class QuotaProtos {
}
+ public interface GetSpaceQuotaRegionSizesRequestOrBuilder extends
+ //
@@protoc_insertion_point(interface_extends:hbase.pb.GetSpaceQuotaRegionSizesRequest)
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
+ }
+ /**
+ * Protobuf type {@code hbase.pb.GetSpaceQuotaRegionSizesRequest}
+ */
+ public static final class GetSpaceQuotaRegionSizesRequest extends
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3
implements
+ //
@@protoc_insertion_point(message_implements:hbase.pb.GetSpaceQuotaRegionSizesRequest)
+ GetSpaceQuotaRegionSizesRequestOrBuilder {
+// Use GetSpaceQuotaRegionSizesRequest.newBuilder() to construct.
+private
GetSpaceQuotaRegionSizesRequest(org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.Builder
builder) {
+ super(builder);
+}
+private GetSpaceQuotaRegionSizesRequest() {
+}
+
+@java.lang.Override
+public final
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+ return this.unknownFields;
+}
+private GetSpaceQuotaRegionSizesRequest(
+org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream
input,
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite
extensionRegistry)
+throws
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
{
+ this();
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.Builder
unknownFields =
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+boolean done = false;
+while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+case 0:
+ done = true;
+ break;
+default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+ }
+ break;
+}
+ }
+}
+ } catch
(org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
e) {
+throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+throw new
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException(
+e).setUnfinishedMessage(this);
+ } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+ }
+}
+public static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+ return
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.internal_static_hbase_pb_GetSpaceQuotaRegionSizesRequest_descriptor;
+}
+
+protected
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+internalGetFieldAccessorTable() {
+ return
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.internal_static_hbase_pb_GetSpaceQuotaRegionSizesRequest_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest.class,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest.Builder.class);
+}
+
+private byte memoizedIsInitialized = -1;
+public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized == 1) return true;
+ if (isInitialized == 0) return false;
+
+ memoizedIsInitialized = 1;
+ return true;
+}
+
+public void
writeTo(org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedOutputStream
output)
+throws java.io.IOException {
+ unknownFields.writeTo(output);
+}
+
+public int getSerializedSize() {
+ int size = memoizedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ size += unknownFields.getSerializedSize();
+ memoizedSize = size;
+ return size;
+}
+
+private static final long serialVersionUID = 0
http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-protocol-shaded/src/main/protobuf/Master.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto
b/hbase-protocol-shaded/src/main/protobuf/Master.proto
index 58e6f77..90a7f07 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Master.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto
@@ -930,7 +930,11 @@ service MasterService {
rpc removeDrainFromRegionServers(RemoveDrainFromRegionServersRequest)
returns(RemoveDrainFromRegionServersResponse);
- /** Fetches the Master's view of space quotas */
+ /** Fetches the Master's view of space utilization */
rpc GetSpaceQuotaRegionSizes(GetSpaceQuotaRegionSizesRequest)
returns(GetSpaceQuotaRegionSizesResponse);
+
+ /** Fetches the Master's view of quotas */
+ rpc GetQuotaStates(GetQuotaStatesRequest)
+returns(GetQuotaStatesResponse);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-protocol-shaded/src/main/protobuf/Quota.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
index 2d7e5f5..1a6d5ed 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
@@ -119,6 +119,7 @@ message GetSpaceQuotaRegionSizesResponse {
message RegionSizes {
optional TableName table_name = 1;
optional uint64 size = 2;
+
}
repeated RegionSizes sizes = 1;
}
@@ -146,3 +147,19 @@ message GetSpaceQuotaEnforcementsResponse {
}
repeated TableViolationPolicy violation_policies = 1;
}
+
+message GetQuotaStatesRequest {
+}
+
+message GetQuotaStatesResponse {
+ message TableQuotaSnapshot {
+optional TableName table_name = 1;
+optional SpaceQuotaSnapshot snapshot = 2;
+ }
+ message NamespaceQuotaSnapshot {
+optional string namespace = 1;
+optional SpaceQuotaSnapshot snapshot = 2;
+ }
+ repeated TableQuotaSnapshot table_snapshots = 1;
+ repeated NamespaceQuotaSnapshot ns_snapshots = 2;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 0243934..92c1461 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -908,7 +908,7 @@ public class HMaster extends HRegionServer implements
MasterServices {
// Create the quota snapshot notifier
spaceQuotaSnapshotNotifier = createQuotaSnapshotNotifier();
spaceQuotaSnapshotNotifier.initialize(getClusterConnection());
- this.quotaObserverChore = new QuotaObserverChore(this);
+ this.quotaObserverChore = new QuotaObserverChore(this,
getMasterMetrics());
// Start the chore to read the region FS space reports and act on them
getChoreService().scheduleChore(quotaObserverChore);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/51adf1e8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index eb2711c..1c4abc0 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -62,7 +62,9 @@ import
org.apache.hadoop.hbase.procedure.MasterProcedureManager;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
import org.apache.hadoop.hbase.quotas.QuotaUtil;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -214,8 +216,12 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTa
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.TruncateTableResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.UnassignRegionResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetQuotaStatesReques
HBASE-17428 Implement informational RPCs for space quotas
Create some RPCs that can expose the in-memory state that the
RegionServers and Master hold to drive the space quota "state machine".
Then, create some hbase shell commands to interact with those.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a1ffeea4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a1ffeea4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a1ffeea4
Branch: refs/heads/HBASE-16961
Commit: a1ffeea4aac904f449fc9cdb3a7a3f57c09052b3
Parents: 376ac5d
Author: Josh Elser
Authored: Tue Feb 21 15:36:39 2017 -0500
Committer: Josh Elser
Committed: Tue Apr 11 13:10:03 2017 -0400
--
.../hbase/client/ConnectionImplementation.java |9 +
.../hadoop/hbase/client/QuotaStatusCalls.java | 125 +
.../client/ShortCircuitMasterConnection.java|7 +
.../hadoop/hbase/quotas/QuotaTableUtil.java | 77 +
.../hbase/shaded/protobuf/RequestConverter.java | 33 +
.../shaded/protobuf/generated/AdminProtos.java | 394 +-
.../shaded/protobuf/generated/MasterProtos.java | 92 +-
.../shaded/protobuf/generated/QuotaProtos.java | 5686 +-
.../src/main/protobuf/Admin.proto |9 +
.../src/main/protobuf/Master.proto |4 +
.../src/main/protobuf/Quota.proto | 35 +
.../hadoop/hbase/master/MasterRpcServices.java | 40 +
.../hbase/quotas/ActivePolicyEnforcement.java |8 +
.../hbase/regionserver/RSRpcServices.java | 57 +
.../hadoop/hbase/master/MockRegionServer.java | 18 +
.../hbase/quotas/TestQuotaStatusRPCs.java | 192 +
hbase-shell/src/main/ruby/hbase/quotas.rb | 16 +
hbase-shell/src/main/ruby/shell.rb |3 +
.../ruby/shell/commands/list_quota_snapshots.rb | 59 +
.../shell/commands/list_quota_table_sizes.rb| 47 +
.../shell/commands/list_quota_violations.rb | 48 +
hbase-shell/src/test/ruby/hbase/quotas_test.rb | 24 -
.../test/ruby/hbase/quotas_test_no_cluster.rb | 69 +
23 files changed, 6899 insertions(+), 153 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a1ffeea4/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index 99feb14..3f27e1c 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -94,6 +94,8 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCa
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SecurityCapabilitiesResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetNormalizerRunningResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.AddReplicationPeerResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.ReplicationProtos.DisableReplicationPeerRequest;
@@ -1731,6 +1733,13 @@ class ConnectionImplementation implements
ClusterConnection, Closeable {
ListReplicationPeersRequest request) throws ServiceException {
return stub.listReplicationPeers(controller, request);
}
+
+ @Override
+ public GetSpaceQuotaRegionSizesResponse getSpaceQuotaRegionSizes(
+ RpcController controller, GetSpaceQuotaRegionSizesRequest request)
+ throws ServiceException {
+return stub.getSpaceQuotaRegionSizes(controller, request);
+ }
};
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a1ffeea4/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
new file mode 100644
index 000..f0f385d
--- /dev/null
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/QuotaStatusCalls.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundat
HBASE-17794 Swap "violation" for "snapshot" where appropriate
A couple of variables and comments in which violation is incorrectly
used to describe what the code is doing. This was a hold over from early
implementation -- need to scrub these out for clarity.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/59148f64
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/59148f64
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/59148f64
Branch: refs/heads/HBASE-16961
Commit: 59148f64e03f6f7f33309a926be9fb3a43125e80
Parents: 9560356
Author: Josh Elser
Authored: Thu Mar 16 19:26:14 2017 -0400
Committer: Josh Elser
Committed: Tue Apr 11 13:10:03 2017 -0400
--
.../java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java| 4 ++--
hbase-protocol-shaded/src/main/protobuf/Quota.proto| 2 +-
.../org/apache/hadoop/hbase/quotas/QuotaObserverChore.java | 6 +++---
.../apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java| 2 +-
4 files changed, 7 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/59148f64/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index ed5da95..725f170 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -227,7 +227,7 @@ public class QuotaTableUtil {
}
/**
- * Creates a {@link Scan} which returns only quota violations from the quota
table.
+ * Creates a {@link Scan} which returns only quota snapshots from the quota
table.
*/
public static Scan makeQuotaSnapshotScan() {
Scan s = new Scan();
@@ -245,7 +245,7 @@ public class QuotaTableUtil {
* will throw an {@link IllegalArgumentException}.
*
* @param result A row from the quota table.
- * @param snapshots A map of violations to add the result of this method
into.
+ * @param snapshots A map of snapshots to add the result of this method into.
*/
public static void extractQuotaSnapshot(
Result result, Map snapshots) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/59148f64/hbase-protocol-shaded/src/main/protobuf/Quota.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
index 1a6d5ed..364c58b 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Quota.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Quota.proto
@@ -98,7 +98,7 @@ message SpaceLimitRequest {
}
// Represents the state of a quota on a table. Either the quota is not in
violation
-// or it is in violatino there is a violation policy which should be in effect.
+// or it is in violation there is a violation policy which should be in effect.
message SpaceQuotaStatus {
optional SpaceViolationPolicy policy = 1;
optional bool in_violation = 2;
http://git-wip-us.apache.org/repos/asf/hbase/blob/59148f64/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index 94c5c87..254f2a1 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -532,9 +532,9 @@ public class QuotaObserverChore extends ScheduledChore {
}
/**
- * Stores the quota violation state for the given table.
+ * Stores the quota state for the given table.
*/
- void setTableQuotaViolation(TableName table, SpaceQuotaSnapshot snapshot) {
+ void setTableQuotaSnapshot(TableName table, SpaceQuotaSnapshot snapshot) {
this.tableQuotaSnapshots.put(table, snapshot);
}
@@ -552,7 +552,7 @@ public class QuotaObserverChore extends ScheduledChore {
}
/**
- * Stores the quota violation state for the given namespace.
+ * Stores the quota state for the given namespace.
*/
void setNamespaceQuotaSnapshot(String namespace, SpaceQuotaSnapshot
snapshot) {
this.namespaceQuotaSnapshots.put(namespace, snapshot);
http://git-wip-us.apache.org/repos/asf/hbase/blob/59148f64/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java
---
HBASE-17001 Enforce quota violation policies in the RegionServer
The nuts-and-bolts of filesystem quotas. The Master must inform
RegionServers of the violation of a quota by a table. The RegionServer
must apply the violation policy as configured. Need to ensure
that the proper interfaces exist to satisfy all necessary policies.
This required a massive rewrite of the internal tracking by
the general space quota feature. Instead of tracking "violations",
we need to start tracking "usage". This allows us to make the decision
at the RegionServer level as to when the files in a bulk load request
should be accept or rejected which ultimately lets us avoid bulk loads
dramatically exceeding a configured space quota.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7f42b498
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7f42b498
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7f42b498
Branch: refs/heads/HBASE-16961
Commit: 7f42b498c1cf8108f3eabd5b89b35e83443324f4
Parents: 4a71545
Author: Josh Elser
Authored: Thu Dec 15 13:27:56 2016 -0500
Committer: Josh Elser
Committed: Tue Apr 11 13:10:02 2017 -0400
--
.../hbase/quotas/QuotaExceededException.java|4 +
.../hadoop/hbase/quotas/QuotaTableUtil.java | 47 +-
.../hadoop/hbase/quotas/SpaceQuotaSnapshot.java | 192 +++
.../shaded/protobuf/generated/QuotaProtos.java | 1384 +-
.../src/main/protobuf/Quota.proto | 15 +
.../hbase/protobuf/generated/QuotaProtos.java | 1324 -
hbase-protocol/src/main/protobuf/Quota.proto| 15 +
.../org/apache/hadoop/hbase/master/HMaster.java | 29 +-
.../hbase/quotas/ActivePolicyEnforcement.java | 86 ++
.../quotas/NamespaceQuotaSnapshotStore.java | 127 ++
.../quotas/NamespaceQuotaViolationStore.java| 127 --
.../hadoop/hbase/quotas/QuotaObserverChore.java | 344 +++--
.../hadoop/hbase/quotas/QuotaSnapshotStore.java | 96 ++
.../hbase/quotas/QuotaViolationStore.java | 89 --
.../quotas/RegionServerSpaceQuotaManager.java | 179 ++-
.../hbase/quotas/SpaceLimitingException.java| 95 ++
.../hbase/quotas/SpaceQuotaRefresherChore.java | 225 +++
.../quotas/SpaceQuotaSnapshotNotifier.java | 45 +
.../SpaceQuotaSnapshotNotifierFactory.java | 62 +
.../quotas/SpaceQuotaViolationNotifier.java | 54 -
.../SpaceQuotaViolationNotifierFactory.java | 62 -
.../SpaceQuotaViolationNotifierForTest.java | 54 -
...SpaceQuotaViolationPolicyRefresherChore.java | 154 --
.../quotas/SpaceViolationPolicyEnforcement.java | 91 ++
.../SpaceViolationPolicyEnforcementFactory.java | 95 ++
.../hbase/quotas/TableQuotaSnapshotStore.java | 127 ++
.../hbase/quotas/TableQuotaViolationStore.java | 127 --
.../quotas/TableSpaceQuotaSnapshotNotifier.java | 52 +
.../TableSpaceQuotaViolationNotifier.java | 55 -
.../AbstractViolationPolicyEnforcement.java | 118 ++
...LoadVerifyingViolationPolicyEnforcement.java | 50 +
.../DisableTableViolationPolicyEnforcement.java | 80 +
.../NoInsertsViolationPolicyEnforcement.java| 55 +
...esCompactionsViolationPolicyEnforcement.java | 64 +
.../NoWritesViolationPolicyEnforcement.java | 54 +
.../hbase/regionserver/CompactSplitThread.java | 12 +
.../hbase/regionserver/RSRpcServices.java | 92 +-
.../hbase/quotas/SpaceQuotaHelperForTests.java | 228 +++
.../SpaceQuotaSnapshotNotifierForTest.java | 55 +
.../quotas/TestActivePolicyEnforcement.java | 74 +
.../quotas/TestFileSystemUtilizationChore.java |3 +-
.../TestNamespaceQuotaViolationStore.java | 16 +-
.../hbase/quotas/TestQuotaObserverChore.java| 30 +-
.../TestQuotaObserverChoreWithMiniCluster.java | 351 ++---
.../hadoop/hbase/quotas/TestQuotaTableUtil.java | 34 +-
.../TestRegionServerSpaceQuotaManager.java | 174 ++-
...SpaceQuotaViolationPolicyRefresherChore.java | 193 ++-
.../hadoop/hbase/quotas/TestSpaceQuotas.java| 452 ++
.../quotas/TestTableQuotaViolationStore.java| 22 +-
.../TestTableSpaceQuotaViolationNotifier.java | 48 +-
.../hbase/quotas/TestTablesWithQuotas.java |8 +-
.../BaseViolationPolicyEnforcement.java | 31 +
...kLoadCheckingViolationPolicyEnforcement.java | 142 ++
...tDisableTableViolationPolicyEnforcement.java | 59 +
...TestNoInsertsViolationPolicyEnforcement.java | 57 +
...esCompactionsViolationPolicyEnforcement.java | 58 +
.../TestNoWritesViolationPolicyEnforcement.java | 57 +
57 files changed, 6492 insertions(+), 1481 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f42b498/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaExceededException.java
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/2a21313a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
index d7d4db0..e90c934 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
@@ -10164,6 +10164,1912 @@ public final class RegionServerStatusProtos {
}
+ public interface RegionSpaceUseOrBuilder extends
+ // @@protoc_insertion_point(interface_extends:hbase.pb.RegionSpaceUse)
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
+
+/**
+ *
+ * A region identifier
+ *
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+boolean hasRegion();
+/**
+ *
+ * A region identifier
+ *
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo
getRegion();
+/**
+ *
+ * A region identifier
+ *
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfoOrBuilder
getRegionOrBuilder();
+
+/**
+ *
+ * The size in bytes of the region
+ *
+ *
+ * optional uint64 size = 2;
+ */
+boolean hasSize();
+/**
+ *
+ * The size in bytes of the region
+ *
+ *
+ * optional uint64 size = 2;
+ */
+long getSize();
+ }
+ /**
+ * Protobuf type {@code hbase.pb.RegionSpaceUse}
+ */
+ public static final class RegionSpaceUse extends
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3
implements
+ // @@protoc_insertion_point(message_implements:hbase.pb.RegionSpaceUse)
+ RegionSpaceUseOrBuilder {
+// Use RegionSpaceUse.newBuilder() to construct.
+private
RegionSpaceUse(org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.Builder
builder) {
+ super(builder);
+}
+private RegionSpaceUse() {
+ size_ = 0L;
+}
+
+@java.lang.Override
+public final
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+ return this.unknownFields;
+}
+private RegionSpaceUse(
+org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream
input,
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite
extensionRegistry)
+throws
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
{
+ this();
+ int mutable_bitField0_ = 0;
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.Builder
unknownFields =
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+boolean done = false;
+while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+case 0:
+ done = true;
+ break;
+default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+ }
+ break;
+}
+case 10: {
+
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.Builder
subBuilder = null;
+ if (((bitField0_ & 0x0001) == 0x0001)) {
+subBuilder = region_.toBuilder();
+ }
+ region_ =
input.readMessage(org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.PARSER,
extensionRegistry);
+ if (subBuilder != null) {
+subBuilder.mergeFrom(region_);
+region_ = subBuilder.buildPartial();
+ }
+ bitField0_ |= 0x0001;
+ break;
+}
+case 16: {
+ bitField0_ |= 0x0002;
+ size_ = input.readUInt64();
+ break;
+}
+ }
+}
+ } catch
(org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
e) {
+throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+throw new
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException(
+e).setUnfinishedMessage(this);
+ } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+ }
+}
+
HBASE-17863-addendum: Reverted the order of updateStoreOnExec() and
store.isRunning() in execProcedure()
Change-Id: I1c9d5ee264f4f593a6b2a09011853ab63693f677
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/59e8b8e2
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/59e8b8e2
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/59e8b8e2
Branch: refs/heads/HBASE-16961
Commit: 59e8b8e2ba4d403d042fe4cc02f8f9f80aad67af
Parents: 18c5ecf
Author: Umesh Agashe
Authored: Fri Apr 7 14:01:37 2017 -0700
Committer: Apekshit Sharma
Committed: Fri Apr 7 16:13:37 2017 -0700
--
.../hadoop/hbase/procedure2/ProcedureExecutor.java | 11 ---
.../hadoop/hbase/procedure2/ProcedureTestingUtility.java | 2 +-
.../procedure2/store/wal/TestWALProcedureStore.java | 3 +--
3 files changed, 10 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/59e8b8e2/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
index 8832637..43f5839 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/ProcedureExecutor.java
@@ -1373,12 +1373,17 @@ public class ProcedureExecutor {
return;
}
- // if the store is not running we are aborting
- if (!store.isRunning()) return;
-
+ // TODO: The code here doesn't check if store is running before
persisting to the store as
+ // it relies on the method call below to throw RuntimeException to wind
up the stack and
+ // executor thread to stop. The statement following the method call
below seems to check if
+ // store is not running, to prevent scheduling children procedures,
re-execution or yield
+ // of this procedure. This may need more scrutiny and subsequent cleanup
in future
// Commit the transaction
updateStoreOnExec(procStack, procedure, subprocs);
+ // if the store is not running we are aborting
+ if (!store.isRunning()) return;
+
// if the procedure is kind enough to pass the slot to someone else,
yield
if (procedure.isRunnable() && !suspended &&
procedure.isYieldAfterExecutionStep(getEnvironment())) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/59e8b8e2/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
--
diff --git
a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
index 1f4244a..dd3c8f4 100644
---
a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
+++
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/ProcedureTestingUtility.java
@@ -408,7 +408,7 @@ public class ProcedureTestingUtility {
addStackIndex(index);
}
-public void setFinishedState() {
+public void setSuccessState() {
setState(ProcedureState.SUCCESS);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/59e8b8e2/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java
--
diff --git
a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java
index f8c3486..525a663 100644
---
a/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java
+++
b/hbase-procedure/src/test/java/org/apache/hadoop/hbase/procedure2/store/wal/TestWALProcedureStore.java
@@ -26,7 +26,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashSet;
-import java.util.List;
import java.util.Set;
import org.apache.commons.logging.Log;
@@ -785,7 +784,7 @@ public class TestWALProcedureStore {
// back to A
a.addStackId(5);
-a.setFinishedState();
+a.setSuccessState();
procStore.delete(a, new long[] { b.getProcId(), c.getProcId() });
restartAndAssert(3, 0, 1, 0);
}
HBASE-17854 Use StealJobQueue in HFileCleaner after HBASE-17215
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cbcbcf4d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cbcbcf4d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cbcbcf4d
Branch: refs/heads/HBASE-16961
Commit: cbcbcf4dcd3401327cc36173f3ca8e5362da1e0c
Parents: a66d491
Author: Yu Li
Authored: Wed Apr 5 17:53:21 2017 +0800
Committer: Yu Li
Committed: Wed Apr 5 17:53:21 2017 +0800
--
.../hbase/master/cleaner/HFileCleaner.java | 98 +---
.../apache/hadoop/hbase/util/StealJobQueue.java | 22 +
.../hbase/master/cleaner/TestHFileCleaner.java | 28 +++---
.../hadoop/hbase/util/TestStealJobQueue.java| 2 +-
4 files changed, 102 insertions(+), 48 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/cbcbcf4d/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
index 3a68252..8b3515a 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/cleaner/HFileCleaner.java
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -35,6 +34,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.Stoppable;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
+import org.apache.hadoop.hbase.util.StealJobQueue;
import com.google.common.annotations.VisibleForTesting;
/**
@@ -57,23 +57,23 @@ public class HFileCleaner extends
CleanerChore impleme
"hbase.regionserver.thread.hfilecleaner.throttle";
public final static int DEFAULT_HFILE_DELETE_THROTTLE_THRESHOLD = 64 * 1024
* 1024;// 64M
- // Configuration key for large queue size
- public final static String LARGE_HFILE_DELETE_QUEUE_SIZE =
+ // Configuration key for large queue initial size
+ public final static String LARGE_HFILE_QUEUE_INIT_SIZE =
"hbase.regionserver.hfilecleaner.large.queue.size";
- public final static int DEFAULT_LARGE_HFILE_DELETE_QUEUE_SIZE = 1048576;
+ public final static int DEFAULT_LARGE_HFILE_QUEUE_INIT_SIZE = 10240;
- // Configuration key for small queue size
- public final static String SMALL_HFILE_DELETE_QUEUE_SIZE =
+ // Configuration key for small queue initial size
+ public final static String SMALL_HFILE_QUEUE_INIT_SIZE =
"hbase.regionserver.hfilecleaner.small.queue.size";
- public final static int DEFAULT_SMALL_HFILE_DELETE_QUEUE_SIZE = 1048576;
+ public final static int DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE = 10240;
private static final Log LOG = LogFactory.getLog(HFileCleaner.class);
- BlockingQueue largeFileQueue;
+ StealJobQueue largeFileQueue;
BlockingQueue smallFileQueue;
private int throttlePoint;
- private int largeQueueSize;
- private int smallQueueSize;
+ private int largeQueueInitSize;
+ private int smallQueueInitSize;
private List threads = new ArrayList();
private boolean running;
@@ -94,12 +94,12 @@ public class HFileCleaner extends
CleanerChore impleme
directory, MASTER_HFILE_CLEANER_PLUGINS, params);
throttlePoint =
conf.getInt(HFILE_DELETE_THROTTLE_THRESHOLD,
DEFAULT_HFILE_DELETE_THROTTLE_THRESHOLD);
-largeQueueSize =
-conf.getInt(LARGE_HFILE_DELETE_QUEUE_SIZE,
DEFAULT_LARGE_HFILE_DELETE_QUEUE_SIZE);
-smallQueueSize =
-conf.getInt(SMALL_HFILE_DELETE_QUEUE_SIZE,
DEFAULT_SMALL_HFILE_DELETE_QUEUE_SIZE);
-largeFileQueue = new
LinkedBlockingQueue(largeQueueSize);
-smallFileQueue = new
LinkedBlockingQueue(smallQueueSize);
+largeQueueInitSize =
+conf.getInt(LARGE_HFILE_QUEUE_INIT_SIZE,
DEFAULT_LARGE_HFILE_QUEUE_INIT_SIZE);
+smallQueueInitSize =
+conf.getInt(SMALL_HFILE_QUEUE_INIT_SIZE,
DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE);
+largeFileQueue = new StealJobQueue<>(largeQueueInitSize,
smallQueueInitSize);
+smallFileQueue = largeFileQueue.getStealFromQueue();
startHFileDeleteThreads();
}
@@ -152,6 +152,7 @@ public class HFileCleaner extends
CleanerChore impleme
private boolean dispatch(HFileDeleteTask task) {
if (task.fileLength >= this.throttlePoint) {
if (!this.largeFileQueue.offer(task)) {
+// should never arrive here as long as we use PriorityQu
HBASE-17516 Correctly handle case where table and NS quotas both apply
The logic surrounding when a table and namespace quota both apply
to a table was incorrect, leading to a case where a table quota
violation which should have fired did not because of the less-strict
namespace quota.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/36ee5796
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/36ee5796
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/36ee5796
Branch: refs/heads/HBASE-16961
Commit: 36ee57961bd1d6e10dcd9486e9de1b2c6257ceaf
Parents: a1ffeea
Author: Josh Elser
Authored: Wed Feb 22 18:32:55 2017 -0500
Committer: Josh Elser
Committed: Tue Apr 11 13:10:03 2017 -0400
--
.../hadoop/hbase/quotas/QuotaObserverChore.java | 10 ++-
.../TestQuotaObserverChoreWithMiniCluster.java | 66
.../hbase/quotas/TestQuotaStatusRPCs.java | 21 ++-
.../hadoop/hbase/quotas/TestSpaceQuotas.java| 32 +-
4 files changed, 97 insertions(+), 32 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/36ee5796/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index 973ac8c..b9f4592 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -287,7 +287,8 @@ public class QuotaObserverChore extends ScheduledChore {
// We want to have a policy of "NONE", moving out of violation
if (!targetStatus.isInViolation()) {
for (TableName tableInNS : tablesByNamespace.get(namespace)) {
- if
(!tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation())
{
+ // If there is a quota on this table in violation
+ if
(tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation())
{
// Table-level quota violation policy is being applied here.
if (LOG.isTraceEnabled()) {
LOG.trace("Not activating Namespace violation policy because a
Table violation"
@@ -298,16 +299,21 @@ public class QuotaObserverChore extends ScheduledChore {
this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
}
}
+ // We want to move into violation at the NS level
} else {
// Moving tables in the namespace into violation or to a different
violation policy
for (TableName tableInNS : tablesByNamespace.get(namespace)) {
- if
(tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation())
{
+ final SpaceQuotaSnapshot tableQuotaSnapshot =
+tableSnapshotStore.getCurrentState(tableInNS);
+ final boolean hasTableQuota = QuotaSnapshotStore.NO_QUOTA !=
tableQuotaSnapshot;
+ if (hasTableQuota &&
tableQuotaSnapshot.getQuotaStatus().isInViolation()) {
// Table-level quota violation policy is being applied here.
if (LOG.isTraceEnabled()) {
LOG.trace("Not activating Namespace violation policy because a
Table violation"
+ " policy is already in effect for " + tableInNS);
}
} else {
+// No table quota present or a table quota present that is not in
violation
LOG.info(tableInNS + " moving into violation of namespace space
quota with policy " + targetStatus.getPolicy());
this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/36ee5796/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
index 943c898..63198a8 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -193,40 +193,42 @@ public class TestQuotaObserverChoreWithMiniCluster {
helper.writeData(tn1, 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
admin.flush(tn1);
-Map violatedQuotas =
snapshotNotifier.copySnapshots();
+Map snapshots =
snapshotNotifier.copySnapshots();
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Increment.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Increment.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Increment.java
index eb1cbc5..179a566 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Increment.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Increment.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.io.TimeRange;
import org.apache.hadoop.hbase.security.access.Permission;
import org.apache.hadoop.hbase.security.visibility.CellVisibility;
@@ -49,7 +48,6 @@ import org.apache.hadoop.hbase.util.ClassSize;
* {@link #addColumn(byte[], byte[], long)} method.
*/
@InterfaceAudience.Public
-@InterfaceStability.Stable
public class Increment extends Mutation implements Comparable {
private static final long HEAP_OVERHEAD = ClassSize.REFERENCE +
ClassSize.TIMERANGE;
private TimeRange tr = new TimeRange();
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-client/src/main/java/org/apache/hadoop/hbase/client/IsolationLevel.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/IsolationLevel.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/IsolationLevel.java
index 01aba6f..ad0897e 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/IsolationLevel.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/IsolationLevel.java
@@ -21,7 +21,6 @@
package org.apache.hadoop.hbase.client;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
/**
* Specify Isolation levels in Scan operations.
@@ -33,7 +32,6 @@ import
org.apache.hadoop.hbase.classification.InterfaceStability;
* not have been committed yet.
*/
@InterfaceAudience.Public
-@InterfaceStability.Stable
public enum IsolationLevel {
READ_COMMITTED(1),
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MasterSwitchType.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MasterSwitchType.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MasterSwitchType.java
index 7e31b25..5fa9ec2 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MasterSwitchType.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MasterSwitchType.java
@@ -17,13 +17,11 @@
*/
package org.apache.hadoop.hbase.client;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
/**
* Represents the master switch type
*/
@InterfaceAudience.Public
-@InterfaceStability.Evolving
public enum MasterSwitchType {
SPLIT,
MERGE
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MobCompactPartitionPolicy.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MobCompactPartitionPolicy.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MobCompactPartitionPolicy.java
index f550572..076ab6f 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MobCompactPartitionPolicy.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MobCompactPartitionPolicy.java
@@ -19,13 +19,11 @@
package org.apache.hadoop.hbase.client;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
/**
* Enum describing the mob compact partition policy types.
*/
@InterfaceAudience.Public
-@InterfaceStability.Evolving
public enum MobCompactPartitionPolicy {
/**
* Compact daily mob files into one file
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
index fb55fdd..b010c2f 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
+++ b/hbase-client/src/main/java/org/apache/hadoop
HBASE-17863: Procedure V2: Some cleanup around Procedure.isFinished() and
procedure executor
Signed-off-by: Michael Stack
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/91098038
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/91098038
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/91098038
Branch: refs/heads/HBASE-16961
Commit: 9109803891e256f8c047af72572f07695e604a3f
Parents: ec5188d
Author: Umesh Agashe
Authored: Mon Apr 3 17:37:41 2017 -0700
Committer: Michael Stack
Committed: Thu Apr 6 12:05:23 2017 -0700
--
.../org/apache/hadoop/hbase/ProcedureState.java | 2 +-
.../hadoop/hbase/procedure2/Procedure.java | 48
.../hbase/procedure2/ProcedureExecutor.java | 32 ++---
.../hadoop/hbase/procedure2/ProcedureUtil.java | 2 +-
.../hbase/procedure2/store/ProcedureStore.java | 2 +-
.../store/wal/ProcedureWALFormatReader.java | 19
.../procedure2/ProcedureTestingUtility.java | 4 +-
...ProcedureWALLoaderPerformanceEvaluation.java | 2 +-
.../protobuf/generated/ProcedureProtos.java | 40 +++-
.../src/main/protobuf/Procedure.proto | 3 +-
.../master/procedure/TestProcedureAdmin.java| 5 +-
11 files changed, 92 insertions(+), 67 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/91098038/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureState.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureState.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureState.java
index 5d95add..0080baa 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureState.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ProcedureState.java
@@ -24,5 +24,5 @@ import
org.apache.hadoop.hbase.classification.InterfaceAudience;
*/
@InterfaceAudience.Public
public enum ProcedureState {
- INITIALIZING, RUNNABLE, WAITING, WAITING_TIMEOUT, ROLLEDBACK, FINISHED;
+ INITIALIZING, RUNNABLE, WAITING, WAITING_TIMEOUT, ROLLEDBACK, SUCCESS,
FAILED;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/91098038/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
--
diff --git
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
index 2a7fa6e..761ab3a 100644
---
a/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
+++
b/hbase-procedure/src/main/java/org/apache/hadoop/hbase/procedure2/Procedure.java
@@ -216,9 +216,9 @@ public abstract class Procedure implements
Comparable {
}
/**
- * By default, the executor will try ro run procedures start to finish.
+ * By default, the executor will try to run procedures start to finish.
* Return true to make the executor yield between each execution step to
- * give other procedures time to run their steps.
+ * give other procedures a chance to run.
* @param env the environment passed to the ProcedureExecutor
* @return Return true if the executor should yield on completion of an
execution step.
* Defaults to return false.
@@ -271,7 +271,7 @@ public abstract class Procedure implements
Comparable {
toStringState(sb);
if (hasException()) {
- sb.append(", failed=" + getException());
+ sb.append(", exception=" + getException());
}
sb.append(", ");
@@ -506,6 +506,25 @@ public abstract class Procedure implements
Comparable {
//
==
/**
+ * Procedure has states which are defined in proto file. At some places in
the code, we
+ * need to determine more about those states. Following Methods help
determine:
+ *
+ * {@link #isFailed()} - A procedure has executed at least once and has
failed. The procedure
+ * may or may not have rolled back yet. Any procedure
in FAILED state
+ * will be eventually moved to ROLLEDBACK state.
+ *
+ * {@link #isSuccess()} - A procedure is completed successfully without any
exception.
+ *
+ * {@link #isFinished()} - As a procedure in FAILED state will be tried
forever for rollback, only
+ * condition when scheduler/ executor will drop
procedure from further
+ * processing is when procedure state is ROLLEDBACK
or isSuccess()
+ * returns true. This is a terminal state of the
procedure.
+ *
+ * {@link #isWaiting()} - Procedure is in one of the two wait
HBASE-17602 Reduce some quota chore periods/delays
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/10b004b7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/10b004b7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/10b004b7
Branch: refs/heads/HBASE-16961
Commit: 10b004b7240975d663a65a48d67bde386dd451d6
Parents: 36ee579
Author: Josh Elser
Authored: Tue Feb 7 11:21:08 2017 -0500
Committer: Josh Elser
Committed: Tue Apr 11 13:10:03 2017 -0400
--
.../java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java | 4 ++--
.../org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/10b004b7/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index b9f4592..7f894e4 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -55,11 +55,11 @@ public class QuotaObserverChore extends ScheduledChore {
private static final Log LOG = LogFactory.getLog(QuotaObserverChore.class);
static final String QUOTA_OBSERVER_CHORE_PERIOD_KEY =
"hbase.master.quotas.observer.chore.period";
- static final int QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 5
minutes in millis
+ static final int QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 1; // 1
minutes in millis
static final String QUOTA_OBSERVER_CHORE_DELAY_KEY =
"hbase.master.quotas.observer.chore.delay";
- static final long QUOTA_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1
minute
+ static final long QUOTA_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 15L; // 15
seconds in millis
static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_KEY =
"hbase.master.quotas.observer.chore.timeunit";
http://git-wip-us.apache.org/repos/asf/hbase/blob/10b004b7/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
index e1a2693..8587e79 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
@@ -44,11 +44,11 @@ public class SpaceQuotaRefresherChore extends
ScheduledChore {
static final String POLICY_REFRESHER_CHORE_PERIOD_KEY =
"hbase.regionserver.quotas.policy.refresher.chore.period";
- static final int POLICY_REFRESHER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 5
minutes in millis
+ static final int POLICY_REFRESHER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 1; // 1
minute in millis
static final String POLICY_REFRESHER_CHORE_DELAY_KEY =
"hbase.regionserver.quotas.policy.refresher.chore.delay";
- static final long POLICY_REFRESHER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1
minute
+ static final long POLICY_REFRESHER_CHORE_DELAY_DEFAULT = 1000L * 15L; // 15
seconds in millis
static final String POLICY_REFRESHER_CHORE_TIMEUNIT_KEY =
"hbase.regionserver.quotas.policy.refresher.chore.timeunit";
HBASE-17858 Update refguide about the IS annotation if necessary
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/17737b27
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/17737b27
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/17737b27
Branch: refs/heads/HBASE-16961
Commit: 17737b2710a2a1271eb791478eb99f7a573ecac1
Parents: 029fa29
Author: zhangduo
Authored: Fri Mar 31 18:20:45 2017 +0800
Committer: zhangduo
Committed: Thu Apr 6 09:48:18 2017 +0800
--
src/main/asciidoc/_chapters/upgrading.adoc | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/17737b27/src/main/asciidoc/_chapters/upgrading.adoc
--
diff --git a/src/main/asciidoc/_chapters/upgrading.adoc
b/src/main/asciidoc/_chapters/upgrading.adoc
index df5bbfe..46f637d 100644
--- a/src/main/asciidoc/_chapters/upgrading.adoc
+++ b/src/main/asciidoc/_chapters/upgrading.adoc
@@ -128,7 +128,7 @@ for warning about incompatible changes). All effort will be
made to provide a de
HBase has a lot of API points, but for the compatibility matrix above, we
differentiate between Client API, Limited Private API, and Private API. HBase
uses a version of
link:https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/Compatibility.html[Hadoop's
Interface classification]. HBase's Interface classification classes can be
found
link:https://hbase.apache.org/apidocs/org/apache/hadoop/hbase/classification/package-summary.html[here].
* InterfaceAudience: captures the intended audience, possible values are
Public (for end users and external projects), LimitedPrivate (for other
Projects, Coprocessors or other plugin points), and Private (for internal use).
-* InterfaceStability: describes what types of interface changes are permitted.
Possible values are Stable, Evolving, Unstable, and Deprecated.
+* InterfaceStability: describes what types of interface changes are permitted.
Possible values are Stable, Evolving, Unstable, and Deprecated. Notice that
this annotation is only valid for classes which are marked as
IA.LimitedPrivate. The stability of IA.Public classes is only related to the
upgrade type(major, minor or patch). And for IA.Private classes, there is no
guarantee on the stability between releases. Refer to the Compatibility Matrix
above for more details.
[[hbase.client.api]]
HBase Client API::
HBASE-17785 RSGroupBasedLoadBalancer fails to assign new table regions when
cloning snapshot
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/029fa297
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/029fa297
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/029fa297
Branch: refs/heads/HBASE-16961
Commit: 029fa297129f7ced276d19c4877d19bf32dcfde0
Parents: cbcbcf4
Author: Andrew Purtell
Authored: Wed Apr 5 16:25:56 2017 -0700
Committer: Andrew Purtell
Committed: Wed Apr 5 16:25:56 2017 -0700
--
.../hbase/rsgroup/RSGroupAdminEndpoint.java | 28 ++--
.../hadoop/hbase/rsgroup/TestRSGroups.java | 19 +
2 files changed, 39 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/029fa297/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
--
diff --git
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
index 83389e4..14907ba 100644
---
a/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
+++
b/hbase-rsgroup/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupAdminEndpoint.java
@@ -67,6 +67,7 @@ import
org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.MoveTablesR
import
org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RSGroupAdminService;
import
org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupRequest;
import
org.apache.hadoop.hbase.protobuf.generated.RSGroupAdminProtos.RemoveRSGroupResponse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription;
@InterfaceAudience.Private
public class RSGroupAdminEndpoint implements MasterObserver,
CoprocessorService {
@@ -267,14 +268,7 @@ public class RSGroupAdminEndpoint implements
MasterObserver, CoprocessorService
}
}
- /
- // MasterObserver overrides
- /
-
- // Assign table to default RSGroup.
- @Override
- public void preCreateTable(ObserverContext ctx,
- HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
+ void assignTableToGroup(HTableDescriptor desc) throws IOException {
String groupName =
master.getClusterSchema().getNamespace(desc.getTableName().getNamespaceAsString())
.getConfigurationValue(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP);
@@ -292,6 +286,17 @@ public class RSGroupAdminEndpoint implements
MasterObserver, CoprocessorService
}
}
+ /
+ // MasterObserver overrides
+ /
+
+ // Assign table to default RSGroup.
+ @Override
+ public void preCreateTable(ObserverContext ctx,
+ HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
+assignTableToGroup(desc);
+ }
+
// Remove table from its RSGroup.
@Override
public void postDeleteTable(ObserverContext
ctx,
@@ -322,5 +327,12 @@ public class RSGroupAdminEndpoint implements
MasterObserver, CoprocessorService
NamespaceDescriptor ns) throws IOException {
preCreateNamespace(ctx, ns);
}
+
+ @Override
+ public void preCloneSnapshot(ObserverContext
ctx,
+ SnapshotDescription snapshot, HTableDescriptor desc) throws IOException {
+assignTableToGroup(desc);
+ }
+
/
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/029fa297/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
--
diff --git
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
index 3886684..d6bd43b 100644
---
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
+++
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroups.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.ServerManager;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.net.Address;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawString.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawString.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawString.java
index 7e3b350..b70e103 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawString.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawString.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.types;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Order;
import org.apache.hadoop.hbase.util.PositionedByteRange;
@@ -32,7 +31,6 @@ import org.apache.hadoop.hbase.util.PositionedByteRange;
* @see RawStringTerminated
*/
@InterfaceAudience.Public
-@InterfaceStability.Evolving
public class RawString implements DataType {
public static final RawString ASCENDING = new RawString(Order.ASCENDING);
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringFixedLength.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringFixedLength.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringFixedLength.java
index d11bead..24a394c 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringFixedLength.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringFixedLength.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.types;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.util.Order;
/**
@@ -31,7 +30,6 @@ import org.apache.hadoop.hbase.util.Order;
* @see RawString
*/
@InterfaceAudience.Public
-@InterfaceStability.Evolving
public class RawStringFixedLength extends FixedLengthWrapper {
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringTerminated.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringTerminated.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringTerminated.java
index 4d89d5b..408b57a 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringTerminated.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/RawStringTerminated.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hbase.types;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.util.Order;
/**
@@ -33,7 +32,6 @@ import org.apache.hadoop.hbase.util.Order;
* @see OrderedString
*/
@InterfaceAudience.Public
-@InterfaceStability.Evolving
public class RawStringTerminated extends TerminatedWrapper {
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java
index 550088a..eea64d9 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/Struct.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.types;
import java.util.Iterator;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.util.Order;
import org.apache.hadoop.hbase.util.PositionedByteRange;
@@ -78,7 +77,6 @@ import org.apache.hadoop.hbase.util.PositionedByteRange;
* @see DataType#isNullable()
*/
@InterfaceAudience.Public
-@InterfaceStability.Evolving
public class Struct implements DataType {
@SuppressWarnings("rawtypes")
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructBuilder.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructBuilder.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructBuilder.java
index d73a17d..ad4f021 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructBuilder.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/types/StructBuilder.java
@@ -21,13 +2
http://git-wip-us.apache.org/repos/asf/hbase/blob/acf21536/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
new file mode 100644
index 000..98236c2
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -0,0 +1,596 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.NamespaceNotFoundException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore.TablesWithQuotas;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+
+/**
+ * Test class for {@link QuotaObserverChore} that uses a live HBase cluster.
+ */
+@Category(LargeTests.class)
+public class TestQuotaObserverChoreWithMiniCluster {
+ private static final Log LOG =
LogFactory.getLog(TestQuotaObserverChoreWithMiniCluster.class);
+ private static final int SIZE_PER_VALUE = 256;
+ private static final String F1 = "f1";
+ private static final HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
+ private static final AtomicLong COUNTER = new AtomicLong(0);
+ private static final long ONE_MEGABYTE = 1024L * 1024L;
+ private static final long DEFAULT_WAIT_MILLIS = 500;
+
+ @Rule
+ public TestName testName = new TestName();
+
+ private HMaster master;
+ private QuotaObserverChore chore;
+ private SpaceQuotaViolationNotifierForTest violationNotifier;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+Configuration conf = TEST_UTIL.getConfiguration();
+conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY,
1000);
+conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY,
1000);
+conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_DELAY_KEY, 1000);
+conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_PERIOD_KEY, 1000);
+conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
+TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void removeAllQuotas() throws Exception {
+final Connection conn = TEST_UTIL.getConnection();
+
HBASE-17872 The MSLABImpl generates the invaild cells when unsafe is not
availble
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/df96d328
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/df96d328
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/df96d328
Branch: refs/heads/HBASE-16961
Commit: df96d328fb9fa11f04f84607e9a23f254f513202
Parents: 59e8b8e
Author: CHIA-PING TSAI
Authored: Sat Apr 8 17:37:37 2017 +0800
Committer: Chia-Ping Tsai
Committed: Sun Apr 9 23:28:34 2017 +0800
--
.../hadoop/hbase/util/ByteBufferUtils.java | 30 ++--
.../hadoop/hbase/util/TestByteBufferUtils.java | 165 ++-
.../hbase/util/TestFromClientSide3WoUnsafe.java | 43 +
3 files changed, 213 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/df96d328/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
index ff4c843..34a4e02 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
@@ -43,15 +43,14 @@ import sun.nio.ch.DirectBuffer;
@SuppressWarnings("restriction")
@InterfaceAudience.Public
public final class ByteBufferUtils {
-
// "Compressed integer" serialization helper constants.
public final static int VALUE_MASK = 0x7f;
public final static int NEXT_BIT_SHIFT = 7;
public final static int NEXT_BIT_MASK = 1 << 7;
@VisibleForTesting
- static boolean UNSAFE_AVAIL = UnsafeAvailChecker.isAvailable();
+ final static boolean UNSAFE_AVAIL = UnsafeAvailChecker.isAvailable();
@VisibleForTesting
- static boolean UNSAFE_UNALIGNED = UnsafeAvailChecker.unaligned();
+ final static boolean UNSAFE_UNALIGNED = UnsafeAvailChecker.unaligned();
private ByteBufferUtils() {
}
@@ -404,12 +403,11 @@ public final class ByteBufferUtils {
} else if (UNSAFE_AVAIL) {
UnsafeAccess.copy(in, sourceOffset, out, destinationOffset, length);
} else {
- int outOldPos = out.position();
- out.position(destinationOffset);
+ ByteBuffer outDup = out.duplicate();
+ outDup.position(destinationOffset);
ByteBuffer inDup = in.duplicate();
inDup.position(sourceOffset).limit(sourceOffset + length);
- out.put(inDup);
- out.position(outOldPos);
+ outDup.put(inDup);
}
return destinationOffset + length;
}
@@ -990,7 +988,7 @@ public final class ByteBufferUtils {
/**
* Copies bytes from given array's offset to length part into the given
buffer. Puts the bytes
- * to buffer's given position.
+ * to buffer's given position. This doesn't affact the position of buffer.
* @param out
* @param in
* @param inOffset
@@ -1003,16 +1001,15 @@ public final class ByteBufferUtils {
} else if (UNSAFE_AVAIL) {
UnsafeAccess.copy(in, inOffset, out, outOffset, length);
} else {
- int oldPos = out.position();
- out.position(outOffset);
- out.put(in, inOffset, length);
- out.position(oldPos);
+ ByteBuffer outDup = out.duplicate();
+ outDup.position(outOffset);
+ outDup.put(in, inOffset, length);
}
}
/**
* Copies specified number of bytes from given offset of 'in' ByteBuffer to
- * the array.
+ * the array. This doesn't affact the position of buffer.
* @param out
* @param in
* @param sourceOffset
@@ -1026,10 +1023,9 @@ public final class ByteBufferUtils {
} else if (UNSAFE_AVAIL) {
UnsafeAccess.copy(in, sourceOffset, out, destinationOffset, length);
} else {
- int oldPos = in.position();
- in.position(sourceOffset);
- in.get(out, destinationOffset, length);
- in.position(oldPos);
+ ByteBuffer inDup = in.duplicate();
+ inDup.position(sourceOffset);
+ inDup.get(out, destinationOffset, length);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/df96d328/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java
--
diff --git
a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java
index 053fb24..ee03c7b 100644
---
a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java
+++
b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java
@@ -27,14 +27,22 @@ import java.io.ByteArrayOutputStream;
import java.io.DataInputS
HBASE-16995 Build client Java API and client protobuf messages (Josh Elser)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5def44f5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5def44f5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5def44f5
Branch: refs/heads/HBASE-16961
Commit: 5def44f5a187938034de1529060d679285d79edb
Parents: 82d554e
Author: tedyu
Authored: Thu Nov 17 10:19:52 2016 -0800
Committer: Josh Elser
Committed: Tue Apr 11 13:10:01 2017 -0400
--
.../hbase/quotas/QuotaSettingsFactory.java | 47 +
.../apache/hadoop/hbase/quotas/QuotaType.java |1 +
.../hadoop/hbase/quotas/SpaceLimitSettings.java | 166 ++
.../hbase/quotas/SpaceViolationPolicy.java | 44 +
.../hbase/shaded/protobuf/ProtobufUtil.java | 51 +
.../hbase/quotas/TestQuotaSettingsFactory.java | 148 ++
.../hbase/quotas/TestSpaceLimitSettings.java| 119 ++
.../shaded/protobuf/generated/MasterProtos.java | 584 --
.../shaded/protobuf/generated/QuotaProtos.java | 1739 +-
.../src/main/protobuf/Master.proto |2 +
.../src/main/protobuf/Quota.proto | 21 +
.../hbase/protobuf/generated/QuotaProtos.java | 1682 -
hbase-protocol/src/main/protobuf/Quota.proto| 21 +
13 files changed, 4291 insertions(+), 334 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/5def44f5/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
index 3622a32..8512e39 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
@@ -27,6 +27,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRe
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
@InterfaceAudience.Public
public class QuotaSettingsFactory {
@@ -89,6 +90,9 @@ public class QuotaSettingsFactory {
if (quotas.getBypassGlobals() == true) {
settings.add(new QuotaGlobalsSettingsBypass(userName, tableName,
namespace, true));
}
+if (quotas.hasSpace()) {
+ settings.add(fromSpace(tableName, namespace, quotas.getSpace()));
+}
return settings;
}
@@ -122,6 +126,18 @@ public class QuotaSettingsFactory {
return settings;
}
+ static QuotaSettings fromSpace(TableName table, String namespace, SpaceQuota
protoQuota) {
+if ((null == table && null == namespace) || (null != table && null !=
namespace)) {
+ throw new IllegalArgumentException("Can only construct
SpaceLimitSettings for a table or namespace.");
+}
+if (null != table) {
+ return SpaceLimitSettings.fromSpaceQuota(table, protoQuota);
+} else {
+ // namespace must be non-null
+ return SpaceLimitSettings.fromSpaceQuota(namespace, protoQuota);
+}
+ }
+
/* ==
* RPC Throttle
*/
@@ -278,4 +294,35 @@ public class QuotaSettingsFactory {
public static QuotaSettings bypassGlobals(final String userName, final
boolean bypassGlobals) {
return new QuotaGlobalsSettingsBypass(userName, null, null, bypassGlobals);
}
+
+ /* ==
+ * FileSystem Space Settings
+ */
+
+ /**
+ * Creates a {@link QuotaSettings} object to limit the FileSystem space
usage for the given table to the given size in bytes.
+ * When the space usage is exceeded by the table, the provided {@link
SpaceViolationPolicy} is enacted on the table.
+ *
+ * @param tableName The name of the table on which the quota should be
applied.
+ * @param sizeLimit The limit of a table's size in bytes.
+ * @param violationPolicy The action to take when the quota is exceeded.
+ * @return An {@link QuotaSettings} object.
+ */
+ public static QuotaSettings limitTableSpace(final TableName tableName, long
sizeLimit, final SpaceViolationPolicy violationPolicy) {
+return new SpaceLimitSettings(tableName, sizeLimit, violationPolicy);
+ }
+
+ /**
+ * Creates a {@link QuotaSettings} object to limit the FileSystem space
usage for the given namespace to the given size in bytes.
+ * When the space usage is exceeded by a
HBASE-16998 Implement Master-side analysis of region space reports
Adds a new Chore to the Master that analyzes the reports that are
sent by RegionServers. The Master must then, for all tables with
quotas, determine the tables that are violating quotas and move
those tables into violation. Similarly, tables no longer violating
the quota can be moved out of violation.
The Chore is the "stateful" bit, managing which tables are and
are not in violation. Everything else is just performing
computation and informing the Chore on the updated state.
Added InterfaceAudience annotations and clean up the QuotaObserverChore
constructor. Cleaned up some javadoc and QuotaObserverChore. Reuse
the QuotaViolationStore impl objects.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/acf21536
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/acf21536
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/acf21536
Branch: refs/heads/HBASE-16961
Commit: acf215361d6a098184fc5ea8030bde2dfca56cc1
Parents: b8c5387
Author: Josh Elser
Authored: Tue Nov 8 18:55:12 2016 -0500
Committer: Josh Elser
Committed: Tue Apr 11 13:10:02 2017 -0400
--
.../hadoop/hbase/quotas/QuotaRetriever.java | 27 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 20 +
.../hadoop/hbase/quotas/MasterQuotaManager.java | 1 +
.../quotas/NamespaceQuotaViolationStore.java| 127
.../hadoop/hbase/quotas/QuotaObserverChore.java | 618 +++
.../hbase/quotas/QuotaViolationStore.java | 89 +++
.../quotas/SpaceQuotaViolationNotifier.java | 44 ++
.../SpaceQuotaViolationNotifierForTest.java | 50 ++
.../hbase/quotas/TableQuotaViolationStore.java | 127
.../TestNamespaceQuotaViolationStore.java | 156 +
.../hbase/quotas/TestQuotaObserverChore.java| 106
.../TestQuotaObserverChoreWithMiniCluster.java | 596 ++
.../hadoop/hbase/quotas/TestQuotaTableUtil.java | 4 -
.../quotas/TestTableQuotaViolationStore.java| 151 +
.../hbase/quotas/TestTablesWithQuotas.java | 198 ++
15 files changed, 2305 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/acf21536/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
index cba6a24..b011cf4 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
@@ -22,6 +22,7 @@ import java.io.Closeable;
import java.io.IOException;
import java.util.Iterator;
import java.util.LinkedList;
+import java.util.Objects;
import java.util.Queue;
import org.apache.commons.logging.Log;
@@ -54,11 +55,23 @@ public class QuotaRetriever implements Closeable,
Iterable {
private Connection connection;
private Table table;
- private QuotaRetriever() {
+ /**
+ * Should QutoaRetriever manage the state of the connection, or leave it be.
+ */
+ private boolean isManagedConnection = false;
+
+ QuotaRetriever() {
}
void init(final Configuration conf, final Scan scan) throws IOException {
-this.connection = ConnectionFactory.createConnection(conf);
+// Set this before creating the connection and passing it down to make sure
+// it's cleaned up if we fail to construct the Scanner.
+this.isManagedConnection = true;
+init(ConnectionFactory.createConnection(conf), scan);
+ }
+
+ void init(final Connection conn, final Scan scan) throws IOException {
+this.connection = Objects.requireNonNull(conn);
this.table = this.connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME);
try {
scanner = table.getScanner(scan);
@@ -77,10 +90,14 @@ public class QuotaRetriever implements Closeable,
Iterable {
this.table.close();
this.table = null;
}
-if (this.connection != null) {
- this.connection.close();
- this.connection = null;
+// Null out the connection on close() even if we didn't explicitly close it
+// to maintain typical semantics.
+if (isManagedConnection) {
+ if (this.connection != null) {
+this.connection.close();
+ }
}
+this.connection = null;
}
public QuotaSettings next() throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/acf21536/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/m
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java
index 7aa807c..ed95a7d 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/InclusiveStopFilter.java
@@ -24,7 +24,6 @@ import java.util.ArrayList;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hadoop.hbase.shaded.com.google.protobuf.UnsafeByteOperations;
@@ -40,7 +39,6 @@ import com.google.common.base.Preconditions;
* Use this filter to include the stop row, eg: [A,Z].
*/
@InterfaceAudience.Public
-@InterfaceStability.Stable
public class InclusiveStopFilter extends FilterBase {
private byte [] stopRowKey;
private boolean done = false;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/IncompatibleFilterException.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/IncompatibleFilterException.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/IncompatibleFilterException.java
index 8eba03c..6410ab4 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/IncompatibleFilterException.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/IncompatibleFilterException.java
@@ -19,13 +19,11 @@
package org.apache.hadoop.hbase.filter;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
/**
* Used to indicate a filter incompatibility
*/
@InterfaceAudience.Public
-@InterfaceStability.Stable
public class IncompatibleFilterException extends RuntimeException {
private static final long serialVersionUID = 3236763276623198231L;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/InvalidRowFilterException.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/InvalidRowFilterException.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/InvalidRowFilterException.java
index 81aae0b..0406058 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/InvalidRowFilterException.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/InvalidRowFilterException.java
@@ -19,13 +19,11 @@
package org.apache.hadoop.hbase.filter;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
/**
* Used to indicate an invalid RowFilter.
*/
@InterfaceAudience.Public
-@InterfaceStability.Stable
public class InvalidRowFilterException extends RuntimeException {
private static final long serialVersionUID = 2667894046345657865L;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java
index adbf304..b082941 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.hbase.ByteBufferCell;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.shaded.protobuf.generated.FilterProtos;
import org.apache.hadoop.hbase.util.Bytes;
@@ -43,7 +42,6 @@ import
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferE
* the values.
*/
@InterfaceAudience.Public
-@InterfaceStability.Stable
public class KeyOnlyFilter extends FilterBase {
boolean lenAsVal;
http://git-wip-us.apache.org/repos/asf/hbase/blob/a66d4918/hbase-client/src/main/java/org/apache/hadoop/hbase/filte
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f42b498/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
index cc40536..d466e59 100644
---
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
@@ -5778,6 +5778,1284 @@ public final class QuotaProtos {
// @@protoc_insertion_point(class_scope:hbase.pb.SpaceLimitRequest)
}
+ public interface SpaceQuotaStatusOrBuilder
+ extends com.google.protobuf.MessageOrBuilder {
+
+// optional .hbase.pb.SpaceViolationPolicy policy = 1;
+/**
+ * optional .hbase.pb.SpaceViolationPolicy policy = 1;
+ */
+boolean hasPolicy();
+/**
+ * optional .hbase.pb.SpaceViolationPolicy policy = 1;
+ */
+
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy
getPolicy();
+
+// optional bool in_violation = 2;
+/**
+ * optional bool in_violation = 2;
+ */
+boolean hasInViolation();
+/**
+ * optional bool in_violation = 2;
+ */
+boolean getInViolation();
+ }
+ /**
+ * Protobuf type {@code hbase.pb.SpaceQuotaStatus}
+ *
+ *
+ * Represents the state of a quota on a table. Either the quota is not in
violation
+ * or it is in violatino there is a violation policy which should be in
effect.
+ *
+ */
+ public static final class SpaceQuotaStatus extends
+ com.google.protobuf.GeneratedMessage
+ implements SpaceQuotaStatusOrBuilder {
+// Use SpaceQuotaStatus.newBuilder() to construct.
+private SpaceQuotaStatus(com.google.protobuf.GeneratedMessage.Builder
builder) {
+ super(builder);
+ this.unknownFields = builder.getUnknownFields();
+}
+private SpaceQuotaStatus(boolean noInit) { this.unknownFields =
com.google.protobuf.UnknownFieldSet.getDefaultInstance(); }
+
+private static final SpaceQuotaStatus defaultInstance;
+public static SpaceQuotaStatus getDefaultInstance() {
+ return defaultInstance;
+}
+
+public SpaceQuotaStatus getDefaultInstanceForType() {
+ return defaultInstance;
+}
+
+private final com.google.protobuf.UnknownFieldSet unknownFields;
+@java.lang.Override
+public final com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+ return this.unknownFields;
+}
+private SpaceQuotaStatus(
+com.google.protobuf.CodedInputStream input,
+com.google.protobuf.ExtensionRegistryLite extensionRegistry)
+throws com.google.protobuf.InvalidProtocolBufferException {
+ initFields();
+ int mutable_bitField0_ = 0;
+ com.google.protobuf.UnknownFieldSet.Builder unknownFields =
+ com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+boolean done = false;
+while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+case 0:
+ done = true;
+ break;
+default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+ }
+ break;
+}
+case 8: {
+ int rawValue = input.readEnum();
+
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy
value =
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.SpaceViolationPolicy.valueOf(rawValue);
+ if (value == null) {
+unknownFields.mergeVarintField(1, rawValue);
+ } else {
+bitField0_ |= 0x0001;
+policy_ = value;
+ }
+ break;
+}
+case 16: {
+ bitField0_ |= 0x0002;
+ inViolation_ = input.readBool();
+ break;
+}
+ }
+}
+ } catch (com.google.protobuf.InvalidProtocolBufferException e) {
+throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+throw new com.google.protobuf.InvalidProtocolBufferException(
+e.getMessage()).setUnfinishedMessage(this);
+ } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+ }
+}
+public static final com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+ return
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.internal_static_hbase_pb_SpaceQuotaStatus_descriptor;
+}
+
+protected com.google.protobuf.GeneratedMessage.FieldAccessorTable
+internalGetFieldAccessorTable() {
+ return
org.apache
HBASE-17871 scan#setBatch(int) call leads wrong result of VerifyReplication
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ec5188df
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ec5188df
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ec5188df
Branch: refs/heads/HBASE-16961
Commit: ec5188df3090d42088b6f4cb8f0c2fd49425f8c1
Parents: d7e3116
Author: Tomu Tsuruhara
Authored: Wed Apr 5 21:42:28 2017 +0900
Committer: tedyu
Committed: Thu Apr 6 07:00:13 2017 -0700
--
.../mapreduce/replication/VerifyReplication.java | 19 +++
1 file changed, 15 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/ec5188df/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index ba5966b..3f8317b 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -77,7 +77,7 @@ public class VerifyReplication extends Configured implements
Tool {
private final static String PEER_CONFIG_PREFIX = NAME + ".peer.";
static long startTime = 0;
static long endTime = Long.MAX_VALUE;
- static int batch = Integer.MAX_VALUE;
+ static int batch = -1;
static int versions = -1;
static String tableName = null;
static String families = null;
@@ -110,6 +110,7 @@ public class VerifyReplication extends Configured
implements Tool {
private int sleepMsBeforeReCompare;
private String delimiter = "";
private boolean verbose = false;
+private int batch = -1;
/**
* Map method that compares every scanned row with the equivalent from
@@ -128,8 +129,11 @@ public class VerifyReplication extends Configured
implements Tool {
sleepMsBeforeReCompare = conf.getInt(NAME +".sleepMsBeforeReCompare",
0);
delimiter = conf.get(NAME + ".delimiter", "");
verbose = conf.getBoolean(NAME +".verbose", false);
+batch = conf.getInt(NAME + ".batch", -1);
final Scan scan = new Scan();
-scan.setBatch(batch);
+if (batch > 0) {
+ scan.setBatch(batch);
+}
scan.setCacheBlocks(false);
scan.setCaching(conf.getInt(TableInputFormat.SCAN_CACHEDROWS, 1));
long startTime = conf.getLong(NAME + ".startTime", 0);
@@ -329,6 +333,7 @@ public class VerifyReplication extends Configured
implements Tool {
conf.setLong(NAME+".endTime", endTime);
conf.setInt(NAME +".sleepMsBeforeReCompare", sleepMsBeforeReCompare);
conf.set(NAME + ".delimiter", delimiter);
+conf.setInt(NAME + ".batch", batch);
conf.setBoolean(NAME +".verbose", verbose);
conf.setBoolean(NAME +".includeDeletedCells", includeDeletedCells);
if (families != null) {
@@ -356,6 +361,10 @@ public class VerifyReplication extends Configured
implements Tool {
Scan scan = new Scan();
scan.setTimeRange(startTime, endTime);
scan.setRaw(includeDeletedCells);
+scan.setCacheBlocks(false);
+if (batch > 0) {
+ scan.setBatch(batch);
+}
if (versions >= 0) {
scan.setMaxVersions(versions);
LOG.info("Number of versions set to " + versions);
@@ -503,7 +512,7 @@ public class VerifyReplication extends Configured
implements Tool {
private static void restoreDefaults() {
startTime = 0;
endTime = Long.MAX_VALUE;
-batch = Integer.MAX_VALUE;
+batch = -1;
versions = -1;
tableName = null;
families = null;
@@ -521,13 +530,15 @@ public class VerifyReplication extends Configured
implements Tool {
}
System.err.println("Usage: verifyrep [--starttime=X]" +
" [--endtime=Y] [--families=A] [--row-prefixes=B] [--delimiter=]
[--recomparesleep=] " +
-"[--verbose] ");
+"[--batch=] [--verbose] ");
System.err.println();
System.err.println("Options:");
System.err.println(" starttimebeginning of the time range");
System.err.println(" without endtime means from starttime to
forever");
System.err.println(" endtime end of the time range");
System.err.println(" versions number of cell versions to verify");
+System.err.println(" batchbatch count for scan, " +
+"note that result row counts will no longer be actual number of rows
when you use this option");
System.err.println(" raw includes raw scan if given in options");
HBASE-16477 Remove Writable interface and related code from WALEdit/WALKey
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/82d554e3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/82d554e3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/82d554e3
Branch: refs/heads/HBASE-16961
Commit: 82d554e3783372cc6b05489452c815b57c06f6cd
Parents: df96d32
Author: Enis Soztutar
Authored: Mon Apr 10 02:31:42 2017 -0700
Committer: Enis Soztutar
Committed: Mon Apr 10 02:31:42 2017 -0700
--
.../regionserver/wal/KeyValueCompression.java | 133 --
.../hadoop/hbase/regionserver/wal/WALEdit.java | 136 +--
.../java/org/apache/hadoop/hbase/wal/WAL.java | 1 -
.../org/apache/hadoop/hbase/wal/WALKey.java | 95 ++---
.../wal/TestKeyValueCompression.java| 116
5 files changed, 14 insertions(+), 467 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/82d554e3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/KeyValueCompression.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/KeyValueCompression.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/KeyValueCompression.java
deleted file mode 100644
index a33ff9e..000
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/KeyValueCompression.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.regionserver.wal;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.io.WritableUtils;
-
-/**
- * DO NOT USE. This class is deprecated and should only be used in pre-PB WAL.
- *
- * Compression class for {@link KeyValue}s written to the WAL. This is not
- * synchronized, so synchronization should be handled outside.
- *
- * Class only compresses and uncompresses row keys, family names, and the
- * qualifier. More may be added depending on use patterns.
- */
-@Deprecated
-@InterfaceAudience.Private
-class KeyValueCompression {
- /**
- * Uncompresses a KeyValue from a DataInput and returns it.
- *
- * @param in the DataInput
- * @param readContext the compressionContext to use.
- * @return an uncompressed KeyValue
- * @throws IOException
- */
-
- public static KeyValue readKV(DataInput in, CompressionContext readContext)
- throws IOException {
-int keylength = WritableUtils.readVInt(in);
-int vlength = WritableUtils.readVInt(in);
-int tagsLength = WritableUtils.readVInt(in);
-int length = (int) KeyValue.getKeyValueDataStructureSize(keylength,
vlength, tagsLength);
-
-byte[] backingArray = new byte[length];
-int pos = 0;
-pos = Bytes.putInt(backingArray, pos, keylength);
-pos = Bytes.putInt(backingArray, pos, vlength);
-
-// the row
-int elemLen = Compressor.uncompressIntoArray(backingArray,
-pos + Bytes.SIZEOF_SHORT, in, readContext.rowDict);
-checkLength(elemLen, Short.MAX_VALUE);
-pos = Bytes.putShort(backingArray, pos, (short)elemLen);
-pos += elemLen;
-
-// family
-elemLen = Compressor.uncompressIntoArray(backingArray,
-pos + Bytes.SIZEOF_BYTE, in, readContext.familyDict);
-checkLength(elemLen, Byte.MAX_VALUE);
-pos = Bytes.putByte(backingArray, pos, (byte)elemLen);
-pos += elemLen;
-
-// qualifier
-elemLen = Compressor.uncompressIntoArray(backingArray, pos, in,
-readContext.qualifierDict);
-pos += elemLen;
-
-// the rest
-in.readFully(backingArray, pos, length - pos);
-
-return new KeyValue(backingArray, 0, length);
- }
-
- private static void checkLength(int len, int max) throws IOException {
-if (len < 0 || len > max
http://git-wip-us.apache.org/repos/asf/hbase/blob/5def44f5/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
index 05894b9..1925828 100644
---
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
@@ -217,12 +217,20 @@ public final class QuotaProtos {
* THROTTLE = 1;
*/
THROTTLE(0, 1),
+/**
+ * SPACE = 2;
+ */
+SPACE(1, 2),
;
/**
* THROTTLE = 1;
*/
public static final int THROTTLE_VALUE = 1;
+/**
+ * SPACE = 2;
+ */
+public static final int SPACE_VALUE = 2;
public final int getNumber() { return value; }
@@ -230,6 +238,7 @@ public final class QuotaProtos {
public static QuotaType valueOf(int value) {
switch (value) {
case 1: return THROTTLE;
+case 2: return SPACE;
default: return null;
}
}
@@ -281,6 +290,142 @@ public final class QuotaProtos {
// @@protoc_insertion_point(enum_scope:hbase.pb.QuotaType)
}
+ /**
+ * Protobuf enum {@code hbase.pb.SpaceViolationPolicy}
+ *
+ *
+ * Defines what action should be taken when the SpaceQuota is violated
+ *
+ */
+ public enum SpaceViolationPolicy
+ implements com.google.protobuf.ProtocolMessageEnum {
+/**
+ * DISABLE = 1;
+ *
+ *
+ * Disable the table(s)
+ *
+ */
+DISABLE(0, 1),
+/**
+ * NO_WRITES_COMPACTIONS = 2;
+ *
+ *
+ * No writes, bulk-loads, or compactions
+ *
+ */
+NO_WRITES_COMPACTIONS(1, 2),
+/**
+ * NO_WRITES = 3;
+ *
+ *
+ * No writes or bulk-loads
+ *
+ */
+NO_WRITES(2, 3),
+/**
+ * NO_INSERTS = 4;
+ *
+ *
+ * No puts or bulk-loads, but deletes are allowed
+ *
+ */
+NO_INSERTS(3, 4),
+;
+
+/**
+ * DISABLE = 1;
+ *
+ *
+ * Disable the table(s)
+ *
+ */
+public static final int DISABLE_VALUE = 1;
+/**
+ * NO_WRITES_COMPACTIONS = 2;
+ *
+ *
+ * No writes, bulk-loads, or compactions
+ *
+ */
+public static final int NO_WRITES_COMPACTIONS_VALUE = 2;
+/**
+ * NO_WRITES = 3;
+ *
+ *
+ * No writes or bulk-loads
+ *
+ */
+public static final int NO_WRITES_VALUE = 3;
+/**
+ * NO_INSERTS = 4;
+ *
+ *
+ * No puts or bulk-loads, but deletes are allowed
+ *
+ */
+public static final int NO_INSERTS_VALUE = 4;
+
+
+public final int getNumber() { return value; }
+
+public static SpaceViolationPolicy valueOf(int value) {
+ switch (value) {
+case 1: return DISABLE;
+case 2: return NO_WRITES_COMPACTIONS;
+case 3: return NO_WRITES;
+case 4: return NO_INSERTS;
+default: return null;
+ }
+}
+
+public static
com.google.protobuf.Internal.EnumLiteMap
+internalGetValueMap() {
+ return internalValueMap;
+}
+private static
com.google.protobuf.Internal.EnumLiteMap
+internalValueMap =
+ new com.google.protobuf.Internal.EnumLiteMap()
{
+public SpaceViolationPolicy findValueByNumber(int number) {
+ return SpaceViolationPolicy.valueOf(number);
+}
+ };
+
+public final com.google.protobuf.Descriptors.EnumValueDescriptor
+getValueDescriptor() {
+ return getDescriptor().getValues().get(index);
+}
+public final com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptorForType() {
+ return getDescriptor();
+}
+public static final com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptor() {
+ return
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.getDescriptor().getEnumTypes().get(3);
+}
+
+private static final SpaceViolationPolicy[] VALUES = values();
+
+public static SpaceViolationPolicy valueOf(
+com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+ if (desc.getType() != getDescriptor()) {
+throw new java.lang.IllegalArgumentException(
+ "EnumValueDescriptor is not for this type.");
+ }
+ return VALUES[desc.getIndex()];
+}
+
+private final int index;
+private final int value;
+
+private SpaceViolationPolicy(int index, int value) {
+ this.index = index;
+ this.value = value;
+}
+
+// @@protoc_insertion_point(enum_scope:hbase.pb.SpaceViolationPolicy)
+ }
+
public interface TimedQuotaOrBuilder
extends com.google.protobuf.MessageOrBuilder {
@@ -3315,6 +3460,20 @@ public final class QuotaProtos {
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f42b498/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
new file mode 100644
index 000..ec8f1bf
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas.policies;
+
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+
+public class BaseViolationPolicyEnforcement {
+
+ static final Append APPEND = new Append(Bytes.toBytes("foo"));
+ static final Delete DELETE = new Delete(Bytes.toBytes("foo"));
+ static final Increment INCREMENT = new Increment(Bytes.toBytes("foo"));
+ static final Put PUT = new Put(Bytes.toBytes("foo"));
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f42b498/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
new file mode 100644
index 000..abe1b9d
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas.policies;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.quotas.SpaceLimitingException;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
+import org.apache.hadoop.hbase.quotas.SpaceViolationPolicyEnforcement;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestBulkLoadCheckingViolationPolicyEnforcement {
+
+ FileSystem fs;
+ RegionServerServices rss;
+ TableName tableName;
+ SpaceViolationPolicyEnforcement policy;
+
+ @Before
+ public void setup() {
+fs = mock(FileSystem.class);
+rss = mock(RegionServerServices.class);
+tableName = TableName.valueOf("foo");
+policy = new BulkLoadVerifyingViolationPolicyEnforcement();
+ }
+
+ @Test
+ public void testFilesUnderLimit() throws Exception {
+final List paths = new ArrayList<>();
+final List
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f42b498/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicyEnforcementFactory.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicyEnforcementFactory.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicyEnforcementFactory.java
new file mode 100644
index 000..6b754b9
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceViolationPolicyEnforcementFactory.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
+import
org.apache.hadoop.hbase.quotas.policies.BulkLoadVerifyingViolationPolicyEnforcement;
+import
org.apache.hadoop.hbase.quotas.policies.DisableTableViolationPolicyEnforcement;
+import
org.apache.hadoop.hbase.quotas.policies.NoInsertsViolationPolicyEnforcement;
+import
org.apache.hadoop.hbase.quotas.policies.NoWritesCompactionsViolationPolicyEnforcement;
+import
org.apache.hadoop.hbase.quotas.policies.NoWritesViolationPolicyEnforcement;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+
+/**
+ * A factory class for instantiating {@link SpaceViolationPolicyEnforcement}
instances.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class SpaceViolationPolicyEnforcementFactory {
+
+ private static final SpaceViolationPolicyEnforcementFactory INSTANCE =
+ new SpaceViolationPolicyEnforcementFactory();
+
+ private SpaceViolationPolicyEnforcementFactory() {}
+
+ /**
+ * Returns an instance of this factory.
+ */
+ public static SpaceViolationPolicyEnforcementFactory getInstance() {
+return INSTANCE;
+ }
+
+ /**
+ * Constructs the appropriate {@link SpaceViolationPolicyEnforcement} for
tables that are
+ * in violation of their space quota.
+ */
+ public SpaceViolationPolicyEnforcement create(
+ RegionServerServices rss, TableName tableName, SpaceQuotaSnapshot
snapshot) {
+SpaceViolationPolicyEnforcement enforcement;
+SpaceQuotaStatus status = snapshot.getQuotaStatus();
+if (!status.isInViolation()) {
+ throw new IllegalArgumentException(tableName + " is not in violation.
Snapshot=" + snapshot);
+}
+switch (status.getPolicy()) {
+ case DISABLE:
+enforcement = new DisableTableViolationPolicyEnforcement();
+break;
+ case NO_WRITES_COMPACTIONS:
+enforcement = new NoWritesCompactionsViolationPolicyEnforcement();
+break;
+ case NO_WRITES:
+enforcement = new NoWritesViolationPolicyEnforcement();
+break;
+ case NO_INSERTS:
+enforcement = new NoInsertsViolationPolicyEnforcement();
+break;
+ default:
+throw new IllegalArgumentException("Unhandled SpaceViolationPolicy: "
+ status.getPolicy());
+}
+enforcement.initialize(rss, tableName, snapshot);
+return enforcement;
+ }
+
+ /**
+ * Creates the "default" {@link SpaceViolationPolicyEnforcement} for a table
that isn't in
+ * violation. This is used to have uniform policy checking for tables in and
not quotas.
+ */
+ public SpaceViolationPolicyEnforcement createWithoutViolation(
+ RegionServerServices rss, TableName tableName, SpaceQuotaSnapshot
snapshot) {
+SpaceQuotaStatus status = snapshot.getQuotaStatus();
+if (status.isInViolation()) {
+ throw new IllegalArgumentException(
+ tableName + " is in violation. Logic error. Snapshot=" + snapshot);
+}
+BulkLoadVerifyingViolationPolicyEnforcement enforcement = new
BulkLoadVerifyingViolationPolicyEnforcement();
+enforcement.initialize(rss, tableName, snapshot);
+return enforcement;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7f42b498/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/TableQuotaSnapshotStore.java
---
HBASE-17803 PE always re-creates table when we specify the split policy
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/23abc900
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/23abc900
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/23abc900
Branch: refs/heads/HBASE-16961
Commit: 23abc90068f0ea75f09c3eecf6ef758f1aee9219
Parents: b299c13
Author: CHIA-PING TSAI
Authored: Sat Mar 18 12:20:07 2017 +0800
Committer: Chia-Ping Tsai
Committed: Sun Mar 19 18:27:54 2017 +0800
--
.../test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/23abc900/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index f8345b1..3addb1a 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -325,7 +325,7 @@ public class PerformanceEvaluation extends Configured
implements Tool {
// recreate the table when user has requested presplit or when existing
// {RegionSplitPolicy,replica count} does not match requested.
if ((exists && opts.presplitRegions != DEFAULT_OPTS.presplitRegions)
- || (!isReadCmd && desc != null && desc.getRegionSplitPolicyClassName()
!= opts.splitPolicy)
+ || (!isReadCmd && desc != null &&
!desc.getRegionSplitPolicyClassName().equals(opts.splitPolicy))
|| (!isReadCmd && desc != null && desc.getRegionReplication() !=
opts.replicas)) {
needsDelete = true;
// wait, why did it delete my table?!?
HBASE-17447 Implement a MasterObserver for automatically deleting space quotas
When a table or namespace is deleted, it would be nice to automatically
delete the quota on said table/NS. It's possible that not all people
would want this functionality so we can leave it up to the user to
configure this Observer.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8271fd53
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8271fd53
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8271fd53
Branch: refs/heads/HBASE-16961
Commit: 8271fd53f81ebba9be9e0f7e8f2cdae6db2a3d71
Parents: 6238590
Author: Josh Elser
Authored: Thu Mar 16 18:54:01 2017 -0400
Committer: Josh Elser
Committed: Mon Mar 20 18:48:51 2017 -0400
--
.../hbase/quotas/MasterSpaceQuotaObserver.java | 85 ++
.../quotas/TestMasterSpaceQuotaObserver.java| 169 +++
src/main/asciidoc/_chapters/ops_mgt.adoc| 17 ++
3 files changed, 271 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/8271fd53/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
new file mode 100644
index 000..a3abf32
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterSpaceQuotaObserver.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.MasterObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+
+/**
+ * An observer to automatically delete space quotas when a table/namespace
+ * are deleted.
+ */
+@InterfaceAudience.Private
+public class MasterSpaceQuotaObserver implements MasterObserver {
+ private CoprocessorEnvironment cpEnv;
+ private Configuration conf;
+ private boolean quotasEnabled = false;
+
+ @Override
+ public void start(CoprocessorEnvironment ctx) throws IOException {
+this.cpEnv = ctx;
+this.conf = cpEnv.getConfiguration();
+this.quotasEnabled = QuotaUtil.isQuotaEnabled(conf);
+ }
+
+ @Override
+ public void postDeleteTable(
+ ObserverContext ctx, TableName tableName)
throws IOException {
+// Do nothing if quotas aren't enabled
+if (!quotasEnabled) {
+ return;
+}
+final MasterServices master = ctx.getEnvironment().getMasterServices();
+final Connection conn = master.getConnection();
+Quotas quotas = QuotaUtil.getTableQuota(master.getConnection(), tableName);
+if (null != quotas && quotas.hasSpace()) {
+ QuotaSettings settings =
QuotaSettingsFactory.removeTableSpaceLimit(tableName);
+ try (Admin admin = conn.getAdmin()) {
+admin.setQuota(settings);
+ }
+}
+ }
+
+ @Override
+ public void postDeleteNamespace(
+ ObserverContext ctx, String namespace)
throws IOException {
+// Do nothing if quotas aren't enabled
+if (!quotasEnabled) {
+ return;
+}
+final MasterServices master = ctx.getEnvironment().getMasterServices();
+final Connection conn = master.getConnection();
+Quotas quotas = QuotaUtil.getNamespaceQuota(master.getConnection(),
namespace);
+if (null != quotas && quotas.hasSpace()) {
+ QuotaSettings settings =
QuotaSettingsFactory
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e042231/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
index 8f368e9..899ae9b 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/RegionServerStatusProtos.java
@@ -10164,6 +10164,1912 @@ public final class RegionServerStatusProtos {
}
+ public interface RegionSpaceUseOrBuilder extends
+ // @@protoc_insertion_point(interface_extends:hbase.pb.RegionSpaceUse)
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
+
+/**
+ *
+ * A region identifier
+ *
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+boolean hasRegion();
+/**
+ *
+ * A region identifier
+ *
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo
getRegion();
+/**
+ *
+ * A region identifier
+ *
+ *
+ * optional .hbase.pb.RegionInfo region = 1;
+ */
+
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfoOrBuilder
getRegionOrBuilder();
+
+/**
+ *
+ * The size in bytes of the region
+ *
+ *
+ * optional uint64 size = 2;
+ */
+boolean hasSize();
+/**
+ *
+ * The size in bytes of the region
+ *
+ *
+ * optional uint64 size = 2;
+ */
+long getSize();
+ }
+ /**
+ * Protobuf type {@code hbase.pb.RegionSpaceUse}
+ */
+ public static final class RegionSpaceUse extends
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3
implements
+ // @@protoc_insertion_point(message_implements:hbase.pb.RegionSpaceUse)
+ RegionSpaceUseOrBuilder {
+// Use RegionSpaceUse.newBuilder() to construct.
+private
RegionSpaceUse(org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.Builder
builder) {
+ super(builder);
+}
+private RegionSpaceUse() {
+ size_ = 0L;
+}
+
+@java.lang.Override
+public final
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+ return this.unknownFields;
+}
+private RegionSpaceUse(
+org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream
input,
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite
extensionRegistry)
+throws
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
{
+ this();
+ int mutable_bitField0_ = 0;
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.Builder
unknownFields =
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+boolean done = false;
+while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+case 0:
+ done = true;
+ break;
+default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+ }
+ break;
+}
+case 10: {
+
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.Builder
subBuilder = null;
+ if (((bitField0_ & 0x0001) == 0x0001)) {
+subBuilder = region_.toBuilder();
+ }
+ region_ =
input.readMessage(org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo.PARSER,
extensionRegistry);
+ if (subBuilder != null) {
+subBuilder.mergeFrom(region_);
+region_ = subBuilder.buildPartial();
+ }
+ bitField0_ |= 0x0001;
+ break;
+}
+case 16: {
+ bitField0_ |= 0x0002;
+ size_ = input.readUInt64();
+ break;
+}
+ }
+}
+ } catch
(org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
e) {
+throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+throw new
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException(
+e).setUnfinishedMessage(this);
+ } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/b99e9cf9/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
new file mode 100644
index 000..ec8f1bf
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/BaseViolationPolicyEnforcement.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas.policies;
+
+import org.apache.hadoop.hbase.client.Append;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Increment;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+
+public class BaseViolationPolicyEnforcement {
+
+ static final Append APPEND = new Append(Bytes.toBytes("foo"));
+ static final Delete DELETE = new Delete(Bytes.toBytes("foo"));
+ static final Increment INCREMENT = new Increment(Bytes.toBytes("foo"));
+ static final Put PUT = new Put(Bytes.toBytes("foo"));
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b99e9cf9/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
new file mode 100644
index 000..abe1b9d
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/policies/TestBulkLoadCheckingViolationPolicyEnforcement.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas.policies;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.quotas.SpaceLimitingException;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
+import org.apache.hadoop.hbase.quotas.SpaceViolationPolicyEnforcement;
+import org.apache.hadoop.hbase.regionserver.RegionServerServices;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestBulkLoadCheckingViolationPolicyEnforcement {
+
+ FileSystem fs;
+ RegionServerServices rss;
+ TableName tableName;
+ SpaceViolationPolicyEnforcement policy;
+
+ @Before
+ public void setup() {
+fs = mock(FileSystem.class);
+rss = mock(RegionServerServices.class);
+tableName = TableName.valueOf("foo");
+policy = new BulkLoadVerifyingViolationPolicyEnforcement();
+ }
+
+ @Test
+ public void testFilesUnderLimit() throws Exception {
+final List paths = new ArrayList<>();
+final List
HBASE-17003 Documentation updates for space quotas
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/201da141
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/201da141
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/201da141
Branch: refs/heads/HBASE-16961
Commit: 201da1411c96735cdf6a8f0aa1e8c84894acd417
Parents: 6d1558c
Author: Josh Elser
Authored: Thu Mar 16 16:21:14 2017 -0400
Committer: Josh Elser
Committed: Mon Mar 20 18:31:30 2017 -0400
--
src/main/asciidoc/_chapters/ops_mgt.adoc | 64 ++-
1 file changed, 63 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/201da141/src/main/asciidoc/_chapters/ops_mgt.adoc
--
diff --git a/src/main/asciidoc/_chapters/ops_mgt.adoc
b/src/main/asciidoc/_chapters/ops_mgt.adoc
index e4c077f..f9009f3 100644
--- a/src/main/asciidoc/_chapters/ops_mgt.adoc
+++ b/src/main/asciidoc/_chapters/ops_mgt.adoc
@@ -1705,7 +1705,7 @@ handling multiple workloads:
[[quota]]
=== Quotas
-HBASE-11598 introduces quotas, which allow you to throttle requests based on
+HBASE-11598 introduces RPC quotas, which allow you to throttle requests based
on
the following limits:
. <>
@@ -1885,6 +1885,68 @@ at the same time and that fewer scans can be executed at
the same time. A value
`0.9` will give more queue/handlers to scans, so the number of scans executed
will
increase and the number of gets will decrease.
+[[space-quotas]]
+=== Space Quotas
+
+link:https://issues.apache.org/jira/browse/HBASE-16961[HBASE-16961] introduces
a new type of
+quotas for HBase to leverage: filesystem quotas. These "space" quotas limit
the amount of space
+on the filesystem that HBase namespaces and tables can consume. If a user,
malicious or ignorant,
+has the ability to write data into HBase, with enough time, that user can
effectively crash HBase
+(or worse HDFS) by consuming all available space. When there is no filesystem
space available,
+HBase crashes because it can no longer create/sync data to the write-ahead log.
+
+This feature allows a for a limit to be set on the size of a table or
namespace. When a space quota is set
+on a namespace, the quota's limit applies to the sum of usage of all tables in
that namespace.
+When a table with a quota exists in a namespace with a quota, the table quota
takes priority
+over the namespace quota. This allows for a scenario where a large limit can
be placed on
+a collection of tables, but a single table in that collection can have a
fine-grained limit set.
+
+The existing `set_quota` and `list_quota` HBase shell commands can be used to
interact with
+space quotas. Space quotas are quotas with a `TYPE` of `SPACE` and have
`LIMIT` and `POLICY`
+attributes. The `LIMIT` is a string that refers to the amount of space on the
filesystem
+that the quota subject (e.g. the table or namespace) may consume. For example,
valid values
+of `LIMIT` are `'10G'`, `'2T'`, or `'256M'`. The `POLICY` refers to the action
that HBase will
+take when the quota subject's usage exceeds the `LIMIT`. The following are
valid `POLICY` values.
+
+* `NO_INSERTS` - No new data may be written (e.g. `Put`, `Increment`,
`Append`).
+* `NO_WRITES` - Same as `NO_INSERTS` but `Deletes` are also disallowed.
+* `NO_WRITES_COMPACTIONS` - Same as `NO_WRITES` but compactions are also
disallowed.
+* `DISABLE` - The table(s) are disabled, preventing all read/write access.
+
+.Setting simple space quotas
+
+# Sets a quota on the table 't1' with a limit of 1GB, disallowing
Puts/Increments/Appends when the table exceeds 1GB
+hbase> set_quota TYPE => SPACE, TABLE => 't1', LIMIT => '1G', POLICY =>
NO_INSERTS
+
+# Sets a quota on the namespace 'ns1' with a limit of 50TB, disallowing
Puts/Increments/Appends/Deletes
+hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => '50T', POLICY =>
NO_WRITES
+
+# Sets a quota on the table 't3' with a limit of 2TB, disallowing any writes
and compactions when the table exceeds 2TB.
+hbase> set_quota TYPE => SPACE, TABLE => 't3', LIMIT => '2T', POLICY =>
NO_WRITES_COMPACTIONS
+
+# Sets a quota on the table 't2' with a limit of 50GB, disabling the table
when it exceeds 50GB
+hbase> set_quota TYPE => SPACE, TABLE => 't2', LIMIT => '50G', POLICY =>
DISABLE
+
+
+Consider the following scenario to set up quotas on a namespace, overriding
the quota on tables in that namespace
+
+.Table and Namespace space quotas
+
+hbase> create_namespace 'ns1'
+hbase> create 'ns1:t1'
+hbase> create 'ns1:t2'
+hbase> create 'ns1:t3'
+hbase> set_quota TYPE => SPACE, NAMESPACE => 'ns1', LIMIT => '100T', POLICY =>
NO_INSERTS
+hbase> set_quota TYPE => SPACE, TABLE => 'ns1:t2', LIMIT => '200G', POLICY =>
NO_WRITES
+h
http://git-wip-us.apache.org/repos/asf/hbase/blob/341cf7e1/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index d8cd701..f207472 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -7476,6 +7476,5554 @@ public final class QuotaProtos {
}
+ public interface GetSpaceQuotaRegionSizesRequestOrBuilder extends
+ //
@@protoc_insertion_point(interface_extends:hbase.pb.GetSpaceQuotaRegionSizesRequest)
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.MessageOrBuilder {
+ }
+ /**
+ * Protobuf type {@code hbase.pb.GetSpaceQuotaRegionSizesRequest}
+ */
+ public static final class GetSpaceQuotaRegionSizesRequest extends
+ org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3
implements
+ //
@@protoc_insertion_point(message_implements:hbase.pb.GetSpaceQuotaRegionSizesRequest)
+ GetSpaceQuotaRegionSizesRequestOrBuilder {
+// Use GetSpaceQuotaRegionSizesRequest.newBuilder() to construct.
+private
GetSpaceQuotaRegionSizesRequest(org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.Builder
builder) {
+ super(builder);
+}
+private GetSpaceQuotaRegionSizesRequest() {
+}
+
+@java.lang.Override
+public final
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet
+getUnknownFields() {
+ return this.unknownFields;
+}
+private GetSpaceQuotaRegionSizesRequest(
+org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedInputStream
input,
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.ExtensionRegistryLite
extensionRegistry)
+throws
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
{
+ this();
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.Builder
unknownFields =
+
org.apache.hadoop.hbase.shaded.com.google.protobuf.UnknownFieldSet.newBuilder();
+ try {
+boolean done = false;
+while (!done) {
+ int tag = input.readTag();
+ switch (tag) {
+case 0:
+ done = true;
+ break;
+default: {
+ if (!parseUnknownField(input, unknownFields,
+ extensionRegistry, tag)) {
+done = true;
+ }
+ break;
+}
+ }
+}
+ } catch
(org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException
e) {
+throw e.setUnfinishedMessage(this);
+ } catch (java.io.IOException e) {
+throw new
org.apache.hadoop.hbase.shaded.com.google.protobuf.InvalidProtocolBufferException(
+e).setUnfinishedMessage(this);
+ } finally {
+this.unknownFields = unknownFields.build();
+makeExtensionsImmutable();
+ }
+}
+public static final
org.apache.hadoop.hbase.shaded.com.google.protobuf.Descriptors.Descriptor
+getDescriptor() {
+ return
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.internal_static_hbase_pb_GetSpaceQuotaRegionSizesRequest_descriptor;
+}
+
+protected
org.apache.hadoop.hbase.shaded.com.google.protobuf.GeneratedMessageV3.FieldAccessorTable
+internalGetFieldAccessorTable() {
+ return
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.internal_static_hbase_pb_GetSpaceQuotaRegionSizesRequest_fieldAccessorTable
+ .ensureFieldAccessorsInitialized(
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest.class,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.GetSpaceQuotaRegionSizesRequest.Builder.class);
+}
+
+private byte memoizedIsInitialized = -1;
+public final boolean isInitialized() {
+ byte isInitialized = memoizedIsInitialized;
+ if (isInitialized == 1) return true;
+ if (isInitialized == 0) return false;
+
+ memoizedIsInitialized = 1;
+ return true;
+}
+
+public void
writeTo(org.apache.hadoop.hbase.shaded.com.google.protobuf.CodedOutputStream
output)
+throws java.io.IOException {
+ unknownFields.writeTo(output);
+}
+
+public int getSerializedSize() {
+ int size = memoizedSize;
+ if (size != -1) return size;
+
+ size = 0;
+ size += unknownFields.getSerializedSize();
+ memoizedSize = size;
+ return size;
+}
+
+private static final long serialVersionUID = 0
HBASE-17802 Add note that minor versions can add methods to Interfaces
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7c03a213
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7c03a213
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7c03a213
Branch: refs/heads/HBASE-16961
Commit: 7c03a213ffc074c941333677065031a5c2c12d41
Parents: 261aa94
Author: Michael Stack
Authored: Fri Mar 17 16:53:47 2017 -0700
Committer: Michael Stack
Committed: Sun Mar 19 14:51:11 2017 -0700
--
src/main/asciidoc/_chapters/upgrading.adoc | 7 +--
1 file changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/7c03a213/src/main/asciidoc/_chapters/upgrading.adoc
--
diff --git a/src/main/asciidoc/_chapters/upgrading.adoc
b/src/main/asciidoc/_chapters/upgrading.adoc
index b0a5565..df5bbfe 100644
--- a/src/main/asciidoc/_chapters/upgrading.adoc
+++ b/src/main/asciidoc/_chapters/upgrading.adoc
@@ -74,12 +74,15 @@ In addition to the usual API versioning considerations
HBase has other compatibi
* An API needs to be deprecated for a major version before we will
change/remove it.
* APIs available in a patch version will be available in all later patch
versions. However, new APIs may be added which will not be available in earlier
patch versions.
* New APIs introduced in a patch version will only be added in a source
compatible way footnote:[See 'Source Compatibility'
https://blogs.oracle.com/darcy/entry/kinds_of_compatibility]: i.e. code that
implements public APIs will continue to compile.
-* Example: A user using a newly deprecated API does not need to modify
application code with HBase API calls until the next major version.
+** Example: A user using a newly deprecated API does not need to modify
application code with HBase API calls until the next major version.
+*
.Client Binary compatibility
* Client code written to APIs available in a given patch release can run
unchanged (no recompilation needed) against the new jars of later patch
versions.
* Client code written to APIs available in a given patch release might not run
against the old jars from an earlier patch version.
-* Example: Old compiled client code will work unchanged with the new jars.
+** Example: Old compiled client code will work unchanged with the new jars.
+* If a Client implements an HBase Interface, a recompile MAY be required
upgrading to a newer minor version (See release notes
+for warning about incompatible changes). All effort will be made to provide a
default implementation so this case should not arise.
.Server-Side Limited API compatibility (taken from Hadoop)
* Internal APIs are marked as Stable, Evolving, or Unstable
HBASE-16995 Build client Java API and client protobuf messages - addendum fixes
white spaces (Josh Elser)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d688893c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d688893c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d688893c
Branch: refs/heads/HBASE-16961
Commit: d688893c99cf23d826d0832f0ae8bee0c041a660
Parents: 87fa04c
Author: tedyu
Authored: Thu Nov 17 10:42:18 2016 -0800
Committer: Josh Elser
Committed: Mon Mar 20 17:39:21 2017 -0400
--
.../hbase/quotas/TestQuotaSettingsFactory.java| 2 +-
.../shaded/protobuf/generated/MasterProtos.java | 2 +-
.../shaded/protobuf/generated/QuotaProtos.java| 18 +-
.../hbase/protobuf/generated/QuotaProtos.java | 4 ++--
4 files changed, 13 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/d688893c/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
--
diff --git
a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
index 17015d6..e0012a7 100644
---
a/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
+++
b/hbase-client/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaSettingsFactory.java
@@ -44,7 +44,7 @@ import org.junit.experimental.categories.Category;
*/
@Category(SmallTests.class)
public class TestQuotaSettingsFactory {
-
+
@Test
public void testAllQuotasAddedToList() {
final SpaceQuota spaceQuota = SpaceQuota.newBuilder()
http://git-wip-us.apache.org/repos/asf/hbase/blob/d688893c/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
index cfa1bf3..668b4f6 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/MasterProtos.java
@@ -63752,7 +63752,7 @@ public final class MasterProtos {
* optional .hbase.pb.SpaceLimitRequest space_limit = 8;
*/
private
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequestOrBuilder>
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequestOrBuilder>
getSpaceLimitFieldBuilder() {
if (spaceLimitBuilder_ == null) {
spaceLimitBuilder_ = new
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
http://git-wip-us.apache.org/repos/asf/hbase/blob/d688893c/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
index a715115..673fb2c 100644
---
a/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol-shaded/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/generated/QuotaProtos.java
@@ -4362,7 +4362,7 @@ public final class QuotaProtos {
* optional .hbase.pb.SpaceQuota space = 3;
*/
private
org.apache.hadoop.hbase.shaded.com.google.protobuf.SingleFieldBuilderV3<
-
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuotaOrBuilder>
+
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota,
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota.Builder,
org.apache.hadoop.hbase.shaded.protobuf.gene
HBASE-16084 Cleaned up the stale references in Javadoc
Signed-off-by: tedyu
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/55d6dcaf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/55d6dcaf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/55d6dcaf
Branch: refs/heads/HBASE-16961
Commit: 55d6dcaf877cc5223e679736eb613173229c18be
Parents: 4088f82
Author: Jan Hentschel
Authored: Sun Mar 19 20:49:28 2017 +0100
Committer: tedyu
Committed: Mon Mar 20 10:55:36 2017 -0700
--
.../org/apache/hadoop/hbase/HTableDescriptor.java | 14 +++---
.../apache/hadoop/hbase/client/AsyncProcess.java | 6 +++---
.../hadoop/hbase/client/ConnectionFactory.java | 2 +-
.../apache/hadoop/hbase/client/MasterCallable.java | 2 +-
.../java/org/apache/hadoop/hbase/client/Query.java | 2 +-
.../hbase/client/coprocessor/package-info.java | 8
.../FirstKeyValueMatchingQualifiersFilter.java | 2 +-
.../hadoop/hbase/ipc/ServerRpcController.java | 4 ++--
.../hbase/zookeeper/RecoverableZooKeeper.java | 2 +-
.../java/org/apache/hadoop/hbase/nio/ByteBuff.java | 2 +-
.../org/apache/hadoop/hbase/util/OrderedBytes.java | 2 +-
.../hadoop/hbase/HBaseCommonTestingUtility.java| 3 +--
.../codec/prefixtree/scanner/CellSearcher.java | 2 +-
.../store/wal/ProcedureWALFormatReader.java| 3 +--
.../apache/hadoop/hbase/backup/HFileArchiver.java | 2 --
.../hbase/backup/example/HFileArchiveManager.java | 3 ++-
.../hadoop/hbase/backup/util/RestoreTool.java | 4 ++--
.../apache/hadoop/hbase/constraint/Constraint.java | 8
.../hbase/io/hfile/CompoundBloomFilterWriter.java | 2 +-
.../apache/hadoop/hbase/io/hfile/HFileBlock.java | 2 +-
.../hadoop/hbase/io/hfile/HFileBlockIndex.java | 2 +-
.../apache/hadoop/hbase/mapreduce/RowCounter.java | 3 +--
.../hadoop/hbase/master/TableNamespaceManager.java | 2 +-
.../master/balancer/StochasticLoadBalancer.java| 4 ++--
.../regionserver/MiniBatchOperationInProgress.java | 4 ++--
.../hadoop/hbase/regionserver/StoreFileReader.java | 2 +-
.../hadoop/hbase/regionserver/wal/FSHLog.java | 8
.../regionserver/wal/SequenceIdAccounting.java | 6 +++---
.../hadoop/hbase/regionserver/wal/SyncFuture.java | 4 ++--
.../access/CoprocessorWhitelistMasterObserver.java | 2 +-
.../hbase/security/access/TableAuthManager.java| 2 +-
.../apache/hadoop/hbase/HBaseTestingUtility.java | 6 +++---
.../apache/hadoop/hbase/TestMetaTableLocator.java | 8
.../hbase/TestPartialResultsFromClientSide.java| 2 +-
.../org/apache/hadoop/hbase/TestSerialization.java | 4 ++--
.../hbase/client/TestMultipleTimestamps.java | 2 +-
.../TestFirstKeyValueMatchingQualifiersFilter.java | 2 +-
.../apache/hadoop/hbase/io/hfile/TestSeekTo.java | 2 +-
.../hbase/mapreduce/TestHFileOutputFormat2.java| 17 +++--
.../hadoop/hbase/mapreduce/TestImportExport.java | 2 +-
.../hadoop/hbase/master/MockRegionServer.java | 4 ++--
.../hadoop/hbase/master/TestWarmupRegion.java | 2 +-
.../hbase/procedure/TestProcedureMember.java | 4 ++--
.../hbase/regionserver/DataBlockEncodingTool.java | 2 +-
.../hbase/regionserver/OOMERegionServer.java | 3 +--
.../TestRegionMergeTransactionOnCluster.java | 6 --
.../org/apache/hadoop/hbase/util/LoadTestTool.java | 2 +-
.../hadoop/hbase/util/MultiThreadedAction.java | 4 ++--
48 files changed, 86 insertions(+), 100 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/55d6dcaf/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
index a49cf1c..25fd896 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/HTableDescriptor.java
@@ -723,7 +723,7 @@ public class HTableDescriptor implements
Comparable {
/**
* This sets the class associated with the region split policy which
* determines when a region split should occur. The class used by
- * default is defined in {@link
org.apache.hadoop.hbase.regionserver.RegionSplitPolicy}
+ * default is defined in
org.apache.hadoop.hbase.regionserver.RegionSplitPolicy
* @param clazz the class name
*/
public HTableDescriptor setRegionSplitPolicyClassName(String clazz) {
@@ -734,7 +734,7 @@ public class HTableDescriptor implements
Comparable {
/**
* This gets the class associated with the region split policy which
* determines when a region split should occu
http://git-wip-us.apache.org/repos/asf/hbase/blob/75d0f49d/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
new file mode 100644
index 000..0f1453e
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
@@ -0,0 +1,344 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupInfo;
+import org.apache.hadoop.hbase.backup.impl.BackupSystemTable.WALItem;
+import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
+import org.apache.hadoop.hbase.backup.util.BackupUtils;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
+
+/**
+ * After a full backup was created, the incremental backup will only store the
changes made after
+ * the last full or incremental backup. Creating the backup copies the
logfiles in .logs and
+ * .oldlogs since the last backup timestamp.
+ */
+@InterfaceAudience.Private
+public class IncrementalBackupManager extends BackupManager {
+ public static final Log LOG =
LogFactory.getLog(IncrementalBackupManager.class);
+
+ public IncrementalBackupManager(Connection conn, Configuration conf) throws
IOException {
+super(conn, conf);
+ }
+
+ /**
+ * Obtain the list of logs that need to be copied out for this incremental
backup. The list is set
+ * in BackupInfo.
+ * @param conn the Connection
+ * @param backupInfo backup info
+ * @return The new HashMap of RS log timestamps after the log roll for this
incremental backup.
+ * @throws IOException exception
+ */
+ public HashMap getIncrBackupLogFileList(Connection conn,
BackupInfo backupInfo)
+ throws IOException {
+List logList;
+HashMap newTimestamps;
+HashMap previousTimestampMins;
+
+String savedStartCode = readBackupStartCode();
+
+// key: tableName
+// value:
+HashMap> previousTimestampMap =
readLogTimestampMap();
+
+previousTimestampMins =
BackupUtils.getRSLogTimestampMins(previousTimestampMap);
+
+if (LOG.isDebugEnabled()) {
+ LOG.debug("StartCode " + savedStartCode + "for backupID " +
backupInfo.getBackupId());
+}
+// get all new log files from .logs and .oldlogs after last TS and before
new timestamp
+if (savedStartCode == null || previousTimestampMins == null
+|| previousTimestampMins.isEmpty()) {
+ throw new IOException(
+ "Cannot read any previous back up timestamps from backup system
table. "
+ + "In order to create an incremental backup, at least one full
backup is needed.");
+}
+
+LOG.info("Execute roll log procedure for incremental backup ...");
+HashMap props = new HashMap();
+props.put("backupRoot", backupInfo.getBackupRootDir());
+
+try (Admin admin = conn.getAdmin();) {
+
+
admin.execProcedure(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE,
+LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props);
+
+}
+newTimestamps = readRegionServerLastLogRollResult();
+
+logList = getLogFilesForNewBackup(previousTimestampMins, newTimestamps,
conf, savedStartCode);
+List logFromSystemTable =
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/87fa04cc/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
--
diff --git
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
index 05894b9..1925828 100644
---
a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
+++
b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/QuotaProtos.java
@@ -217,12 +217,20 @@ public final class QuotaProtos {
* THROTTLE = 1;
*/
THROTTLE(0, 1),
+/**
+ * SPACE = 2;
+ */
+SPACE(1, 2),
;
/**
* THROTTLE = 1;
*/
public static final int THROTTLE_VALUE = 1;
+/**
+ * SPACE = 2;
+ */
+public static final int SPACE_VALUE = 2;
public final int getNumber() { return value; }
@@ -230,6 +238,7 @@ public final class QuotaProtos {
public static QuotaType valueOf(int value) {
switch (value) {
case 1: return THROTTLE;
+case 2: return SPACE;
default: return null;
}
}
@@ -281,6 +290,142 @@ public final class QuotaProtos {
// @@protoc_insertion_point(enum_scope:hbase.pb.QuotaType)
}
+ /**
+ * Protobuf enum {@code hbase.pb.SpaceViolationPolicy}
+ *
+ *
+ * Defines what action should be taken when the SpaceQuota is violated
+ *
+ */
+ public enum SpaceViolationPolicy
+ implements com.google.protobuf.ProtocolMessageEnum {
+/**
+ * DISABLE = 1;
+ *
+ *
+ * Disable the table(s)
+ *
+ */
+DISABLE(0, 1),
+/**
+ * NO_WRITES_COMPACTIONS = 2;
+ *
+ *
+ * No writes, bulk-loads, or compactions
+ *
+ */
+NO_WRITES_COMPACTIONS(1, 2),
+/**
+ * NO_WRITES = 3;
+ *
+ *
+ * No writes or bulk-loads
+ *
+ */
+NO_WRITES(2, 3),
+/**
+ * NO_INSERTS = 4;
+ *
+ *
+ * No puts or bulk-loads, but deletes are allowed
+ *
+ */
+NO_INSERTS(3, 4),
+;
+
+/**
+ * DISABLE = 1;
+ *
+ *
+ * Disable the table(s)
+ *
+ */
+public static final int DISABLE_VALUE = 1;
+/**
+ * NO_WRITES_COMPACTIONS = 2;
+ *
+ *
+ * No writes, bulk-loads, or compactions
+ *
+ */
+public static final int NO_WRITES_COMPACTIONS_VALUE = 2;
+/**
+ * NO_WRITES = 3;
+ *
+ *
+ * No writes or bulk-loads
+ *
+ */
+public static final int NO_WRITES_VALUE = 3;
+/**
+ * NO_INSERTS = 4;
+ *
+ *
+ * No puts or bulk-loads, but deletes are allowed
+ *
+ */
+public static final int NO_INSERTS_VALUE = 4;
+
+
+public final int getNumber() { return value; }
+
+public static SpaceViolationPolicy valueOf(int value) {
+ switch (value) {
+case 1: return DISABLE;
+case 2: return NO_WRITES_COMPACTIONS;
+case 3: return NO_WRITES;
+case 4: return NO_INSERTS;
+default: return null;
+ }
+}
+
+public static
com.google.protobuf.Internal.EnumLiteMap
+internalGetValueMap() {
+ return internalValueMap;
+}
+private static
com.google.protobuf.Internal.EnumLiteMap
+internalValueMap =
+ new com.google.protobuf.Internal.EnumLiteMap()
{
+public SpaceViolationPolicy findValueByNumber(int number) {
+ return SpaceViolationPolicy.valueOf(number);
+}
+ };
+
+public final com.google.protobuf.Descriptors.EnumValueDescriptor
+getValueDescriptor() {
+ return getDescriptor().getValues().get(index);
+}
+public final com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptorForType() {
+ return getDescriptor();
+}
+public static final com.google.protobuf.Descriptors.EnumDescriptor
+getDescriptor() {
+ return
org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.getDescriptor().getEnumTypes().get(3);
+}
+
+private static final SpaceViolationPolicy[] VALUES = values();
+
+public static SpaceViolationPolicy valueOf(
+com.google.protobuf.Descriptors.EnumValueDescriptor desc) {
+ if (desc.getType() != getDescriptor()) {
+throw new java.lang.IllegalArgumentException(
+ "EnumValueDescriptor is not for this type.");
+ }
+ return VALUES[desc.getIndex()];
+}
+
+private final int index;
+private final int value;
+
+private SpaceViolationPolicy(int index, int value) {
+ this.index = index;
+ this.value = value;
+}
+
+// @@protoc_insertion_point(enum_scope:hbase.pb.SpaceViolationPolicy)
+ }
+
public interface TimedQuotaOrBuilder
extends com.google.protobuf.MessageOrBuilder {
@@ -3315,6 +3460,20 @@ public final class QuotaProtos {
HBASE-16998 Implement Master-side analysis of region space reports
Adds a new Chore to the Master that analyzes the reports that are
sent by RegionServers. The Master must then, for all tables with
quotas, determine the tables that are violating quotas and move
those tables into violation. Similarly, tables no longer violating
the quota can be moved out of violation.
The Chore is the "stateful" bit, managing which tables are and
are not in violation. Everything else is just performing
computation and informing the Chore on the updated state.
Added InterfaceAudience annotations and clean up the QuotaObserverChore
constructor. Cleaned up some javadoc and QuotaObserverChore. Reuse
the QuotaViolationStore impl objects.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a19a95df
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a19a95df
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a19a95df
Branch: refs/heads/HBASE-16961
Commit: a19a95dff66fc205efb032185944d910c8ea79b2
Parents: d9ebc1e
Author: Josh Elser
Authored: Tue Nov 8 18:55:12 2016 -0500
Committer: Josh Elser
Committed: Mon Mar 20 17:46:40 2017 -0400
--
.../hadoop/hbase/quotas/QuotaRetriever.java | 29 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 20 +
.../hadoop/hbase/quotas/MasterQuotaManager.java | 1 +
.../quotas/NamespaceQuotaViolationStore.java| 127
.../hadoop/hbase/quotas/QuotaObserverChore.java | 618 +++
.../hbase/quotas/QuotaViolationStore.java | 89 +++
.../quotas/SpaceQuotaViolationNotifier.java | 44 ++
.../SpaceQuotaViolationNotifierForTest.java | 50 ++
.../hbase/quotas/TableQuotaViolationStore.java | 127
.../TestNamespaceQuotaViolationStore.java | 156 +
.../hbase/quotas/TestQuotaObserverChore.java| 106
.../TestQuotaObserverChoreWithMiniCluster.java | 596 ++
.../hadoop/hbase/quotas/TestQuotaTableUtil.java | 4 -
.../quotas/TestTableQuotaViolationStore.java| 151 +
.../hbase/quotas/TestTablesWithQuotas.java | 198 ++
15 files changed, 2306 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a19a95df/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
index fecd2d1..8cd5cf0 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaRetriever.java
@@ -22,6 +22,7 @@ import java.io.Closeable;
import java.io.IOException;
import java.util.Iterator;
import java.util.LinkedList;
+import java.util.Objects;
import java.util.Queue;
import org.apache.commons.logging.Log;
@@ -56,11 +57,23 @@ public class QuotaRetriever implements Closeable,
Iterable {
private Connection connection;
private Table table;
- private QuotaRetriever() {
+ /**
+ * Should QutoaRetriever manage the state of the connection, or leave it be.
+ */
+ private boolean isManagedConnection = false;
+
+ QuotaRetriever() {
}
void init(final Configuration conf, final Scan scan) throws IOException {
-this.connection = ConnectionFactory.createConnection(conf);
+// Set this before creating the connection and passing it down to make sure
+// it's cleaned up if we fail to construct the Scanner.
+this.isManagedConnection = true;
+init(ConnectionFactory.createConnection(conf), scan);
+ }
+
+ void init(final Connection conn, final Scan scan) throws IOException {
+this.connection = Objects.requireNonNull(conn);
this.table = this.connection.getTable(QuotaTableUtil.QUOTA_TABLE_NAME);
try {
scanner = table.getScanner(scan);
@@ -79,10 +92,14 @@ public class QuotaRetriever implements Closeable,
Iterable {
this.table.close();
this.table = null;
}
-if (this.connection != null) {
- this.connection.close();
- this.connection = null;
+// Null out the connection on close() even if we didn't explicitly close it
+// to maintain typical semantics.
+if (isManagedConnection) {
+ if (this.connection != null) {
+this.connection.close();
+ }
}
+this.connection = null;
}
public QuotaSettings next() throws IOException {
@@ -182,4 +199,4 @@ public class QuotaRetriever implements Closeable,
Iterable {
scanner.init(conf, scan);
return scanner;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a19a95df/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
---
http://git-wip-us.apache.org/repos/asf/hbase/blob/75d0f49d/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java
new file mode 100644
index 000..47e428c
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/LogRollMasterProcedureManager.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup.master;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.CoordinatedStateManagerFactory;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
+import org.apache.hadoop.hbase.backup.impl.BackupManager;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
+import org.apache.hadoop.hbase.errorhandling.ForeignException;
+import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
+import org.apache.hadoop.hbase.master.MasterServices;
+import org.apache.hadoop.hbase.master.MetricsMaster;
+import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
+import org.apache.hadoop.hbase.procedure.Procedure;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinator;
+import org.apache.hadoop.hbase.procedure.ProcedureCoordinatorRpcs;
+import org.apache.hadoop.hbase.procedure.RegionServerProcedureManager;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameStringPair;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.ProcedureDescription;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Master procedure manager for coordinated cluster-wide WAL roll operation,
which is run during
+ * backup operation, see {@link MasterProcedureManager} and and {@link
RegionServerProcedureManager}
+ */
+@InterfaceAudience.Private
+public class LogRollMasterProcedureManager extends MasterProcedureManager {
+
+ public static final String ROLLLOG_PROCEDURE_SIGNATURE = "rolllog-proc";
+ public static final String ROLLLOG_PROCEDURE_NAME = "rolllog";
+ private static final Log LOG =
LogFactory.getLog(LogRollMasterProcedureManager.class);
+
+ private MasterServices master;
+ private ProcedureCoordinator coordinator;
+ private boolean done;
+
+ @Override
+ public void stop(String why) {
+LOG.info("stop: " + why);
+ }
+
+ @Override
+ public boolean isStopped() {
+return false;
+ }
+
+ @Override
+ public void initialize(MasterServices master, MetricsMaster metricsMaster)
+ throws KeeperException, IOException, UnsupportedOperationException {
+this.master = master;
+this.done = false;
+
+// setup the default procedure coordinator
+String name = master.getServerName().toString();
+ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, 1);
+BaseCoordinatedStateManager coordManager =
+(BaseCoordinatedStateManager) CoordinatedStateManagerFactory
+.getCoordinatedStateManager(master.getConfiguration());
+coordManager.initialize(master);
+
+ProcedureCoordinatorRpcs comms =
+coordManager.getProcedureCoordinatorRpcs(getProcedureSignature(),
name);
+
+this.coordinator = new ProcedureCoordinator(comms, tpool);
+ }
+
+ @Override
+ public String getProcedureSignature() {
+return ROLLLOG_PROCEDURE_SIGNATURE;
+ }
+
+ @Override
+ public void execProcedure(ProcedureDescription desc) throws IOException {
+if (!isBackupEnabled()) {
+ LOG.warn("Backup is not enabled. Check your " +
BackupRestoreConstants.BACKUP_ENABLE_KEY
+ + " setting");
+ return;
+}
+this.done = false;
+// start the process on the RS
+ForeignExceptionDispatcher monitor = new
HBASE-17602 Reduce some quota chore periods/delays
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/517eb828
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/517eb828
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/517eb828
Branch: refs/heads/HBASE-16961
Commit: 517eb828507c1ae07f7521fdf95b3e7fc454027a
Parents: a44e50f
Author: Josh Elser
Authored: Tue Feb 7 11:21:08 2017 -0500
Committer: Josh Elser
Committed: Mon Mar 20 18:19:31 2017 -0400
--
.../java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java | 4 ++--
.../org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/517eb828/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index b9f4592..7f894e4 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -55,11 +55,11 @@ public class QuotaObserverChore extends ScheduledChore {
private static final Log LOG = LogFactory.getLog(QuotaObserverChore.class);
static final String QUOTA_OBSERVER_CHORE_PERIOD_KEY =
"hbase.master.quotas.observer.chore.period";
- static final int QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 5
minutes in millis
+ static final int QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 1; // 1
minutes in millis
static final String QUOTA_OBSERVER_CHORE_DELAY_KEY =
"hbase.master.quotas.observer.chore.delay";
- static final long QUOTA_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1
minute
+ static final long QUOTA_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 15L; // 15
seconds in millis
static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_KEY =
"hbase.master.quotas.observer.chore.timeunit";
http://git-wip-us.apache.org/repos/asf/hbase/blob/517eb828/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
index e1a2693..8587e79 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/SpaceQuotaRefresherChore.java
@@ -44,11 +44,11 @@ public class SpaceQuotaRefresherChore extends
ScheduledChore {
static final String POLICY_REFRESHER_CHORE_PERIOD_KEY =
"hbase.regionserver.quotas.policy.refresher.chore.period";
- static final int POLICY_REFRESHER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 5
minutes in millis
+ static final int POLICY_REFRESHER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 1; // 1
minute in millis
static final String POLICY_REFRESHER_CHORE_DELAY_KEY =
"hbase.regionserver.quotas.policy.refresher.chore.delay";
- static final long POLICY_REFRESHER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1
minute
+ static final long POLICY_REFRESHER_CHORE_DELAY_DEFAULT = 1000L * 15L; // 15
seconds in millis
static final String POLICY_REFRESHER_CHORE_TIMEUNIT_KEY =
"hbase.regionserver.quotas.policy.refresher.chore.timeunit";
http://git-wip-us.apache.org/repos/asf/hbase/blob/75d0f49d/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java
new file mode 100644
index 000..c1d5258
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java
@@ -0,0 +1,524 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupAdmin;
+import org.apache.hadoop.hbase.backup.BackupInfo;
+import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
+import org.apache.hadoop.hbase.backup.BackupRequest;
+import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
+import org.apache.hadoop.hbase.backup.BackupType;
+import org.apache.hadoop.hbase.backup.HBackupFileSystem;
+import org.apache.hadoop.hbase.backup.RestoreRequest;
+import org.apache.hadoop.hbase.backup.util.BackupSet;
+import org.apache.hadoop.hbase.backup.util.BackupUtils;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+
+import com.google.common.collect.Lists;
+
+@InterfaceAudience.Private
+public class BackupAdminImpl implements BackupAdmin {
+ public final static String CHECK_OK = "Checking backup images: OK";
+ public final static String CHECK_FAILED =
+ "Checking backup images: Failed. Some dependencies are missing for
restore";
+ private static final Log LOG = LogFactory.getLog(BackupAdminImpl.class);
+
+ private final Connection conn;
+
+ public BackupAdminImpl(Connection conn) {
+this.conn = conn;
+ }
+
+ @Override
+ public void close() throws IOException {
+if (conn != null) {
+ conn.close();
+}
+ }
+
+
+ @Override
+ public BackupInfo getBackupInfo(String backupId) throws IOException {
+BackupInfo backupInfo = null;
+try (final BackupSystemTable table = new BackupSystemTable(conn)) {
+ if (backupId == null) {
+ArrayList recentSessions =
table.getBackupInfos(BackupState.RUNNING);
+if (recentSessions.isEmpty()) {
+ LOG.warn("No ongoing sessions found.");
+ return null;
+}
+// else show status for ongoing session
+// must be one maximum
+return recentSessions.get(0);
+ } else {
+backupInfo = table.readBackupInfo(backupId);
+return backupInfo;
+ }
+}
+ }
+
+ @Override
+ public int deleteBackups(String[] backupIds) throws IOException {
+// TODO: requires Fault tolerance support, failure will leave system
+// in a non-consistent state
+// see HBASE-15227
+int totalDeleted = 0;
+Map> allTablesMap = new HashMap>();
+
+try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
+ for (int i = 0; i < backupIds.length; i++) {
+BackupInfo info = sysTable.readBackupInfo(backupIds[i]);
+if (info != null) {
+ String rootDir = info.getBackupRootDir();
+ HashSet allTables = allTablesMap.get(rootDir);
+ if (allTables == null) {
+allTables = new HashSet();
+allTablesMap.put(rootDir, allTables);
+ }
+ allTables.addAll(info.getTableNames());
+ totalDeleted += deleteBackup(backupIds[i], sysTable);
+}
+ }
+ finalizeDelete(allTablesMap, sysTable);
+}
+return totalDeleted;
+ }
+
+ /*
HBASE-17516 Correctly handle case where table and NS quotas both apply
The logic surrounding when a table and namespace quota both apply
to a table was incorrect, leading to a case where a table quota
violation which should have fired did not because of the less-strict
namespace quota.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a44e50f5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a44e50f5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a44e50f5
Branch: refs/heads/HBASE-16961
Commit: a44e50f59f444acd6a87d83c1c3e9b16dc6575f2
Parents: 341cf7e
Author: Josh Elser
Authored: Wed Feb 22 18:32:55 2017 -0500
Committer: Josh Elser
Committed: Mon Mar 20 18:19:31 2017 -0400
--
.../hadoop/hbase/quotas/QuotaObserverChore.java | 10 ++-
.../TestQuotaObserverChoreWithMiniCluster.java | 66
.../hbase/quotas/TestQuotaStatusRPCs.java | 21 ++-
.../hadoop/hbase/quotas/TestSpaceQuotas.java| 32 +-
4 files changed, 97 insertions(+), 32 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/a44e50f5/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index 973ac8c..b9f4592 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -287,7 +287,8 @@ public class QuotaObserverChore extends ScheduledChore {
// We want to have a policy of "NONE", moving out of violation
if (!targetStatus.isInViolation()) {
for (TableName tableInNS : tablesByNamespace.get(namespace)) {
- if
(!tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation())
{
+ // If there is a quota on this table in violation
+ if
(tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation())
{
// Table-level quota violation policy is being applied here.
if (LOG.isTraceEnabled()) {
LOG.trace("Not activating Namespace violation policy because a
Table violation"
@@ -298,16 +299,21 @@ public class QuotaObserverChore extends ScheduledChore {
this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
}
}
+ // We want to move into violation at the NS level
} else {
// Moving tables in the namespace into violation or to a different
violation policy
for (TableName tableInNS : tablesByNamespace.get(namespace)) {
- if
(tableSnapshotStore.getCurrentState(tableInNS).getQuotaStatus().isInViolation())
{
+ final SpaceQuotaSnapshot tableQuotaSnapshot =
+tableSnapshotStore.getCurrentState(tableInNS);
+ final boolean hasTableQuota = QuotaSnapshotStore.NO_QUOTA !=
tableQuotaSnapshot;
+ if (hasTableQuota &&
tableQuotaSnapshot.getQuotaStatus().isInViolation()) {
// Table-level quota violation policy is being applied here.
if (LOG.isTraceEnabled()) {
LOG.trace("Not activating Namespace violation policy because a
Table violation"
+ " policy is already in effect for " + tableInNS);
}
} else {
+// No table quota present or a table quota present that is not in
violation
LOG.info(tableInNS + " moving into violation of namespace space
quota with policy " + targetStatus.getPolicy());
this.snapshotNotifier.transitionTable(tableInNS, targetSnapshot);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a44e50f5/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
index 943c898..63198a8 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -193,40 +193,42 @@ public class TestQuotaObserverChoreWithMiniCluster {
helper.writeData(tn1, 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
admin.flush(tn1);
-Map violatedQuotas =
snapshotNotifier.copySnapshots();
+Map snapshots =
snapshotNotifier.copySnapshots();
http://git-wip-us.apache.org/repos/asf/hbase/blob/a19a95df/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
new file mode 100644
index 000..98236c2
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -0,0 +1,596 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.NamespaceNotFoundException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.quotas.QuotaObserverChore.TablesWithQuotas;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+
+/**
+ * Test class for {@link QuotaObserverChore} that uses a live HBase cluster.
+ */
+@Category(LargeTests.class)
+public class TestQuotaObserverChoreWithMiniCluster {
+ private static final Log LOG =
LogFactory.getLog(TestQuotaObserverChoreWithMiniCluster.class);
+ private static final int SIZE_PER_VALUE = 256;
+ private static final String F1 = "f1";
+ private static final HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
+ private static final AtomicLong COUNTER = new AtomicLong(0);
+ private static final long ONE_MEGABYTE = 1024L * 1024L;
+ private static final long DEFAULT_WAIT_MILLIS = 500;
+
+ @Rule
+ public TestName testName = new TestName();
+
+ private HMaster master;
+ private QuotaObserverChore chore;
+ private SpaceQuotaViolationNotifierForTest violationNotifier;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+Configuration conf = TEST_UTIL.getConfiguration();
+conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY,
1000);
+conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY,
1000);
+conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_DELAY_KEY, 1000);
+conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_PERIOD_KEY, 1000);
+conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
+TEST_UTIL.startMiniCluster(1);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+TEST_UTIL.shutdownMiniCluster();
+ }
+
+ @Before
+ public void removeAllQuotas() throws Exception {
+final Connection conn = TEST_UTIL.getConnection();
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/75d0f49d/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
new file mode 100644
index 000..ec88549
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
@@ -0,0 +1,293 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
+import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
+import org.apache.hadoop.hbase.backup.impl.BackupManager;
+import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Durability;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.wal.WALFactory;
+import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+/**
+ * This class is only a base for other integration-level backup tests. Do not
add tests here.
+ * TestBackupSmallTests is where tests that don't require bring machines
up/down should go All other
+ * tests should have their own classes and extend this one
+ */
+public class TestBackupBase {
+
+ private static final Log LOG = LogFactory.getLog(TestBackupBase.class);
+
+ protected static Configuration conf1;
+ protected static Configuration conf2;
+
+ protected static HBaseTestingUtility TEST_UTIL;
+ protected static HBaseTestingUtility TEST_UTIL2;
+ protected static TableName table1 = TableName.valueOf("table1");
+ protected static HTableDescriptor table1Desc;
+ protected static TableName table2 = TableName.valueOf("table2");
+ protected static TableName table3 = TableName.valueOf("table3");
+ protected static TableName table4 = TableName.valueOf("table4");
+
+ protected static TableName table1_restore =
TableName.valueOf("ns1:table1_restore");
+ protected static TableName table2_restore =
TableName.valueOf("ns2:table2_restore");
+ protected static TableName table3_restore =
TableName.valueOf("ns3:table3_restore");
+ protected static TableName table4_restore =
TableName.valueOf("ns4:table4_restore");
+
+ protected static final int NB_ROWS_IN_BATCH = 99;
+ protected static final byte[] qualName = Bytes.toBytes("q1");
+ protected static final byte[] famName = Bytes.toBytes("f");
+
+ protected static String BACKUP_ROOT_DIR = "/backupUT";
+ protected static String BACKUP_REMOTE_ROOT_DIR = "/backupUT";
+ protected static String provider = "defaultProvider";
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @BeforeClass
+ public static void setUpBeforeClass() throws Exception {
+TEST_UTIL = new HBaseTestingUtility();
+conf1 = TEST_UTIL.getConfiguration();
+conf1.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
+BackupManager.decorateM
http://git-wip-us.apache.org/repos/asf/hbase/blob/75d0f49d/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java
new file mode 100644
index 000..e90d5c1
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileInputFormat.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.mapreduce;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.io.hfile.HFile;
+import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
+import org.apache.hadoop.hbase.io.hfile.HFileScanner;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Simple MR input format for HFiles.
+ * This code was borrowed from Apache Crunch project.
+ * Updated to the recent version of HBase.
+ */
+public class HFileInputFormat extends FileInputFormat {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(HFileInputFormat.class);
+
+ /**
+ * File filter that removes all "hidden" files. This might be something
worth removing from
+ * a more general purpose utility; it accounts for the presence of metadata
files created
+ * in the way we're doing exports.
+ */
+ static final PathFilter HIDDEN_FILE_FILTER = new PathFilter() {
+@Override
+public boolean accept(Path p) {
+ String name = p.getName();
+ return !name.startsWith("_") && !name.startsWith(".");
+}
+ };
+
+ /**
+ * Record reader for HFiles.
+ */
+ private static class HFileRecordReader extends RecordReader {
+
+private Reader in;
+protected Configuration conf;
+private HFileScanner scanner;
+
+/**
+ * A private cache of the key value so it doesn't need to be loaded twice
from the scanner.
+ */
+private Cell value = null;
+private long count;
+private boolean seeked = false;
+
+@Override
+public void initialize(InputSplit split, TaskAttemptContext context)
+throws IOException, InterruptedException {
+ FileSplit fileSplit = (FileSplit) split;
+ conf = context.getConfiguration();
+ Path path = fileSplit.getPath();
+ FileSystem fs = path.getFileSystem(conf);
+ LOG.info("Initialize HFileRecordReader for {}", path);
+ this.in = HFile.createReader(fs, path, conf);
+
+ // The file info must be loaded before the scanner can be used.
+ // This seems like a bug in HBase, but it's easily worked around.
+ this.in.loadFileInfo();
+ this.scanner = in.getScanner(false, false);
+
+}
+
+
+@Override
+public boolean nextKeyValue() throws IOException, InterruptedException {
+ boolean hasNext;
+ if (!seeked) {
+LOG.info("Seeking to start");
+hasNext = scanner.seekTo();
+seeked = true;
+ } else {
+hasNext = scanner.next();
+ }
+ if (!hasNext) {
+return false;
+ }
+ value = scanner.getCell();
+ count++;
+ return true;
+}
+
+@Override
+public NullWritable getCurrentKey() throws IOException,
InterruptedException {
+ return NullWritable.get();
+}
+
+@Override
+public Cell getCurrentValue() throws IOException, InterruptedException {
+ return value;
+}
+
+@Override
+public float getProgress() throws IOException, InterruptedException {
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/c544ed1f/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
new file mode 100644
index 000..4a7000c
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestTableSpaceQuotaViolationNotifier.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.Objects;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.mockito.ArgumentMatcher;
+
+/**
+ * Test case for {@link TableSpaceQuotaViolationNotifier}.
+ */
+@Category(SmallTests.class)
+public class TestTableSpaceQuotaViolationNotifier {
+
+ private TableSpaceQuotaViolationNotifier notifier;
+ private Connection conn;
+
+ @Before
+ public void setup() throws Exception {
+notifier = new TableSpaceQuotaViolationNotifier();
+conn = mock(Connection.class);
+notifier.initialize(conn);
+ }
+
+ @Test
+ public void testToViolation() throws Exception {
+final TableName tn = TableName.valueOf("inviolation");
+final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
+final Table quotaTable = mock(Table.class);
+
when(conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable);
+
+final Put expectedPut = new Put(Bytes.toBytes("t." +
tn.getNameAsString()));
+final SpaceQuota protoQuota = SpaceQuota.newBuilder()
+.setViolationPolicy(ProtobufUtil.toProtoViolationPolicy(policy))
+.build();
+expectedPut.addColumn(Bytes.toBytes("u"), Bytes.toBytes("v"),
protoQuota.toByteArray());
+
+notifier.transitionTableToViolation(tn, policy);
+
+verify(quotaTable).put(argThat(new SingleCellPutMatcher(expectedPut)));
+ }
+
+ @Test
+ public void testToObservance() throws Exception {
+final TableName tn = TableName.valueOf("notinviolation");
+final Table quotaTable = mock(Table.class);
+
when(conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)).thenReturn(quotaTable);
+
+final Delete expectedDelete = new Delete(Bytes.toBytes("t." +
tn.getNameAsString()));
+expectedDelete.addColumn(Bytes.toBytes("u"), Bytes.toBytes("v"));
+
+notifier.transitionTableToObservance(tn);
+
+verify(quotaTable).delete(argThat(new
SingleCellDeleteMatcher(expectedDelete)));
+ }
+
+ /**
+ * Parameterized for Puts.
+ */
+ private static class SingleCellPutMatcher extends
SingleCellMutationMatcher {
+private SingleCellPutMatcher(Put expected) {
+ super(expected);
+}
+ }
+
+ /**
+ * Parameterized for Deletes.
+ */
+ private static class SingleCellDeleteMatcher extends
SingleCellMutationMatcher {
+private SingleCellDeleteMatcher(Delete expected) {
+ super(expected);
+}
+ }
+
+ /**
+ * Quick hack to verify a Mutation with one column.
+ */
+ private static class SingleCellMutationMatcher extends ArgumentMatcher
{
+private final Mutation expected;
+
+private SingleCellMutationMatcher(Mutation expected) {
+ this.
HBASE-17806 TestRSGroups#testMoveServersAndTables is flaky in master branch
(Guangxu Cheng)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4088f822
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4088f822
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4088f822
Branch: refs/heads/HBASE-16961
Commit: 4088f822a449acc39c2408a287f820ec26acabf4
Parents: 5b4bb82
Author: tedyu
Authored: Mon Mar 20 09:26:34 2017 -0700
Committer: tedyu
Committed: Mon Mar 20 09:26:34 2017 -0700
--
.../org/apache/hadoop/hbase/rsgroup/RSGroupInfo.java | 3 +++
.../apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java| 11 +--
2 files changed, 12 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/4088f822/hbase-common/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfo.java
--
diff --git
a/hbase-common/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfo.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfo.java
index 35563c5..9219c23 100644
---
a/hbase-common/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfo.java
+++
b/hbase-common/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfo.java
@@ -132,6 +132,9 @@ public class RSGroupInfo {
sb.append(", ");
sb.append(" Servers:");
sb.append(this.servers);
+sb.append(", ");
+sb.append(" Tables:");
+sb.append(this.tables);
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/4088f822/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
--
diff --git
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
index e8cdb78..e5c89c3 100644
---
a/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
+++
b/hbase-rsgroup/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java
@@ -694,6 +694,7 @@ public abstract class TestRSGroupsBase {
@Test
public void testMoveServersAndTables() throws Exception {
+LOG.info("testMoveServersAndTables");
final RSGroupInfo newGroup = addGroup(getGroupName(name.getMethodName()),
1);
//create table
final byte[] familyNameBytes = Bytes.toBytes("f");
@@ -718,6 +719,12 @@ public abstract class TestRSGroupsBase {
}
}
+LOG.debug("Print group info : " + rsGroupAdmin.listRSGroups());
+int oldDefaultGroupServerSize =
+
rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size();
+int oldDefaultGroupTableSize =
+
rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size();
+
//test fail bogus server move
try {
rsGroupAdmin.moveServersAndTables(Sets.newHashSet(Address.fromString("foo:")),
@@ -742,9 +749,9 @@ public abstract class TestRSGroupsBase {
}
//verify default group info
-Assert.assertEquals(3,
+Assert.assertEquals(oldDefaultGroupServerSize,
rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getServers().size());
-Assert.assertEquals(4,
+Assert.assertEquals(oldDefaultGroupTableSize,
rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP).getTables().size());
//verify new group info
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e042231/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
--
diff --git a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
index 1c373ee..23ddd43 100644
--- a/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/RegionServerStatus.proto
@@ -141,6 +141,22 @@ message SplitTableRegionResponse {
optional uint64 proc_id = 1;
}
+message RegionSpaceUse {
+ optional RegionInfo region = 1; // A region identifier
+ optional uint64 size = 2; // The size in bytes of the region
+}
+
+/**
+ * Reports filesystem usage for regions.
+ */
+message RegionSpaceUseReportRequest {
+ repeated RegionSpaceUse space_use = 1;
+}
+
+message RegionSpaceUseReportResponse {
+
+}
+
service RegionServerStatusService {
/** Called when a region server first starts. */
rpc RegionServerStartup(RegionServerStartupRequest)
@@ -182,4 +198,10 @@ service RegionServerStatusService {
*/
rpc getProcedureResult(GetProcedureResultRequest)
returns(GetProcedureResultResponse);
+
+ /**
+ * Reports Region filesystem space use
+ */
+ rpc ReportRegionSpaceUse(RegionSpaceUseReportRequest)
+returns(RegionSpaceUseReportResponse);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e042231/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index f86f800..f10d4bb 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureUtil;
+import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
import org.apache.hadoop.hbase.regionserver.RSRpcServices;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
@@ -216,6 +217,9 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProto
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStartupResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionServerStatusService;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUse;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionSpaceUseReportResponse;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRSFatalErrorResponse;
@@ -2006,4 +2010,19 @@ public class MasterRpcServices extends RSRpcServices
throw new ServiceException(e);
}
}
+
+ @Override
+ public RegionSpaceUseReportResponse reportRegionSpaceUse(RpcController
controller,
+ RegionSpaceUseReportRequest request) throws ServiceException {
+try {
+ master.checkInitialized();
+ MasterQuotaManager quotaManager = this.master.getMasterQuotaManager();
+ for (RegionSpaceUse report : request.getSpaceUseList()) {
+quotaManager.addRegionSize(HRegionInfo.convert(report.getRegion()),
report.getSize());
+ }
+ return RegionSpaceUseReportResponse.newBuilder().build();
+} catch (Exception e) {
+ throw new ServiceException(e);
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/6e042231/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
new file mode 100644
index 000..01540eb
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/FileSystemUtilizationChore.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or mor
HBASE-16996 Implement storage/retrieval of filesystem-use quotas into quota
table (Josh Elser)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/72715494
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/72715494
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/72715494
Branch: refs/heads/HBASE-16961
Commit: 727154949e47d3411f76bc967b0ed687fe672134
Parents: 9684493
Author: tedyu
Authored: Sat Dec 3 14:30:48 2016 -0800
Committer: Josh Elser
Committed: Mon Mar 20 17:39:21 2017 -0400
--
.../hadoop/hbase/quotas/QuotaTableUtil.java | 13 +-
.../hadoop/hbase/quotas/MasterQuotaManager.java | 30 +
.../hadoop/hbase/quotas/TestQuotaAdmin.java | 125 ++-
3 files changed, 165 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/72715494/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
index 116dd0c..1640ddc 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaTableUtil.java
@@ -52,7 +52,9 @@ import org.apache.hadoop.hbase.util.Strings;
*
* ROW-KEY FAM/QUALDATA
* n. q:s
+ * n. u:du
* t. q:s
+ * t. u:du
* u. q:s
* u. q:s.
* u. q:s.:
@@ -71,6 +73,7 @@ public class QuotaTableUtil {
protected static final byte[] QUOTA_FAMILY_USAGE = Bytes.toBytes("u");
protected static final byte[] QUOTA_QUALIFIER_SETTINGS = Bytes.toBytes("s");
protected static final byte[] QUOTA_QUALIFIER_SETTINGS_PREFIX =
Bytes.toBytes("s.");
+ protected static final byte[] QUOTA_QUALIFIER_DISKUSAGE =
Bytes.toBytes("du");
protected static final byte[] QUOTA_USER_ROW_KEY_PREFIX =
Bytes.toBytes("u.");
protected static final byte[] QUOTA_TABLE_ROW_KEY_PREFIX =
Bytes.toBytes("t.");
protected static final byte[] QUOTA_NAMESPACE_ROW_KEY_PREFIX =
Bytes.toBytes("n.");
@@ -298,11 +301,16 @@ public class QuotaTableUtil {
* Quotas protobuf helpers
*/
protected static Quotas quotasFromData(final byte[] data) throws IOException
{
+return quotasFromData(data, 0, data.length);
+ }
+
+ protected static Quotas quotasFromData(
+ final byte[] data, int offset, int length) throws IOException {
int magicLen = ProtobufMagic.lengthOfPBMagic();
-if (!ProtobufMagic.isPBMagicPrefix(data, 0, magicLen)) {
+if (!ProtobufMagic.isPBMagicPrefix(data, offset, magicLen)) {
throw new IOException("Missing pb magic prefix");
}
-return Quotas.parseFrom(new ByteArrayInputStream(data, magicLen,
data.length - magicLen));
+return Quotas.parseFrom(new ByteArrayInputStream(data, offset + magicLen,
length - magicLen));
}
protected static byte[] quotasToData(final Quotas data) throws IOException {
@@ -316,6 +324,7 @@ public class QuotaTableUtil {
boolean hasSettings = false;
hasSettings |= quotas.hasThrottle();
hasSettings |= quotas.hasBypassGlobals();
+hasSettings |= quotas.hasSpace();
return !hasSettings;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/72715494/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
index 5dab2e3..1469268 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotaManager.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRequest;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaResponse;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceLimitRequest;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Throttle;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.Quota
HBASE-16995 Build client Java API and client protobuf messages (Josh Elser)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/87fa04cc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/87fa04cc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/87fa04cc
Branch: refs/heads/HBASE-16961
Commit: 87fa04cce2e2701e1a2d425ea89677d97386377c
Parents: 16900c8
Author: tedyu
Authored: Thu Nov 17 10:19:52 2016 -0800
Committer: Josh Elser
Committed: Mon Mar 20 17:39:17 2017 -0400
--
.../hbase/quotas/QuotaSettingsFactory.java | 47 +
.../apache/hadoop/hbase/quotas/QuotaType.java |1 +
.../hadoop/hbase/quotas/SpaceLimitSettings.java | 166 ++
.../hbase/quotas/SpaceViolationPolicy.java | 44 +
.../hbase/shaded/protobuf/ProtobufUtil.java | 51 +
.../hbase/quotas/TestQuotaSettingsFactory.java | 148 ++
.../hbase/quotas/TestSpaceLimitSettings.java| 119 ++
.../shaded/protobuf/generated/MasterProtos.java | 584 --
.../shaded/protobuf/generated/QuotaProtos.java | 1739 +-
.../src/main/protobuf/Master.proto |2 +
.../src/main/protobuf/Quota.proto | 21 +
.../hbase/protobuf/generated/QuotaProtos.java | 1682 -
hbase-protocol/src/main/protobuf/Quota.proto| 21 +
13 files changed, 4291 insertions(+), 334 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hbase/blob/87fa04cc/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
--
diff --git
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
index 1a8b934..a91163f 100644
---
a/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
+++
b/hbase-client/src/main/java/org/apache/hadoop/hbase/quotas/QuotaSettingsFactory.java
@@ -28,6 +28,7 @@ import
org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos.SetQuotaRe
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
@InterfaceAudience.Public
@InterfaceStability.Evolving
@@ -91,6 +92,9 @@ public class QuotaSettingsFactory {
if (quotas.getBypassGlobals() == true) {
settings.add(new QuotaGlobalsSettingsBypass(userName, tableName,
namespace, true));
}
+if (quotas.hasSpace()) {
+ settings.add(fromSpace(tableName, namespace, quotas.getSpace()));
+}
return settings;
}
@@ -124,6 +128,18 @@ public class QuotaSettingsFactory {
return settings;
}
+ static QuotaSettings fromSpace(TableName table, String namespace, SpaceQuota
protoQuota) {
+if ((null == table && null == namespace) || (null != table && null !=
namespace)) {
+ throw new IllegalArgumentException("Can only construct
SpaceLimitSettings for a table or namespace.");
+}
+if (null != table) {
+ return SpaceLimitSettings.fromSpaceQuota(table, protoQuota);
+} else {
+ // namespace must be non-null
+ return SpaceLimitSettings.fromSpaceQuota(namespace, protoQuota);
+}
+ }
+
/* ==
* RPC Throttle
*/
@@ -280,4 +296,35 @@ public class QuotaSettingsFactory {
public static QuotaSettings bypassGlobals(final String userName, final
boolean bypassGlobals) {
return new QuotaGlobalsSettingsBypass(userName, null, null, bypassGlobals);
}
+
+ /* ==
+ * FileSystem Space Settings
+ */
+
+ /**
+ * Creates a {@link QuotaSettings} object to limit the FileSystem space
usage for the given table to the given size in bytes.
+ * When the space usage is exceeded by the table, the provided {@link
SpaceViolationPolicy} is enacted on the table.
+ *
+ * @param tableName The name of the table on which the quota should be
applied.
+ * @param sizeLimit The limit of a table's size in bytes.
+ * @param violationPolicy The action to take when the quota is exceeded.
+ * @return An {@link QuotaSettings} object.
+ */
+ public static QuotaSettings limitTableSpace(final TableName tableName, long
sizeLimit, final SpaceViolationPolicy violationPolicy) {
+return new SpaceLimitSettings(tableName, sizeLimit, violationPolicy);
+ }
+
+ /**
+ * Creates a {@link QuotaSettings} object to limit the FileSystem space
usage for the given namespace to the given size in bytes.
+ * When the space usage is exceeded by all tabl
HBASE-14123 HBase Backup/Restore Phase 2 (Vladimir Rodionov)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/75d0f49d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/75d0f49d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/75d0f49d
Branch: refs/heads/HBASE-16961
Commit: 75d0f49dcd9761d32a8dedfaa169844822a9e7a5
Parents: 8e5eeb4
Author: tedyu
Authored: Sat Mar 18 03:04:19 2017 -0700
Committer: tedyu
Committed: Sat Mar 18 03:04:19 2017 -0700
--
bin/hbase |6 +
.../apache/hadoop/hbase/client/HBaseAdmin.java |4 +-
.../hbase/client/RpcRetryingCallerImpl.java |3 +-
.../apache/hadoop/hbase/backup/BackupType.java | 25 +
.../hadoop/hbase/util/AbstractHBaseTool.java| 18 +-
.../hbase/IntegrationTestBackupRestore.java | 311 +
.../shaded/protobuf/generated/BackupProtos.java | 7013 ++
.../shaded/protobuf/generated/MasterProtos.java | 18 +-
.../src/main/protobuf/Backup.proto | 117 +
hbase-server/pom.xml| 10 +
.../apache/hadoop/hbase/backup/BackupAdmin.java | 128 +
.../hadoop/hbase/backup/BackupCopyJob.java | 55 +
.../hadoop/hbase/backup/BackupDriver.java | 204 +
.../apache/hadoop/hbase/backup/BackupInfo.java | 545 ++
.../hadoop/hbase/backup/BackupRequest.java | 139 +
.../hbase/backup/BackupRestoreConstants.java| 115 +
.../hbase/backup/BackupRestoreFactory.java | 66 +
.../hadoop/hbase/backup/BackupTableInfo.java| 82 +
.../hadoop/hbase/backup/HBackupFileSystem.java | 141 +
.../apache/hadoop/hbase/backup/LogUtils.java| 50 +
.../hadoop/hbase/backup/RestoreDriver.java | 265 +
.../apache/hadoop/hbase/backup/RestoreJob.java | 46 +
.../hadoop/hbase/backup/RestoreRequest.java | 135 +
.../hbase/backup/impl/BackupAdminImpl.java | 524 ++
.../hbase/backup/impl/BackupCommands.java | 780 ++
.../hbase/backup/impl/BackupException.java | 84 +
.../hadoop/hbase/backup/impl/BackupManager.java | 472 ++
.../hbase/backup/impl/BackupManifest.java | 666 ++
.../hbase/backup/impl/BackupSystemTable.java| 1376
.../backup/impl/FullTableBackupClient.java | 189 +
.../backup/impl/IncrementalBackupManager.java | 344 +
.../impl/IncrementalTableBackupClient.java | 216 +
.../hbase/backup/impl/RestoreTablesClient.java | 237 +
.../hbase/backup/impl/TableBackupClient.java| 387 +
.../backup/mapreduce/HFileSplitterJob.java | 181 +
.../mapreduce/MapReduceBackupCopyJob.java | 344 +
.../backup/mapreduce/MapReduceRestoreJob.java | 182 +
.../hbase/backup/master/BackupLogCleaner.java | 142 +
.../master/LogRollMasterProcedureManager.java | 155 +
.../regionserver/LogRollBackupSubprocedure.java | 168 +
.../LogRollBackupSubprocedurePool.java | 139 +
.../LogRollRegionServerProcedureManager.java| 185 +
.../hadoop/hbase/backup/util/BackupSet.java | 58 +
.../hadoop/hbase/backup/util/BackupUtils.java | 702 ++
.../hadoop/hbase/backup/util/RestoreTool.java | 610 ++
.../BaseCoordinatedStateManager.java| 20 +-
.../coordination/ZkCoordinatedStateManager.java | 23 +-
.../hbase/mapreduce/HFileInputFormat.java | 174 +
.../hbase/mapreduce/LoadIncrementalHFiles.java | 25 +-
.../hadoop/hbase/mapreduce/WALInputFormat.java | 42 +-
.../hadoop/hbase/mapreduce/WALPlayer.java | 83 +-
.../hadoop/hbase/master/MasterRpcServices.java | 129 +-
.../hbase/master/snapshot/SnapshotManager.java |4 +-
.../hbase/procedure/ZKProcedureCoordinator.java | 328 +
.../procedure/ZKProcedureCoordinatorRpcs.java | 327 -
.../flush/MasterFlushTableProcedureManager.java |4 +-
.../hbase/regionserver/HRegionServer.java | 17 +-
.../hadoop/hbase/wal/AbstractFSWALProvider.java |5 +
.../hadoop/hbase/HBaseTestingUtility.java | 41 +-
.../hadoop/hbase/backup/TestBackupBase.java | 293 +
.../hbase/backup/TestBackupBoundaryTests.java | 97 +
.../hbase/backup/TestBackupCommandLineTool.java | 431 ++
.../hadoop/hbase/backup/TestBackupDelete.java | 102 +
.../hbase/backup/TestBackupDeleteRestore.java | 70 +
.../hadoop/hbase/backup/TestBackupDescribe.java | 110 +
.../hbase/backup/TestBackupMultipleDeletes.java | 159 +
.../hbase/backup/TestBackupShowHistory.java | 148 +
.../hbase/backup/TestBackupStatusProgress.java | 96 +
.../hbase/backup/TestBackupSystemTable.java | 511 ++
.../hadoop/hbase/backup/TestFullBackup.java | 59 +
.../hadoop/hbase/backup/TestFullBackupSet.java | 103 +
.../backup/TestFullBackupSetRestoreSet.java | 128 +
.../hadoop/hbase/backup/TestFullRestore.java| 345 +
.../hbase/backup/TestIncrementalBackup.java | 200 +
.../TestIncrementalBackupDeleteTable.java | 129 +
http://git-wip-us.apache.org/repos/asf/hbase/blob/b99e9cf9/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
--
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
index c493b25..943c898 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaObserverChoreWithMiniCluster.java
@@ -22,16 +22,12 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import java.io.IOException;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
@@ -40,20 +36,15 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.NamespaceDescriptor;
import org.apache.hadoop.hbase.NamespaceNotFoundException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.quotas.QuotaObserverChore.TablesWithQuotas;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -62,7 +53,6 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
-import com.google.common.collect.HashMultimap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
@@ -72,11 +62,8 @@ import com.google.common.collect.Multimap;
@Category(LargeTests.class)
public class TestQuotaObserverChoreWithMiniCluster {
private static final Log LOG =
LogFactory.getLog(TestQuotaObserverChoreWithMiniCluster.class);
- private static final int SIZE_PER_VALUE = 256;
- private static final String F1 = "f1";
private static final HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
private static final AtomicLong COUNTER = new AtomicLong(0);
- private static final long ONE_MEGABYTE = 1024L * 1024L;
private static final long DEFAULT_WAIT_MILLIS = 500;
@Rule
@@ -84,18 +71,19 @@ public class TestQuotaObserverChoreWithMiniCluster {
private HMaster master;
private QuotaObserverChore chore;
- private SpaceQuotaViolationNotifierForTest violationNotifier;
+ private SpaceQuotaSnapshotNotifierForTest snapshotNotifier;
+ private SpaceQuotaHelperForTests helper;
@BeforeClass
public static void setUp() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY,
1000);
conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY,
1000);
-conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_DELAY_KEY, 1000);
-conf.setInt(QuotaObserverChore.VIOLATION_OBSERVER_CHORE_PERIOD_KEY, 1000);
+conf.setInt(QuotaObserverChore.QUOTA_OBSERVER_CHORE_DELAY_KEY, 1000);
+conf.setInt(QuotaObserverChore.QUOTA_OBSERVER_CHORE_PERIOD_KEY, 1000);
conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
-conf.setClass(SpaceQuotaViolationNotifierFactory.VIOLATION_NOTIFIER_KEY,
-SpaceQuotaViolationNotifierForTest.class,
SpaceQuotaViolationNotifier.class);
+conf.setClass(SpaceQuotaSnapshotNotifierFactory.SNAPSHOT_NOTIFIER_KEY,
+SpaceQuotaSnapshotNotifierForTest.class,
SpaceQuotaSnapshotNotifier.class);
TEST_UTIL.startMiniCluster(1);
}
@@ -131,40 +119,55 @@ public class TestQuotaObserverChoreWithMiniCluster {
}
master = TEST_UTIL.getMiniHBaseCluster().getMaster();
-violationNotifier =
-(SpaceQuotaViolationNotifierForTest)
master.getSpaceQuotaViolationNotifier();
-violationNotifier.clearTableViolations();
+snapshotNotifier =
+(SpaceQuotaSnapshotNotifierForTest)
master.getSpaceQuotaSnapshotNotifier();
+snapshotNotifier.clearSnapshots();
chore = master.getQuotaObserverChore();
+helpe
http://git-wip-us.apache.org/repos/asf/hbase/blob/b99e9cf9/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
--
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
index 8b127d9..973ac8c 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaObserverChore.java
@@ -37,9 +37,8 @@ import
org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.master.HMaster;
-import org.apache.hadoop.hbase.quotas.QuotaViolationStore.ViolationState;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.Quotas;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
+import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot.SpaceQuotaStatus;
import
org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos.SpaceQuota;
import com.google.common.annotations.VisibleForTesting;
@@ -54,51 +53,51 @@ import com.google.common.collect.Multimap;
@InterfaceAudience.Private
public class QuotaObserverChore extends ScheduledChore {
private static final Log LOG = LogFactory.getLog(QuotaObserverChore.class);
- static final String VIOLATION_OBSERVER_CHORE_PERIOD_KEY =
- "hbase.master.quotas.violation.observer.chore.period";
- static final int VIOLATION_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; //
5 minutes in millis
+ static final String QUOTA_OBSERVER_CHORE_PERIOD_KEY =
+ "hbase.master.quotas.observer.chore.period";
+ static final int QUOTA_OBSERVER_CHORE_PERIOD_DEFAULT = 1000 * 60 * 5; // 5
minutes in millis
- static final String VIOLATION_OBSERVER_CHORE_DELAY_KEY =
- "hbase.master.quotas.violation.observer.chore.delay";
- static final long VIOLATION_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1
minute
+ static final String QUOTA_OBSERVER_CHORE_DELAY_KEY =
+ "hbase.master.quotas.observer.chore.delay";
+ static final long QUOTA_OBSERVER_CHORE_DELAY_DEFAULT = 1000L * 60L; // 1
minute
- static final String VIOLATION_OBSERVER_CHORE_TIMEUNIT_KEY =
- "hbase.master.quotas.violation.observer.chore.timeunit";
- static final String VIOLATION_OBSERVER_CHORE_TIMEUNIT_DEFAULT =
TimeUnit.MILLISECONDS.name();
+ static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_KEY =
+ "hbase.master.quotas.observer.chore.timeunit";
+ static final String QUOTA_OBSERVER_CHORE_TIMEUNIT_DEFAULT =
TimeUnit.MILLISECONDS.name();
- static final String VIOLATION_OBSERVER_CHORE_REPORT_PERCENT_KEY =
- "hbase.master.quotas.violation.observer.report.percent";
- static final double VIOLATION_OBSERVER_CHORE_REPORT_PERCENT_DEFAULT= 0.95;
+ static final String QUOTA_OBSERVER_CHORE_REPORT_PERCENT_KEY =
+ "hbase.master.quotas.observer.report.percent";
+ static final double QUOTA_OBSERVER_CHORE_REPORT_PERCENT_DEFAULT= 0.95;
private final Connection conn;
private final Configuration conf;
private final MasterQuotaManager quotaManager;
/*
- * Callback that changes in quota violation are passed to.
+ * Callback that changes in quota snapshots are passed to.
*/
- private final SpaceQuotaViolationNotifier violationNotifier;
+ private final SpaceQuotaSnapshotNotifier snapshotNotifier;
/*
- * Preserves the state of quota violations for tables and namespaces
+ * Preserves the state of quota snapshots for tables and namespaces
*/
- private final Map tableQuotaViolationStates;
- private final Map namespaceQuotaViolationStates;
+ private final Map tableQuotaSnapshots;
+ private final Map namespaceQuotaSnapshots;
/*
- * Encapsulates logic for moving tables/namespaces into or out of quota
violation
+ * Encapsulates logic for tracking the state of a table/namespace WRT space
quotas
*/
- private QuotaViolationStore tableViolationStore;
- private QuotaViolationStore namespaceViolationStore;
+ private QuotaSnapshotStore tableSnapshotStore;
+ private QuotaSnapshotStore namespaceSnapshotStore;
public QuotaObserverChore(HMaster master) {
this(
master.getConnection(), master.getConfiguration(),
-master.getSpaceQuotaViolationNotifier(),
master.getMasterQuotaManager(),
+master.getSpaceQuotaSnapshotNotifier(), master.getMasterQuotaManager(),
master);
}
QuotaObserverChore(
- Connection conn, Configuration conf, SpaceQuotaViolationNotifier
violationNotifier,
+ Connection conn, Configuration conf, SpaceQuotaSnapshotNotifier
snapshotNotifier,
MasterQuotaManager quotaManager, Stoppable stopper) {
super(
QuotaObserverChore.class.getSimpleName(), stopper
1001 - 1100 of 1226 matches
Mail list logo
