(hbase) branch branch-2.5 updated: HBASE-28390 WAL value compression fails for cells with large values (#5696)
This is an automated email from the ASF dual-hosted git repository.
bbeaudreault pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.5 by this push:
new 21d77266463 HBASE-28390 WAL value compression fails for cells with
large values (#5696)
21d77266463 is described below
commit 21d772664634fcc86e85c7656882ebca39b1a0fd
Author: Bryan Beaudreault
AuthorDate: Fri Feb 23 12:29:45 2024 -0500
HBASE-28390 WAL value compression fails for cells with large values (#5696)
Signed-off-by: Andrew Purtell
---
.../aircompressor/TestWALCompressionLz4.java | 13 -
.../aircompressor/TestWALCompressionLzo.java | 14 --
.../aircompressor/TestWALCompressionSnappy.java| 13 -
.../aircompressor/TestWALCompressionZstd.java | 13 -
.../compress/brotli/TestWALCompressionBrotli.java | 13 -
.../io/compress/lz4/TestWALCompressionLz4.java | 14 --
.../compress/xerial/TestWALCompressionSnappy.java | 13 -
.../io/compress/xz/TestWALCompressionLzma.java | 13 -
.../io/compress/zstd/TestWALCompressionZstd.java | 14 --
.../hbase/regionserver/wal/CompressionContext.java | 17
...LDecompressionBoundedDelegatingInputStream.java | 16 ++-
.../hadoop/hbase/wal/CompressedWALTestBase.java| 32 --
.../apache/hadoop/hbase/wal/TestCompressedWAL.java | 13 -
.../wal/TestCompressedWALValueCompression.java | 13 -
14 files changed, 55 insertions(+), 156 deletions(-)
diff --git
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
index 34a7dcfedfc..1361754189a 100644
---
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
+++
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLz4 extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionLz4 extends
CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLz4.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -60,10 +53,4 @@ public class TestWALCompressionLz4 extends
CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
-TableName tableName =
TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
-doTest(tableName);
- }
-
}
diff --git
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
index 9c5bc8838c0..dd91b0cac02 100644
---
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
+++
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hba
(hbase) 02/02: HBASE-28390 WAL value compression fails for cells with large values (#5696)
This is an automated email from the ASF dual-hosted git repository.
bbeaudreault pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 751b75e82fdfa9fd52993ccddb067cf051999ca6
Author: Bryan Beaudreault
AuthorDate: Fri Feb 23 12:29:45 2024 -0500
HBASE-28390 WAL value compression fails for cells with large values (#5696)
Signed-off-by: Andrew Purtell
---
.../aircompressor/TestWALCompressionLz4.java | 13 -
.../aircompressor/TestWALCompressionLzo.java | 14 --
.../aircompressor/TestWALCompressionSnappy.java| 13 -
.../aircompressor/TestWALCompressionZstd.java | 13 -
.../compress/brotli/TestWALCompressionBrotli.java | 13 -
.../io/compress/lz4/TestWALCompressionLz4.java | 14 --
.../compress/xerial/TestWALCompressionSnappy.java | 13 -
.../io/compress/xz/TestWALCompressionLzma.java | 13 -
.../io/compress/zstd/TestWALCompressionZstd.java | 14 --
.../hbase/regionserver/wal/CompressionContext.java | 17
...LDecompressionBoundedDelegatingInputStream.java | 16 ++-
.../hadoop/hbase/wal/CompressedWALTestBase.java| 32 --
.../apache/hadoop/hbase/wal/TestCompressedWAL.java | 13 -
.../wal/TestCompressedWALValueCompression.java | 13 -
14 files changed, 55 insertions(+), 156 deletions(-)
diff --git
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
index 34a7dcfedfc..1361754189a 100644
---
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
+++
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLz4 extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionLz4 extends
CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLz4.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -60,10 +53,4 @@ public class TestWALCompressionLz4 extends
CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
-TableName tableName =
TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
-doTest(tableName);
- }
-
}
diff --git
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
index 9c5bc8838c0..dd91b0cac02 100644
---
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
+++
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.catego
(hbase) 01/02: HBASE-28370 Default user quotas are refreshing too frequently (#5686)
This is an automated email from the ASF dual-hosted git repository.
bbeaudreault pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit aae26cbcfd6fe3b4e44367d7b76cd30d29c11a4d
Author: Ray Mattingly
AuthorDate: Mon Feb 19 15:32:00 2024 -0500
HBASE-28370 Default user quotas are refreshing too frequently (#5686)
Signed-off-by: Bryan Beaudreault
---
.../org/apache/hadoop/hbase/quotas/QuotaCache.java | 12 ++-
.../org/apache/hadoop/hbase/quotas/QuotaUtil.java | 6 +-
.../apache/hadoop/hbase/quotas/TestQuotaCache.java | 89 ++
.../hadoop/hbase/quotas/ThrottleQuotaTestUtil.java | 12 +++
4 files changed, 115 insertions(+), 4 deletions(-)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java
index 67b2aecc544..9b3498ff894 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaCache.java
@@ -71,6 +71,8 @@ public class QuotaCache implements Stoppable {
// for testing purpose only, enforce the cache to be always refreshed
static boolean TEST_FORCE_REFRESH = false;
+ // for testing purpose only, block cache refreshes to reliably verify state
+ static boolean TEST_BLOCK_REFRESH = false;
private final ConcurrentMap namespaceQuotaCache = new
ConcurrentHashMap<>();
private final ConcurrentMap tableQuotaCache = new
ConcurrentHashMap<>();
@@ -138,7 +140,7 @@ public class QuotaCache implements Stoppable {
*/
public UserQuotaState getUserQuotaState(final UserGroupInformation ugi) {
return computeIfAbsent(userQuotaCache, getQuotaUserName(ugi),
- () ->
QuotaUtil.buildDefaultUserQuotaState(rsServices.getConfiguration()),
+ () ->
QuotaUtil.buildDefaultUserQuotaState(rsServices.getConfiguration(), 0L),
this::triggerCacheRefresh);
}
@@ -239,6 +241,14 @@ public class QuotaCache implements Stoppable {
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value =
"GC_UNRELATED_TYPES",
justification = "I do not understand why the complaints, it looks good
to me -- FIX")
protected void chore() {
+ while (TEST_BLOCK_REFRESH) {
+LOG.info("TEST_BLOCK_REFRESH=true, so blocking QuotaCache refresh
until it is false");
+try {
+ Thread.sleep(10);
+} catch (InterruptedException e) {
+ throw new RuntimeException(e);
+}
+ }
// Prefetch online tables/namespaces
for (TableName table : ((HRegionServer)
QuotaCache.this.rsServices).getOnlineTables()) {
if (table.isSystemTable()) {
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java
index 831c0297785..8ced76e3963 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java
@@ -333,7 +333,7 @@ public class QuotaUtil extends QuotaTableUtil {
String user = getUserFromRowKey(key);
if (results[i].isEmpty()) {
-userQuotas.put(user,
buildDefaultUserQuotaState(connection.getConfiguration()));
+userQuotas.put(user,
buildDefaultUserQuotaState(connection.getConfiguration(), nowTs));
continue;
}
@@ -373,7 +373,7 @@ public class QuotaUtil extends QuotaTableUtil {
return userQuotas;
}
- protected static UserQuotaState buildDefaultUserQuotaState(Configuration
conf) {
+ protected static UserQuotaState buildDefaultUserQuotaState(Configuration
conf, long nowTs) {
QuotaProtos.Throttle.Builder throttleBuilder =
QuotaProtos.Throttle.newBuilder();
buildDefaultTimedQuota(conf, QUOTA_DEFAULT_USER_MACHINE_READ_NUM)
@@ -389,7 +389,7 @@ public class QuotaUtil extends QuotaTableUtil {
buildDefaultTimedQuota(conf, QUOTA_DEFAULT_USER_MACHINE_WRITE_SIZE)
.ifPresent(throttleBuilder::setWriteSize);
-UserQuotaState state = new UserQuotaState();
+UserQuotaState state = new UserQuotaState(nowTs);
QuotaProtos.Quotas defaultQuotas =
QuotaProtos.Quotas.newBuilder().setThrottle(throttleBuilder.build()).build();
state.setQuotas(defaultQuotas);
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java
new file mode 100644
index 000..1c431858291
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License,
(hbase) branch branch-2.6 updated (ea1c057e0b1 -> 751b75e82fd)
This is an automated email from the ASF dual-hosted git repository. bbeaudreault pushed a change to branch branch-2.6 in repository https://gitbox.apache.org/repos/asf/hbase.git from ea1c057e0b1 HBASE-28377 Fallback to simple is broken for blocking rpc client (#5690) new aae26cbcfd6 HBASE-28370 Default user quotas are refreshing too frequently (#5686) new 751b75e82fd HBASE-28390 WAL value compression fails for cells with large values (#5696) The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../aircompressor/TestWALCompressionLz4.java | 13 .../aircompressor/TestWALCompressionLzo.java | 14 .../aircompressor/TestWALCompressionSnappy.java| 13 .../aircompressor/TestWALCompressionZstd.java | 13 .../compress/brotli/TestWALCompressionBrotli.java | 13 .../io/compress/lz4/TestWALCompressionLz4.java | 14 .../compress/xerial/TestWALCompressionSnappy.java | 13 .../io/compress/xz/TestWALCompressionLzma.java | 13 .../io/compress/zstd/TestWALCompressionZstd.java | 14 .../org/apache/hadoop/hbase/quotas/QuotaCache.java | 12 ++- .../org/apache/hadoop/hbase/quotas/QuotaUtil.java | 6 +- .../hbase/regionserver/wal/CompressionContext.java | 17 + ...LDecompressionBoundedDelegatingInputStream.java | 16 ++-- .../apache/hadoop/hbase/quotas/TestQuotaCache.java | 89 ++ .../hadoop/hbase/quotas/ThrottleQuotaTestUtil.java | 12 +++ .../hadoop/hbase/wal/CompressedWALTestBase.java| 32 +++- .../apache/hadoop/hbase/wal/TestCompressedWAL.java | 13 .../wal/TestCompressedWALValueCompression.java | 13 18 files changed, 170 insertions(+), 160 deletions(-) create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestQuotaCache.java
(hbase) branch branch-2 updated: HBASE-28390 WAL value compression fails for cells with large values (#5696)
This is an automated email from the ASF dual-hosted git repository.
bbeaudreault pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new c28b1d58219 HBASE-28390 WAL value compression fails for cells with
large values (#5696)
c28b1d58219 is described below
commit c28b1d58219d6bb46cdbb3a4b5b8fbd6371aadce
Author: Bryan Beaudreault
AuthorDate: Fri Feb 23 12:29:45 2024 -0500
HBASE-28390 WAL value compression fails for cells with large values (#5696)
Signed-off-by: Andrew Purtell
---
.../aircompressor/TestWALCompressionLz4.java | 13 -
.../aircompressor/TestWALCompressionLzo.java | 14 --
.../aircompressor/TestWALCompressionSnappy.java| 13 -
.../aircompressor/TestWALCompressionZstd.java | 13 -
.../compress/brotli/TestWALCompressionBrotli.java | 13 -
.../io/compress/lz4/TestWALCompressionLz4.java | 14 --
.../compress/xerial/TestWALCompressionSnappy.java | 13 -
.../io/compress/xz/TestWALCompressionLzma.java | 13 -
.../io/compress/zstd/TestWALCompressionZstd.java | 14 --
.../hbase/regionserver/wal/CompressionContext.java | 17
...LDecompressionBoundedDelegatingInputStream.java | 16 ++-
.../hadoop/hbase/wal/CompressedWALTestBase.java| 32 --
.../apache/hadoop/hbase/wal/TestCompressedWAL.java | 13 -
.../wal/TestCompressedWALValueCompression.java | 13 -
14 files changed, 55 insertions(+), 156 deletions(-)
diff --git
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
index 34a7dcfedfc..1361754189a 100644
---
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
+++
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLz4 extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionLz4 extends
CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLz4.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -60,10 +53,4 @@ public class TestWALCompressionLz4 extends
CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
-TableName tableName =
TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
-doTest(tableName);
- }
-
}
diff --git
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
index 9c5bc8838c0..dd91b0cac02 100644
---
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
+++
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.w
(hbase) branch branch-3 updated: HBASE-28390 WAL value compression fails for cells with large values (#5696)
This is an automated email from the ASF dual-hosted git repository.
bbeaudreault pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new d01633fd256 HBASE-28390 WAL value compression fails for cells with
large values (#5696)
d01633fd256 is described below
commit d01633fd2567b16f5a6069fae2c519d79f1f2b93
Author: Bryan Beaudreault
AuthorDate: Fri Feb 23 12:29:45 2024 -0500
HBASE-28390 WAL value compression fails for cells with large values (#5696)
Signed-off-by: Andrew Purtell
---
.../aircompressor/TestWALCompressionLz4.java | 13 -
.../aircompressor/TestWALCompressionLzo.java | 14 --
.../aircompressor/TestWALCompressionSnappy.java| 13 -
.../aircompressor/TestWALCompressionZstd.java | 13 -
.../compress/brotli/TestWALCompressionBrotli.java | 13 -
.../io/compress/lz4/TestWALCompressionLz4.java | 14 --
.../compress/xerial/TestWALCompressionSnappy.java | 13 -
.../io/compress/xz/TestWALCompressionLzma.java | 13 -
.../io/compress/zstd/TestWALCompressionZstd.java | 14 --
.../hbase/regionserver/wal/CompressionContext.java | 17
...LDecompressionBoundedDelegatingInputStream.java | 16 ++-
.../hadoop/hbase/wal/CompressedWALTestBase.java| 32 --
.../apache/hadoop/hbase/wal/TestCompressedWAL.java | 13 -
.../wal/TestCompressedWALValueCompression.java | 13 -
14 files changed, 55 insertions(+), 156 deletions(-)
diff --git
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
index 34a7dcfedfc..1361754189a 100644
---
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
+++
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLz4 extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionLz4 extends
CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLz4.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -60,10 +53,4 @@ public class TestWALCompressionLz4 extends
CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
-TableName tableName =
TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
-doTest(tableName);
- }
-
}
diff --git
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
index 9c5bc8838c0..dd91b0cac02 100644
---
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
+++
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.w
(hbase) branch master updated: HBASE-28390 WAL value compression fails for cells with large values (#5696)
This is an automated email from the ASF dual-hosted git repository.
bbeaudreault pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 0ea11f2fc3a HBASE-28390 WAL value compression fails for cells with
large values (#5696)
0ea11f2fc3a is described below
commit 0ea11f2fc3a526b24c82248db0346d28bf20d12c
Author: Bryan Beaudreault
AuthorDate: Fri Feb 23 12:29:45 2024 -0500
HBASE-28390 WAL value compression fails for cells with large values (#5696)
Signed-off-by: Andrew Purtell
---
.../aircompressor/TestWALCompressionLz4.java | 13 -
.../aircompressor/TestWALCompressionLzo.java | 14 --
.../aircompressor/TestWALCompressionSnappy.java| 13 -
.../aircompressor/TestWALCompressionZstd.java | 13 -
.../compress/brotli/TestWALCompressionBrotli.java | 13 -
.../io/compress/lz4/TestWALCompressionLz4.java | 14 --
.../compress/xerial/TestWALCompressionSnappy.java | 13 -
.../io/compress/xz/TestWALCompressionLzma.java | 13 -
.../io/compress/zstd/TestWALCompressionZstd.java | 14 --
.../hbase/regionserver/wal/CompressionContext.java | 17
...LDecompressionBoundedDelegatingInputStream.java | 16 ++-
.../hadoop/hbase/wal/CompressedWALTestBase.java| 32 --
.../apache/hadoop/hbase/wal/TestCompressedWAL.java | 13 -
.../wal/TestCompressedWALValueCompression.java | 13 -
14 files changed, 55 insertions(+), 156 deletions(-)
diff --git
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
index 34a7dcfedfc..1361754189a 100644
---
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
+++
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLz4.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.CompressedWALTestBase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
@Category({ RegionServerTests.class, MediumTests.class })
public class TestWALCompressionLz4 extends CompressedWALTestBase {
@@ -41,9 +37,6 @@ public class TestWALCompressionLz4 extends
CompressedWALTestBase {
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestWALCompressionLz4.class);
- @Rule
- public TestName name = new TestName();
-
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
@@ -60,10 +53,4 @@ public class TestWALCompressionLz4 extends
CompressedWALTestBase {
TEST_UTIL.shutdownMiniCluster();
}
- @Test
- public void test() throws Exception {
-TableName tableName =
TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
-doTest(tableName);
- }
-
}
diff --git
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
index 9c5bc8838c0..dd91b0cac02 100644
---
a/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
+++
b/hbase-compression/hbase-compression-aircompressor/src/test/java/org/apache/hadoop/hbase/io/compress/aircompressor/TestWALCompressionLzo.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hbase.io.compress.aircompressor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.wal.CompressionContext;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -29,10 +28,7 @@ import org.apache.hadoop.hbase.wal.C
(hbase-site) branch asf-site updated: INFRA-10751 Empty commit
This is an automated email from the ASF dual-hosted git repository. git-site-role pushed a commit to branch asf-site in repository https://gitbox.apache.org/repos/asf/hbase-site.git The following commit(s) were added to refs/heads/asf-site by this push: new 6f03274117d INFRA-10751 Empty commit 6f03274117d is described below commit 6f03274117de212481efaeca4d6df20f8918348c Author: jenkins AuthorDate: Fri Feb 23 14:44:04 2024 + INFRA-10751 Empty commit
(hbase) branch master updated: HBASE-28342 Decommissioned hosts should be rejected by the HMaster (#5681)
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 8c989c92a67 HBASE-28342 Decommissioned hosts should be rejected by the
HMaster (#5681)
8c989c92a67 is described below
commit 8c989c92a67cc4373c1291f400aa372bca2d33b6
Author: Ahmad Alhour
AuthorDate: Fri Feb 23 12:54:59 2024 +0100
HBASE-28342 Decommissioned hosts should be rejected by the HMaster (#5681)
Signed-off by: Nick Dimiduk
---
.../java/org/apache/hadoop/hbase/HConstants.java | 14 +++
.../DecommissionedHostRejectedException.java | 28 ++
.../org/apache/hadoop/hbase/master/HMaster.java| 4 +-
.../apache/hadoop/hbase/master/ServerManager.java | 109 +
.../hadoop/hbase/regionserver/HRegionServer.java | 6 ++
.../TestRegionServerReportForDuty.java | 77 ++-
6 files changed, 215 insertions(+), 23 deletions(-)
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 5b53d2b2c0d..9597ec23d81 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1622,6 +1622,20 @@ public final class HConstants {
*/
public final static boolean HBASE_SERVER_USEIP_ENABLED_DEFAULT = false;
+ /**
+ * Should the HMaster reject hosts of decommissioned RegionServers, bypass
matching their port and
+ * startcode parts of their ServerName or not? When True, the HMaster will
reject a RegionServer's
+ * request to `reportForDuty` if it's hostname exists in the list of
decommissioned RegionServers
+ * it maintains internally. Added in HBASE-28342.
+ */
+ public final static String REJECT_DECOMMISSIONED_HOSTS_KEY =
+"hbase.master.reject.decommissioned.hosts";
+
+ /**
+ * Default value of {@link #REJECT_DECOMMISSIONED_HOSTS_KEY}
+ */
+ public final static boolean REJECT_DECOMMISSIONED_HOSTS_DEFAULT = false;
+
private HConstants() {
// Can't be instantiated with this ctor.
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DecommissionedHostRejectedException.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DecommissionedHostRejectedException.java
new file mode 100644
index 000..3d28b1e75be
--- /dev/null
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/DecommissionedHostRejectedException.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.yetus.audience.InterfaceAudience;
+
+@InterfaceAudience.Private
+public class DecommissionedHostRejectedException extends HBaseIOException {
+ public DecommissionedHostRejectedException(String message) {
+super(message);
+ }
+}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 88b82f01069..ddef3e27b40 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -546,7 +546,6 @@ public class HMaster extends
HBaseServerBase implements Maste
HConstants.DEFAULT_HBASE_MASTER_BALANCER_MAX_RIT_PERCENT);
// Do we publish the status?
-
boolean shouldPublish =
conf.getBoolean(HConstants.STATUS_PUBLISHED,
HConstants.STATUS_PUBLISHED_DEFAULT);
Class publisherClass =
@@ -997,7 +996,10 @@ public class HMaster extends
HBaseServerBase implements Maste
masterRegion = MasterRegionFactory.create(this);
rsListStorage = new MasterRegionServerList(masterRegion, this);
+// Initialize the ServerManager and register it as a configuration observer
this.serverManager = createServerManager(this, rsListStorage);
+this.configurationManager.registerObserver(this.serverManager);
+
this.syncReplicationReplayWALManage
