This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new a00c02a4d79 HBASE-29966 Fix race in FlushRegionProcedure and rewrite
TestAcidGuarantees (#7886)
a00c02a4d79 is described below
commit a00c02a4d793977d39aea930f8c5936ec5024fda
Author: Duo Zhang <[email protected]>
AuthorDate: Mon Mar 9 15:04:32 2026 +0800
HBASE-29966 Fix race in FlushRegionProcedure and rewrite TestAcidGuarantees
(#7886)
Signed-off-by: Hui Ruan <[email protected]>
Reviewed-by: Liu Xiao <[email protected]>
(cherry picked from commit 7f1ea6aae4baff26ffbeb1397d0b40b64d31f71f)
---
.../hbase/HBaseParameterizedTemplateProvider.java | 33 +++++++---
.../master/procedure/FlushRegionProcedure.java | 4 +-
.../hadoop/hbase/AcidGuaranteesTestBase.java | 77 +++++++++-------------
...sWithEagerPolicy.java => TestGetAtomicity.java} | 21 +++---
...ithBasicPolicy.java => TestMixedAtomicity.java} | 21 +++---
...daptivePolicy.java => TestMobGetAtomicity.java} | 21 +++---
...mCompaction.java => TestMobMixedAtomicity.java} | 21 +++---
...hBasicPolicy.java => TestMobScanAtomicity.java} | 21 +++---
...WithBasicPolicy.java => TestScanAtomicity.java} | 21 +++---
9 files changed, 123 insertions(+), 117 deletions(-)
diff --git
a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseParameterizedTemplateProvider.java
b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseParameterizedTemplateProvider.java
index 90ec0d0d1a6..01632233f31 100644
---
a/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseParameterizedTemplateProvider.java
+++
b/hbase-common/src/test/java/org/apache/hadoop/hbase/HBaseParameterizedTemplateProvider.java
@@ -27,6 +27,8 @@ import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.TestTemplateInvocationContext;
import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider;
import org.junit.jupiter.params.provider.Arguments;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* The entry point class for supporting JUnit4 like Parameterized test, where
we can use constructor
@@ -48,6 +50,9 @@ import org.junit.jupiter.params.provider.Arguments;
@InterfaceAudience.Private
public class HBaseParameterizedTemplateProvider implements
TestTemplateInvocationContextProvider {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HBaseParameterizedTemplateProvider.class);
+
private static final String PARAMETERS_METHOD_NAME = "parameters";
@Override
@@ -56,19 +61,31 @@ public class HBaseParameterizedTemplateProvider implements
TestTemplateInvocatio
.map(c ->
c.isAnnotationPresent(HBaseParameterizedTestTemplate.class)).orElse(false);
}
+ private Method findParametersMethod(final Class<?> testClass) {
+ Class<?> clazz = testClass;
+ for (;;) {
+ try {
+ return clazz.getDeclaredMethod(PARAMETERS_METHOD_NAME);
+ } catch (NoSuchMethodException e) {
+ Class<?> parentClass = clazz.getSuperclass();
+ LOG.debug("Can not find method {} in class {}, try its parent class
{}",
+ PARAMETERS_METHOD_NAME, clazz, parentClass);
+ if (parentClass == null) {
+ throw new ExtensionConfigurationException("Can not find a static "
+ + PARAMETERS_METHOD_NAME + " method in class " + testClass + "or
its super classes");
+ }
+ clazz = parentClass;
+ }
+ }
+
+ }
+
@Override
public Stream<TestTemplateInvocationContext>
provideTestTemplateInvocationContexts(ExtensionContext context) {
Class<?> testClass = context.getRequiredTestClass();
// get parameters
- Method method;
- try {
- method = testClass.getDeclaredMethod(PARAMETERS_METHOD_NAME);
- } catch (NoSuchMethodException e) {
- throw new ExtensionConfigurationException(
- "Test class must declare static " + PARAMETERS_METHOD_NAME + "
method");
- }
-
+ Method method = findParametersMethod(testClass);
if (!Modifier.isStatic(method.getModifiers())) {
throw new ExtensionConfigurationException(PARAMETERS_METHOD_NAME + "
method must be static");
}
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
index 091fdc791bf..aa29bc933e9 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/FlushRegionProcedure.java
@@ -77,7 +77,7 @@ public class FlushRegionProcedure extends
Procedure<MasterProcedureEnv>
}
@Override
- protected Procedure<MasterProcedureEnv>[] execute(MasterProcedureEnv env)
+ protected synchronized Procedure<MasterProcedureEnv>[]
execute(MasterProcedureEnv env)
throws ProcedureYieldException, ProcedureSuspendedException,
InterruptedException {
if (dispatched) {
if (succ) {
@@ -158,7 +158,7 @@ public class FlushRegionProcedure extends
Procedure<MasterProcedureEnv>
complete(env, error);
}
- private void complete(MasterProcedureEnv env, Throwable error) {
+ private synchronized void complete(MasterProcedureEnv env, Throwable error) {
if (isFinished()) {
LOG.info("This procedure {} is already finished, skip the rest
processes", this.getProcId());
return;
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestBase.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestBase.java
index 0e76eb247a4..bff799ad255 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestBase.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/AcidGuaranteesTestBase.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hbase;
import static org.apache.hadoop.hbase.AcidGuaranteesTestTool.FAMILIES;
import static org.apache.hadoop.hbase.AcidGuaranteesTestTool.TABLE_NAME;
+import java.util.Arrays;
import java.util.List;
import java.util.stream.Stream;
import org.apache.hadoop.conf.Configuration;
@@ -28,11 +29,11 @@ import
org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.regionserver.CompactingMemStore;
import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.provider.Arguments;
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
@@ -47,9 +48,17 @@ public abstract class AcidGuaranteesTestBase {
private AcidGuaranteesTestTool tool = new AcidGuaranteesTestTool();
- protected abstract MemoryCompactionPolicy getMemoryCompactionPolicy();
+ private MemoryCompactionPolicy policy;
- @BeforeClass
+ protected AcidGuaranteesTestBase(MemoryCompactionPolicy policy) {
+ this.policy = policy;
+ }
+
+ public static Stream<Arguments> parameters() {
+ return Arrays.stream(MemoryCompactionPolicy.values()).map(Arguments::of);
+ }
+
+ @BeforeAll
public static void setUpBeforeClass() throws Exception {
// Set small flush size for minicluster so we exercise reseeking scanners
Configuration conf = UTIL.getConfiguration();
@@ -61,14 +70,13 @@ public abstract class AcidGuaranteesTestBase {
UTIL.startMiniCluster(1);
}
- @AfterClass
+ @AfterAll
public static void tearDownAfterClass() throws Exception {
UTIL.shutdownMiniCluster();
}
- @Before
+ @BeforeEach
public void setUp() throws Exception {
- MemoryCompactionPolicy policy = getMemoryCompactionPolicy();
TableDescriptorBuilder builder =
TableDescriptorBuilder.newBuilder(TABLE_NAME)
.setValue(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY,
policy.name());
if (policy == MemoryCompactionPolicy.EAGER) {
@@ -77,22 +85,26 @@ public abstract class AcidGuaranteesTestBase {
}
Stream.of(FAMILIES).map(ColumnFamilyDescriptorBuilder::of)
.forEachOrdered(builder::setColumnFamily);
+ for (int i = 0; i < 10; i++) {
+ // try to delete the table several times
+ if (UTIL.getAdmin().tableExists(TABLE_NAME)) {
+ UTIL.deleteTable(TABLE_NAME);
+ Thread.sleep(1000);
+ } else {
+ break;
+ }
+ }
UTIL.getAdmin().createTable(builder.build());
tool.setConf(UTIL.getConfiguration());
}
- @After
+ @AfterEach
public void tearDown() throws Exception {
UTIL.deleteTable(TABLE_NAME);
}
- private void runTestAtomicity(long millisToRun, int numWriters, int
numGetters, int numScanners,
- int numUniqueRows) throws Exception {
- runTestAtomicity(millisToRun, numWriters, numGetters, numScanners,
numUniqueRows, false);
- }
-
- private void runTestAtomicity(long millisToRun, int numWriters, int
numGetters, int numScanners,
- int numUniqueRows, boolean useMob) throws Exception {
+ protected final void runTestAtomicity(long millisToRun, int numWriters, int
numGetters,
+ int numScanners, int numUniqueRows, boolean useMob) throws Exception {
List<String> args = Lists.newArrayList("-millis",
String.valueOf(millisToRun), "-numWriters",
String.valueOf(numWriters), "-numGetters", String.valueOf(numGetters),
"-numScanners",
String.valueOf(numScanners), "-numUniqueRows",
String.valueOf(numUniqueRows), "-crazyFlush");
@@ -102,33 +114,4 @@ public abstract class AcidGuaranteesTestBase {
tool.run(args.toArray(new String[0]));
}
- @Test
- public void testGetAtomicity() throws Exception {
- runTestAtomicity(20000, 5, 5, 0, 3);
- }
-
- @Test
- public void testScanAtomicity() throws Exception {
- runTestAtomicity(20000, 5, 0, 5, 3);
- }
-
- @Test
- public void testMixedAtomicity() throws Exception {
- runTestAtomicity(20000, 5, 2, 2, 3);
- }
-
- @Test
- public void testMobGetAtomicity() throws Exception {
- runTestAtomicity(20000, 5, 5, 0, 3, true);
- }
-
- @Test
- public void testMobScanAtomicity() throws Exception {
- runTestAtomicity(20000, 5, 0, 5, 3, true);
- }
-
- @Test
- public void testMobMixedAtomicity() throws Exception {
- runTestAtomicity(20000, 5, 2, 2, 3, true);
- }
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithEagerPolicy.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestGetAtomicity.java
similarity index 67%
rename from
hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithEagerPolicy.java
rename to
hbase-server/src/test/java/org/apache/hadoop/hbase/TestGetAtomicity.java
index e1d5b55cf52..5f0903a32e8 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithEagerPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestGetAtomicity.java
@@ -18,18 +18,19 @@
package org.apache.hadoop.hbase;
import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.ClassRule;
-import org.junit.experimental.categories.Category;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.TestTemplate;
-@Category(LargeTests.class)
-public class TestAcidGuaranteesWithEagerPolicy extends AcidGuaranteesTestBase {
+@Tag(LargeTests.TAG)
+@HBaseParameterizedTestTemplate(name = "{index}: policy = {0}")
+public class TestGetAtomicity extends AcidGuaranteesTestBase {
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestAcidGuaranteesWithEagerPolicy.class);
+ public TestGetAtomicity(MemoryCompactionPolicy policy) {
+ super(policy);
+ }
- @Override
- protected MemoryCompactionPolicy getMemoryCompactionPolicy() {
- return MemoryCompactionPolicy.EAGER;
+ @TestTemplate
+ public void testGetAtomicity() throws Exception {
+ runTestAtomicity(20000, 5, 5, 0, 3, false);
}
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMixedAtomicity.java
similarity index 67%
copy from
hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java
copy to
hbase-server/src/test/java/org/apache/hadoop/hbase/TestMixedAtomicity.java
index 5f1b5e0e41f..dc8c74f90d2 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMixedAtomicity.java
@@ -18,18 +18,19 @@
package org.apache.hadoop.hbase;
import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.ClassRule;
-import org.junit.experimental.categories.Category;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.TestTemplate;
-@Category(LargeTests.class)
-public class TestAcidGuaranteesWithBasicPolicy extends AcidGuaranteesTestBase {
+@Tag(LargeTests.TAG)
+@HBaseParameterizedTestTemplate(name = "{index}: policy = {0}")
+public class TestMixedAtomicity extends AcidGuaranteesTestBase {
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestAcidGuaranteesWithBasicPolicy.class);
+ public TestMixedAtomicity(MemoryCompactionPolicy policy) {
+ super(policy);
+ }
- @Override
- protected MemoryCompactionPolicy getMemoryCompactionPolicy() {
- return MemoryCompactionPolicy.BASIC;
+ @TestTemplate
+ public void testMixedAtomicity() throws Exception {
+ runTestAtomicity(20000, 5, 2, 2, 3, false);
}
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithAdaptivePolicy.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMobGetAtomicity.java
similarity index 66%
rename from
hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithAdaptivePolicy.java
rename to
hbase-server/src/test/java/org/apache/hadoop/hbase/TestMobGetAtomicity.java
index 8e7f924ce10..1dd6bf1c642 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithAdaptivePolicy.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMobGetAtomicity.java
@@ -18,18 +18,19 @@
package org.apache.hadoop.hbase;
import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.ClassRule;
-import org.junit.experimental.categories.Category;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.TestTemplate;
-@Category(LargeTests.class)
-public class TestAcidGuaranteesWithAdaptivePolicy extends
AcidGuaranteesTestBase {
+@Tag(LargeTests.TAG)
+@HBaseParameterizedTestTemplate(name = "{index}: policy = {0}")
+public class TestMobGetAtomicity extends AcidGuaranteesTestBase {
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestAcidGuaranteesWithAdaptivePolicy.class);
+ public TestMobGetAtomicity(MemoryCompactionPolicy policy) {
+ super(policy);
+ }
- @Override
- protected MemoryCompactionPolicy getMemoryCompactionPolicy() {
- return MemoryCompactionPolicy.ADAPTIVE;
+ @TestTemplate
+ public void testMobScanAtomicity() throws Exception {
+ runTestAtomicity(20000, 5, 0, 5, 3, true);
}
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithNoInMemCompaction.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMobMixedAtomicity.java
similarity index 66%
rename from
hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithNoInMemCompaction.java
rename to
hbase-server/src/test/java/org/apache/hadoop/hbase/TestMobMixedAtomicity.java
index c8daa07724e..217c1f24275 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithNoInMemCompaction.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMobMixedAtomicity.java
@@ -18,18 +18,19 @@
package org.apache.hadoop.hbase;
import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.ClassRule;
-import org.junit.experimental.categories.Category;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.TestTemplate;
-@Category(LargeTests.class)
-public class TestAcidGuaranteesWithNoInMemCompaction extends
AcidGuaranteesTestBase {
+@Tag(LargeTests.TAG)
+@HBaseParameterizedTestTemplate(name = "{index}: policy = {0}")
+public class TestMobMixedAtomicity extends AcidGuaranteesTestBase {
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestAcidGuaranteesWithNoInMemCompaction.class);
+ public TestMobMixedAtomicity(MemoryCompactionPolicy policy) {
+ super(policy);
+ }
- @Override
- protected MemoryCompactionPolicy getMemoryCompactionPolicy() {
- return MemoryCompactionPolicy.NONE;
+ @TestTemplate
+ public void testMobMixedAtomicity() throws Exception {
+ runTestAtomicity(20000, 5, 2, 2, 3, true);
}
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMobScanAtomicity.java
similarity index 67%
copy from
hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java
copy to
hbase-server/src/test/java/org/apache/hadoop/hbase/TestMobScanAtomicity.java
index 5f1b5e0e41f..a0ab556eedd 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMobScanAtomicity.java
@@ -18,18 +18,19 @@
package org.apache.hadoop.hbase;
import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.ClassRule;
-import org.junit.experimental.categories.Category;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.TestTemplate;
-@Category(LargeTests.class)
-public class TestAcidGuaranteesWithBasicPolicy extends AcidGuaranteesTestBase {
+@Tag(LargeTests.TAG)
+@HBaseParameterizedTestTemplate(name = "{index}: policy = {0}")
+public class TestMobScanAtomicity extends AcidGuaranteesTestBase {
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestAcidGuaranteesWithBasicPolicy.class);
+ public TestMobScanAtomicity(MemoryCompactionPolicy policy) {
+ super(policy);
+ }
- @Override
- protected MemoryCompactionPolicy getMemoryCompactionPolicy() {
- return MemoryCompactionPolicy.BASIC;
+ @TestTemplate
+ public void testMobGetAtomicity() throws Exception {
+ runTestAtomicity(20000, 5, 5, 0, 3, true);
}
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestScanAtomicity.java
similarity index 67%
rename from
hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java
rename to
hbase-server/src/test/java/org/apache/hadoop/hbase/TestScanAtomicity.java
index 5f1b5e0e41f..71ee3fac87a 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestAcidGuaranteesWithBasicPolicy.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestScanAtomicity.java
@@ -18,18 +18,19 @@
package org.apache.hadoop.hbase;
import org.apache.hadoop.hbase.testclassification.LargeTests;
-import org.junit.ClassRule;
-import org.junit.experimental.categories.Category;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.TestTemplate;
-@Category(LargeTests.class)
-public class TestAcidGuaranteesWithBasicPolicy extends AcidGuaranteesTestBase {
+@Tag(LargeTests.TAG)
+@HBaseParameterizedTestTemplate(name = "{index}: policy = {0}")
+public class TestScanAtomicity extends AcidGuaranteesTestBase {
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
- HBaseClassTestRule.forClass(TestAcidGuaranteesWithBasicPolicy.class);
+ public TestScanAtomicity(MemoryCompactionPolicy policy) {
+ super(policy);
+ }
- @Override
- protected MemoryCompactionPolicy getMemoryCompactionPolicy() {
- return MemoryCompactionPolicy.BASIC;
+ @TestTemplate
+ public void testScanAtomicity() throws Exception {
+ runTestAtomicity(20000, 5, 0, 5, 3, false);
}
}
