This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new 4741cec0afa HBASE-29982 Upgrade TestTags to JUnit5 and polish the code
(#7897) (#7907)
4741cec0afa is described below
commit 4741cec0afa8d9d87ef2643291360e13a17d6909
Author: Duo Zhang <[email protected]>
AuthorDate: Thu Mar 12 10:58:05 2026 +0800
HBASE-29982 Upgrade TestTags to JUnit5 and polish the code (#7897) (#7907)
Reviewed-by: Liu Xiao <[email protected]>
Signed-off-by: Nihal Jain <[email protected]>
(cherry picked from commit 1606be719c730dd5d9267f3221f88b81d159d3fd)
(cherry picked from commit dc860f37bd39dcbeadd2dbac0c63f2c13e26bf76)
---
.../apache/hadoop/hbase/regionserver/TestTags.java | 603 +++++++--------------
.../hbase/regionserver/TestTagsReverseScan.java | 158 ++++++
2 files changed, 366 insertions(+), 395 deletions(-)
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java
index 9832badccd9..895b00ce8d1 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTags.java
@@ -19,32 +19,27 @@ package org.apache.hadoop.hbase.regionserver;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
+import java.util.concurrent.ConcurrentLinkedQueue;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ArrayBackedTag;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.PrivateCellUtil;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Tag;
-import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Append;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
-import org.apache.hadoop.hbase.client.CompactionState;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Mutation;
@@ -53,6 +48,7 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
@@ -60,40 +56,28 @@ import
org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.RegionObserver;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.wal.WALEdit;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-import org.junit.rules.TestName;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
/**
* Class that test tags
*/
-@Category({ RegionServerTests.class, LargeTests.class })
[email protected](RegionServerTests.TAG)
[email protected](MediumTests.TAG)
public class TestTags {
- @ClassRule
- public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestTags.class);
-
- private static final Logger LOG = LoggerFactory.getLogger(TestTags.class);
-
- static boolean useFilter = false;
-
private final static HBaseTestingUtility TEST_UTIL = new
HBaseTestingUtility();
- @Rule
- public final TestName TEST_NAME = new TestName();
-
- @BeforeClass
+ @BeforeAll
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
conf.setInt("hfile.format.version", 3);
@@ -102,213 +86,99 @@ public class TestTags {
TEST_UTIL.startMiniCluster(2);
}
- @AfterClass
+ @AfterAll
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}
- @After
- public void tearDown() {
- useFilter = false;
+ @AfterEach
+ public void tearDown() throws InterruptedException {
+ resetCoprocessorFlags();
}
- /**
- * Test that we can do reverse scans when writing tags and using
DataBlockEncoding. Fails with an
- * exception for PREFIX, DIFF, and FAST_DIFF prior to HBASE-27580
- */
- @Test
- public void testReverseScanWithDBE() throws IOException {
- byte[] family = Bytes.toBytes("0");
-
- Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
- conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
-
- try (Connection connection = ConnectionFactory.createConnection(conf)) {
- for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
- testReverseScanWithDBE(connection, encoding, family,
HConstants.DEFAULT_BLOCKSIZE, 10);
- }
- }
+ private void resetCoprocessorFlags() throws InterruptedException {
+ TestCoprocessorForTags.checkTagPresence = false;
+ // to avoid data race, here we sleep for a little while before reseting
tags
+ Thread.sleep(100);
+ TestCoprocessorForTags.tags.clear();
}
- /**
- * Test that we can do reverse scans when writing tags and using
DataBlockEncoding. Fails with an
- * exception for PREFIX, DIFF, and FAST_DIFF
- */
@Test
- public void testReverseScanWithDBEWhenCurrentBlockUpdates() throws
IOException {
- byte[] family = Bytes.toBytes("0");
-
- Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
- conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
-
- try (Connection connection = ConnectionFactory.createConnection(conf)) {
- for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
- testReverseScanWithDBE(connection, encoding, family, 1024, 30000);
- }
- }
- }
-
- private void testReverseScanWithDBE(Connection conn, DataBlockEncoding
encoding, byte[] family,
- int blockSize, int maxRows) throws IOException {
- LOG.info("Running test with DBE={}", encoding);
- TableName tableName = TableName.valueOf(TEST_NAME.getMethodName() + "-" +
encoding);
- TEST_UTIL.createTable(
-
TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder
-
.newBuilder(family).setDataBlockEncoding(encoding).setBlocksize(blockSize).build()).build(),
- null);
-
- Table table = conn.getTable(tableName);
-
- byte[] val1 = new byte[10];
- byte[] val2 = new byte[10];
- Bytes.random(val1);
- Bytes.random(val2);
-
- for (int i = 0; i < maxRows; i++) {
- table.put(new Put(Bytes.toBytes(i)).addColumn(family, Bytes.toBytes(1),
val1)
- .addColumn(family, Bytes.toBytes(2), val2).setTTL(600_000));
- }
-
- TEST_UTIL.flush(table.getName());
-
- Scan scan = new Scan();
- scan.setReversed(true);
-
- try (ResultScanner scanner = table.getScanner(scan)) {
- for (int i = maxRows - 1; i >= 0; i--) {
- Result row = scanner.next();
- assertEquals(2, row.size());
-
- Cell cell1 = row.getColumnLatestCell(family, Bytes.toBytes(1));
- assertTrue(CellUtil.matchingRows(cell1, Bytes.toBytes(i)));
- assertTrue(CellUtil.matchingValue(cell1, val1));
-
- Cell cell2 = row.getColumnLatestCell(family, Bytes.toBytes(2));
- assertTrue(CellUtil.matchingRows(cell2, Bytes.toBytes(i)));
- assertTrue(CellUtil.matchingValue(cell2, val2));
- }
- }
-
- }
+ public void testTags(TestInfo testInfo) throws Exception {
+ TableName tableName =
TableName.valueOf(testInfo.getTestMethod().get().getName());
+ byte[] fam = Bytes.toBytes("info");
+ byte[] row = Bytes.toBytes("rowa");
+ // column names
+ byte[] qual = Bytes.toBytes("qual");
+ byte[] row1 = Bytes.toBytes("rowb");
+ byte[] row2 = Bytes.toBytes("rowc");
- @Test
- public void testTags() throws Exception {
- Table table = null;
- try {
- TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
- byte[] fam = Bytes.toBytes("info");
- byte[] row = Bytes.toBytes("rowa");
- // column names
- byte[] qual = Bytes.toBytes("qual");
-
- byte[] row1 = Bytes.toBytes("rowb");
-
- byte[] row2 = Bytes.toBytes("rowc");
-
- HTableDescriptor desc = new HTableDescriptor(tableName);
- HColumnDescriptor colDesc = new HColumnDescriptor(fam);
- colDesc.setBlockCacheEnabled(true);
- colDesc.setDataBlockEncoding(DataBlockEncoding.NONE);
- desc.addFamily(colDesc);
- Admin admin = TEST_UTIL.getAdmin();
- admin.createTable(desc);
+ TableDescriptor tableDescriptor =
TableDescriptorBuilder.newBuilder(tableName)
+
.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(fam).setBlockCacheEnabled(true)
+ .setDataBlockEncoding(DataBlockEncoding.NONE).build())
+ .build();
+ try (Table table = TEST_UTIL.createTable(tableDescriptor, null)) {
byte[] value = Bytes.toBytes("value");
- table = TEST_UTIL.getConnection().getTable(tableName);
Put put = new Put(row);
put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
put.setAttribute("visibility", Bytes.toBytes("myTag"));
table.put(put);
- admin.flush(tableName);
- // We are lacking an API for confirming flush request compaction.
- // Just sleep for a short time. We won't be able to confirm flush
- // completion but the test won't hang now or in the future if
- // default compaction policy causes compaction between flush and
- // when we go to confirm it.
- Thread.sleep(1000);
+ TEST_UTIL.flush(tableName);
Put put1 = new Put(row1);
byte[] value1 = Bytes.toBytes("1000dfsdf");
put1.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value1);
// put1.setAttribute("visibility", Bytes.toBytes("myTag3"));
table.put(put1);
- admin.flush(tableName);
- Thread.sleep(1000);
+ TEST_UTIL.flush(tableName);
Put put2 = new Put(row2);
byte[] value2 = Bytes.toBytes("1000dfsdf");
put2.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value2);
put2.setAttribute("visibility", Bytes.toBytes("myTag3"));
table.put(put2);
- admin.flush(tableName);
- Thread.sleep(1000);
+ TEST_UTIL.flush(tableName);
- result(fam, row, qual, row2, table, value, value2, row1, value1);
+ assertResult(fam, row, qual, row2, table, value, value2, row1, value1);
- admin.compact(tableName);
- while (admin.getCompactionState(tableName) != CompactionState.NONE) {
- Thread.sleep(10);
- }
- result(fam, row, qual, row2, table, value, value2, row1, value1);
- } finally {
- if (table != null) {
- table.close();
- }
+ TEST_UTIL.compact(tableName, false);
+ assertResult(fam, row, qual, row2, table, value, value2, row1, value1);
}
}
@Test
- public void testFlushAndCompactionWithoutTags() throws Exception {
- Table table = null;
- try {
- TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
- byte[] fam = Bytes.toBytes("info");
- byte[] row = Bytes.toBytes("rowa");
- // column names
- byte[] qual = Bytes.toBytes("qual");
-
- byte[] row1 = Bytes.toBytes("rowb");
-
- byte[] row2 = Bytes.toBytes("rowc");
-
- HTableDescriptor desc = new HTableDescriptor(tableName);
- HColumnDescriptor colDesc = new HColumnDescriptor(fam);
- colDesc.setBlockCacheEnabled(true);
- // colDesc.setDataBlockEncoding(DataBlockEncoding.NONE);
- // colDesc.setDataBlockEncoding(DataBlockEncoding.PREFIX_TREE);
- desc.addFamily(colDesc);
- Admin admin = TEST_UTIL.getAdmin();
- admin.createTable(desc);
-
- table = TEST_UTIL.getConnection().getTable(tableName);
+ public void testFlushAndCompactionWithoutTags(TestInfo testInfo) throws
Exception {
+ TableName tableName =
TableName.valueOf(testInfo.getTestMethod().get().getName());
+ byte[] fam = Bytes.toBytes("info");
+ byte[] row = Bytes.toBytes("rowa");
+ // column names
+ byte[] qual = Bytes.toBytes("qual");
+ byte[] row1 = Bytes.toBytes("rowb");
+ byte[] row2 = Bytes.toBytes("rowc");
+
+ TableDescriptor tableDescriptor =
TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(
+
ColumnFamilyDescriptorBuilder.newBuilder(fam).setBlockCacheEnabled(true).build()).build();
+ try (Table table = TEST_UTIL.createTable(tableDescriptor, null)) {
Put put = new Put(row);
byte[] value = Bytes.toBytes("value");
put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
table.put(put);
- admin.flush(tableName);
- // We are lacking an API for confirming flush request compaction.
- // Just sleep for a short time. We won't be able to confirm flush
- // completion but the test won't hang now or in the future if
- // default compaction policy causes compaction between flush and
- // when we go to confirm it.
- Thread.sleep(1000);
+ TEST_UTIL.flush(tableName);
Put put1 = new Put(row1);
byte[] value1 = Bytes.toBytes("1000dfsdf");
put1.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value1);
table.put(put1);
- admin.flush(tableName);
- Thread.sleep(1000);
+ TEST_UTIL.flush(tableName);
Put put2 = new Put(row2);
byte[] value2 = Bytes.toBytes("1000dfsdf");
put2.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value2);
table.put(put2);
- admin.flush(tableName);
- Thread.sleep(1000);
+ TEST_UTIL.flush(tableName);
- Scan s = new Scan(row);
- ResultScanner scanner = table.getScanner(s);
- try {
+ Scan s = new Scan().withStartRow(row);
+ try (ResultScanner scanner = table.getScanner(s)) {
Result[] next = scanner.next(3);
for (Result result : next) {
CellScanner cellScanner = result.cellScanner();
@@ -316,16 +186,11 @@ public class TestTags {
Cell current = cellScanner.current();
assertEquals(0, current.getTagsLength());
}
- } finally {
- if (scanner != null) scanner.close();
}
- admin.compact(tableName);
- while (admin.getCompactionState(tableName) != CompactionState.NONE) {
- Thread.sleep(10);
- }
- s = new Scan(row);
- scanner = table.getScanner(s);
- try {
+
+ TEST_UTIL.compact(tableName, false);
+ s = new Scan().withStartRow(row);
+ try (ResultScanner scanner = table.getScanner(s)) {
Result[] next = scanner.next(3);
for (Result result : next) {
CellScanner cellScanner = result.cellScanner();
@@ -333,155 +198,113 @@ public class TestTags {
Cell current = cellScanner.current();
assertEquals(0, current.getTagsLength());
}
- } finally {
- if (scanner != null) {
- scanner.close();
- }
- }
- } finally {
- if (table != null) {
- table.close();
}
}
}
- @Test
- public void testFlushAndCompactionwithCombinations() throws Exception {
- TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ @ParameterizedTest(name = "{index}: DataBlockEncoding = {0}")
+ @EnumSource(DataBlockEncoding.class)
+ public void testFlushAndCompactionwithCombinations(DataBlockEncoding
encoding, TestInfo testInfo)
+ throws Exception {
+ TableName tableName =
+ TableName.valueOf(testInfo.getTestMethod().get().getName() + "-" +
encoding.name());
byte[] fam = Bytes.toBytes("info");
byte[] row = Bytes.toBytes("rowa");
// column names
byte[] qual = Bytes.toBytes("qual");
-
byte[] row1 = Bytes.toBytes("rowb");
-
byte[] row2 = Bytes.toBytes("rowc");
byte[] rowd = Bytes.toBytes("rowd");
byte[] rowe = Bytes.toBytes("rowe");
- Table table = null;
- for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
- HTableDescriptor desc = new HTableDescriptor(tableName);
- HColumnDescriptor colDesc = new HColumnDescriptor(fam);
- colDesc.setBlockCacheEnabled(true);
- colDesc.setDataBlockEncoding(encoding);
- desc.addFamily(colDesc);
- Admin admin = TEST_UTIL.getAdmin();
- admin.createTable(desc);
- try {
- table = TEST_UTIL.getConnection().getTable(tableName);
- Put put = new Put(row);
- byte[] value = Bytes.toBytes("value");
- put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
- int bigTagLen = Short.MAX_VALUE - 5;
- put.setAttribute("visibility", new byte[bigTagLen]);
- table.put(put);
- Put put1 = new Put(row1);
- byte[] value1 = Bytes.toBytes("1000dfsdf");
- put1.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value1);
- table.put(put1);
- admin.flush(tableName);
- // We are lacking an API for confirming flush request compaction.
- // Just sleep for a short time. We won't be able to confirm flush
- // completion but the test won't hang now or in the future if
- // default compaction policy causes compaction between flush and
- // when we go to confirm it.
- Thread.sleep(1000);
-
- put1 = new Put(row2);
- value1 = Bytes.toBytes("1000dfsdf");
- put1.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value1);
- table.put(put1);
- admin.flush(tableName);
- Thread.sleep(1000);
-
- Put put2 = new Put(rowd);
- byte[] value2 = Bytes.toBytes("1000dfsdf");
- put2.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value2);
- table.put(put2);
- put2 = new Put(rowe);
- value2 = Bytes.toBytes("1000dfsddfdf");
- put2.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value2);
- put.setAttribute("visibility", Bytes.toBytes("ram"));
- table.put(put2);
- admin.flush(tableName);
- Thread.sleep(1000);
-
- TestCoprocessorForTags.checkTagPresence = true;
- Scan s = new Scan(row);
- s.setCaching(1);
- ResultScanner scanner = table.getScanner(s);
- try {
- Result next = null;
- while ((next = scanner.next()) != null) {
- CellScanner cellScanner = next.cellScanner();
- cellScanner.advance();
- Cell current = cellScanner.current();
- if (CellUtil.matchingRows(current, row)) {
- assertEquals(1, TestCoprocessorForTags.tags.size());
- Tag tag = TestCoprocessorForTags.tags.get(0);
- assertEquals(bigTagLen, tag.getValueLength());
- } else {
- assertEquals(0, TestCoprocessorForTags.tags.size());
- }
- }
- } finally {
- if (scanner != null) {
- scanner.close();
+ TableDescriptor tableDescriptor =
+
TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder
+
.newBuilder(fam).setBlockCacheEnabled(true).setDataBlockEncoding(encoding).build()).build();
+ try (Table table = TEST_UTIL.createTable(tableDescriptor, null)) {
+ Put put = new Put(row);
+ byte[] value = Bytes.toBytes("value");
+ put.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value);
+ int bigTagLen = Short.MAX_VALUE - 5;
+ put.setAttribute("visibility", new byte[bigTagLen]);
+ table.put(put);
+ Put put1 = new Put(row1);
+ byte[] value1 = Bytes.toBytes("1000dfsdf");
+ put1.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value1);
+ table.put(put1);
+ TEST_UTIL.flush(tableName);
+
+ put1 = new Put(row2);
+ value1 = Bytes.toBytes("1000dfsdf");
+ put1.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value1);
+ table.put(put1);
+ TEST_UTIL.flush(tableName);
+
+ Put put2 = new Put(rowd);
+ byte[] value2 = Bytes.toBytes("1000dfsdf");
+ put2.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value2);
+ table.put(put2);
+ put2 = new Put(rowe);
+ value2 = Bytes.toBytes("1000dfsddfdf");
+ put2.addColumn(fam, qual, HConstants.LATEST_TIMESTAMP, value2);
+ put.setAttribute("visibility", Bytes.toBytes("ram"));
+ table.put(put2);
+ TEST_UTIL.flush(tableName);
+
+ TestCoprocessorForTags.checkTagPresence = true;
+ Scan s = new Scan().withStartRow(row);
+ s.setCaching(1);
+ try (ResultScanner scanner = table.getScanner(s)) {
+ Result next = null;
+ while ((next = scanner.next()) != null) {
+ CellScanner cellScanner = next.cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ List<Tag> tags = TestCoprocessorForTags.tags.poll();
+ if (CellUtil.matchingRows(current, row)) {
+ assertEquals(1, tags.size());
+ Tag tag = tags.get(0);
+ assertEquals(bigTagLen, tag.getValueLength());
+ } else {
+ assertEquals(0, tags.size());
}
- TestCoprocessorForTags.checkTagPresence = false;
}
- while (admin.getCompactionState(tableName) != CompactionState.NONE) {
- Thread.sleep(10);
- }
- TestCoprocessorForTags.checkTagPresence = true;
- scanner = table.getScanner(s);
- try {
- Result next = null;
- while ((next = scanner.next()) != null) {
- CellScanner cellScanner = next.cellScanner();
- cellScanner.advance();
- Cell current = cellScanner.current();
- if (CellUtil.matchingRows(current, row)) {
- assertEquals(1, TestCoprocessorForTags.tags.size());
- Tag tag = TestCoprocessorForTags.tags.get(0);
- assertEquals(bigTagLen, tag.getValueLength());
- } else {
- assertEquals(0, TestCoprocessorForTags.tags.size());
- }
- }
- } finally {
- if (scanner != null) {
- scanner.close();
+ }
+ resetCoprocessorFlags();
+
+ TEST_UTIL.compact(tableName, false);
+
+ TestCoprocessorForTags.checkTagPresence = true;
+
+ try (ResultScanner scanner = table.getScanner(s)) {
+ Result next = null;
+ while ((next = scanner.next()) != null) {
+ CellScanner cellScanner = next.cellScanner();
+ cellScanner.advance();
+ Cell current = cellScanner.current();
+ List<Tag> tags = TestCoprocessorForTags.tags.poll();
+ if (CellUtil.matchingRows(current, row)) {
+ assertEquals(1, tags.size());
+ Tag tag = tags.get(0);
+ assertEquals(bigTagLen, tag.getValueLength());
+ } else {
+ assertEquals(0, tags.size());
}
- TestCoprocessorForTags.checkTagPresence = false;
- }
- } finally {
- if (table != null) {
- table.close();
}
- // delete the table
- admin.disableTable(tableName);
- admin.deleteTable(tableName);
}
}
}
@Test
- public void testTagsWithAppendAndIncrement() throws Exception {
- TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
+ public void testTagsWithAppendAndIncrement(TestInfo testInfo) throws
Exception {
+ TableName tableName =
TableName.valueOf(testInfo.getTestMethod().get().getName());
byte[] f = Bytes.toBytes("f");
byte[] q = Bytes.toBytes("q");
byte[] row1 = Bytes.toBytes("r1");
byte[] row2 = Bytes.toBytes("r2");
- HTableDescriptor desc = new HTableDescriptor(tableName);
- HColumnDescriptor colDesc = new HColumnDescriptor(f);
- desc.addFamily(colDesc);
- TEST_UTIL.getAdmin().createTable(desc);
+ TableDescriptor tableDescriptor =
TableDescriptorBuilder.newBuilder(tableName)
+ .setColumnFamily(ColumnFamilyDescriptorBuilder.of(f)).build();
- Table table = null;
- try {
- table = TEST_UTIL.getConnection().getTable(tableName);
+ try (Table table = TEST_UTIL.createTable(tableDescriptor, null)) {
Put put = new Put(row1);
byte[] v = Bytes.toBytes(2L);
put.addColumn(f, q, v);
@@ -491,36 +314,38 @@ public class TestTags {
increment.addColumn(f, q, 1L);
table.increment(increment);
TestCoprocessorForTags.checkTagPresence = true;
- ResultScanner scanner = table.getScanner(new Scan());
- Result result = scanner.next();
- KeyValue kv = KeyValueUtil.ensureKeyValue(result.getColumnLatestCell(f,
q));
- List<Tag> tags = TestCoprocessorForTags.tags;
- assertEquals(3L, Bytes.toLong(kv.getValueArray(), kv.getValueOffset(),
kv.getValueLength()));
- assertEquals(1, tags.size());
- assertEquals("tag1", Bytes.toString(Tag.cloneValue(tags.get(0))));
- TestCoprocessorForTags.checkTagPresence = false;
- TestCoprocessorForTags.tags = null;
+ try (ResultScanner scanner = table.getScanner(new Scan())) {
+ Result result = scanner.next();
+ Cell kv = result.getColumnLatestCell(f, q);
+ List<Tag> tags = TestCoprocessorForTags.tags.poll();
+ assertEquals(3L,
+ Bytes.toLong(kv.getValueArray(), kv.getValueOffset(),
kv.getValueLength()));
+ assertEquals(1, tags.size());
+ assertEquals("tag1", Bytes.toString(Tag.cloneValue(tags.get(0))));
+ }
+ resetCoprocessorFlags();
increment = new Increment(row1);
increment.add(new KeyValue(row1, f, q, 1234L, v));
increment.setAttribute("visibility", Bytes.toBytes("tag2"));
table.increment(increment);
TestCoprocessorForTags.checkTagPresence = true;
- scanner = table.getScanner(new Scan());
- result = scanner.next();
- kv = KeyValueUtil.ensureKeyValue(result.getColumnLatestCell(f, q));
- tags = TestCoprocessorForTags.tags;
- assertEquals(5L, Bytes.toLong(kv.getValueArray(), kv.getValueOffset(),
kv.getValueLength()));
- assertEquals(2, tags.size());
- // We cannot assume the ordering of tags
- List<String> tagValues = new ArrayList<>();
- for (Tag tag : tags) {
- tagValues.add(Bytes.toString(Tag.cloneValue(tag)));
+ try (ResultScanner scanner = table.getScanner(new Scan())) {
+ Result result = scanner.next();
+ Cell kv = result.getColumnLatestCell(f, q);
+ List<Tag> tags = TestCoprocessorForTags.tags.poll();
+ assertEquals(5L,
+ Bytes.toLong(kv.getValueArray(), kv.getValueOffset(),
kv.getValueLength()));
+ assertEquals(2, tags.size());
+ // We cannot assume the ordering of tags
+ List<String> tagValues = new ArrayList<>();
+ for (Tag tag : tags) {
+ tagValues.add(Bytes.toString(Tag.cloneValue(tag)));
+ }
+ assertTrue(tagValues.contains("tag1"));
+ assertTrue(tagValues.contains("tag2"));
}
- assertTrue(tagValues.contains("tag1"));
- assertTrue(tagValues.contains("tag2"));
- TestCoprocessorForTags.checkTagPresence = false;
- TestCoprocessorForTags.tags = null;
+ resetCoprocessorFlags();
put = new Put(row2);
v = Bytes.toBytes(2L);
@@ -531,15 +356,16 @@ public class TestTags {
increment.setAttribute("visibility", Bytes.toBytes("tag2"));
table.increment(increment);
TestCoprocessorForTags.checkTagPresence = true;
- scanner = table.getScanner(new Scan().setStartRow(row2));
- result = scanner.next();
- kv = KeyValueUtil.ensureKeyValue(result.getColumnLatestCell(f, q));
- tags = TestCoprocessorForTags.tags;
- assertEquals(4L, Bytes.toLong(kv.getValueArray(), kv.getValueOffset(),
kv.getValueLength()));
- assertEquals(1, tags.size());
- assertEquals("tag2", Bytes.toString(Tag.cloneValue(tags.get(0))));
- TestCoprocessorForTags.checkTagPresence = false;
- TestCoprocessorForTags.tags = null;
+ try (ResultScanner scanner = table.getScanner(new
Scan().withStartRow(row2))) {
+ Result result = scanner.next();
+ Cell kv = result.getColumnLatestCell(f, q);
+ List<Tag> tags = TestCoprocessorForTags.tags.poll();
+ assertEquals(4L,
+ Bytes.toLong(kv.getValueArray(), kv.getValueOffset(),
kv.getValueLength()));
+ assertEquals(1, tags.size());
+ assertEquals("tag2", Bytes.toString(Tag.cloneValue(tags.get(0))));
+ }
+ resetCoprocessorFlags();
// Test Append
byte[] row3 = Bytes.toBytes("r3");
@@ -551,34 +377,32 @@ public class TestTags {
append.addColumn(f, q, Bytes.toBytes("b"));
table.append(append);
TestCoprocessorForTags.checkTagPresence = true;
- scanner = table.getScanner(new Scan().setStartRow(row3));
- result = scanner.next();
- kv = KeyValueUtil.ensureKeyValue(result.getColumnLatestCell(f, q));
- tags = TestCoprocessorForTags.tags;
- assertEquals(1, tags.size());
- assertEquals("tag1", Bytes.toString(Tag.cloneValue(tags.get(0))));
- TestCoprocessorForTags.checkTagPresence = false;
- TestCoprocessorForTags.tags = null;
+ try (ResultScanner scanner = table.getScanner(new
Scan().withStartRow(row3))) {
+ assertNotNull(scanner.next());
+ List<Tag> tags = TestCoprocessorForTags.tags.poll();
+ assertEquals(1, tags.size());
+ assertEquals("tag1", Bytes.toString(Tag.cloneValue(tags.get(0))));
+ }
+ resetCoprocessorFlags();
append = new Append(row3);
append.add(new KeyValue(row3, f, q, 1234L, v));
append.setAttribute("visibility", Bytes.toBytes("tag2"));
table.append(append);
TestCoprocessorForTags.checkTagPresence = true;
- scanner = table.getScanner(new Scan().setStartRow(row3));
- result = scanner.next();
- kv = KeyValueUtil.ensureKeyValue(result.getColumnLatestCell(f, q));
- tags = TestCoprocessorForTags.tags;
- assertEquals(2, tags.size());
- // We cannot assume the ordering of tags
- tagValues.clear();
- for (Tag tag : tags) {
- tagValues.add(Bytes.toString(Tag.cloneValue(tag)));
+ try (ResultScanner scanner = table.getScanner(new
Scan().withStartRow(row3))) {
+ assertNotNull(scanner.next());
+ List<Tag> tags = TestCoprocessorForTags.tags.poll();
+ assertEquals(2, tags.size());
+ // We cannot assume the ordering of tags
+ List<String> tagValues = new ArrayList<>();
+ for (Tag tag : tags) {
+ tagValues.add(Bytes.toString(Tag.cloneValue(tag)));
+ }
+ assertTrue(tagValues.contains("tag1"));
+ assertTrue(tagValues.contains("tag2"));
}
- assertTrue(tagValues.contains("tag1"));
- assertTrue(tagValues.contains("tag2"));
- TestCoprocessorForTags.checkTagPresence = false;
- TestCoprocessorForTags.tags = null;
+ resetCoprocessorFlags();
byte[] row4 = Bytes.toBytes("r4");
put = new Put(row4);
@@ -589,31 +413,23 @@ public class TestTags {
append.setAttribute("visibility", Bytes.toBytes("tag2"));
table.append(append);
TestCoprocessorForTags.checkTagPresence = true;
- scanner = table.getScanner(new Scan().setStartRow(row4));
- result = scanner.next();
- kv = KeyValueUtil.ensureKeyValue(result.getColumnLatestCell(f, q));
- tags = TestCoprocessorForTags.tags;
- assertEquals(1, tags.size());
- assertEquals("tag2", Bytes.toString(Tag.cloneValue(tags.get(0))));
- } finally {
- TestCoprocessorForTags.checkTagPresence = false;
- TestCoprocessorForTags.tags = null;
- if (table != null) {
- table.close();
+ try (ResultScanner scanner = table.getScanner(new
Scan().withStartRow(row4))) {
+ assertNotNull(scanner.next());
+ List<Tag> tags = TestCoprocessorForTags.tags.poll();
+ assertEquals(1, tags.size());
+ assertEquals("tag2", Bytes.toString(Tag.cloneValue(tags.get(0))));
}
}
}
- private void result(byte[] fam, byte[] row, byte[] qual, byte[] row2, Table
table, byte[] value,
- byte[] value2, byte[] row1, byte[] value1) throws IOException {
- Scan s = new Scan(row);
+ private void assertResult(byte[] fam, byte[] row, byte[] qual, byte[] row2,
Table table,
+ byte[] value, byte[] value2, byte[] row1, byte[] value1) throws
IOException {
+ Scan s = new Scan().withStartRow(row);
// If filters are used this attribute can be specifically check for in
// filterKV method and
// kvs can be filtered out if the tags of interest is not found in that kv
s.setAttribute("visibility", Bytes.toBytes("myTag"));
- ResultScanner scanner = null;
- try {
- scanner = table.getScanner(s);
+ try (ResultScanner scanner = table.getScanner(s)) {
Result next = scanner.next();
assertTrue(Bytes.equals(next.getRow(), row));
@@ -628,16 +444,13 @@ public class TestTags {
assertTrue(next2 != null);
assertTrue(Bytes.equals(next2.getRow(), row2));
assertTrue(Bytes.equals(next2.getValue(fam, qual), value2));
-
- } finally {
- if (scanner != null) scanner.close();
}
}
public static class TestCoprocessorForTags implements RegionCoprocessor,
RegionObserver {
public static volatile boolean checkTagPresence = false;
- public static List<Tag> tags = null;
+ public static final ConcurrentLinkedQueue<List<Tag>> tags = new
ConcurrentLinkedQueue<>();
@Override
public Optional<RegionObserver> getRegionObserver() {
@@ -703,7 +516,7 @@ public class TestTags {
CellScanner cellScanner = result.cellScanner();
if (cellScanner.advance()) {
Cell cell = cellScanner.current();
- tags = PrivateCellUtil.getTags(cell);
+ tags.add(PrivateCellUtil.getTags(cell));
}
}
}
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTagsReverseScan.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTagsReverseScan.java
new file mode 100644
index 00000000000..8fca5d28ce9
--- /dev/null
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTagsReverseScan.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.stream.Stream;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseParameterizedTestTemplate;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.TestInfo;
+import org.junit.jupiter.api.TestTemplate;
+import org.junit.jupiter.params.provider.Arguments;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Tag(RegionServerTests.TAG)
+@Tag(LargeTests.TAG)
+@HBaseParameterizedTestTemplate(name = "{index}: DataBlockEncoding = {0}")
+public class TestTagsReverseScan {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(TestTagsReverseScan.class);
+
+ private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
+
+ private DataBlockEncoding encoding;
+
+ public TestTagsReverseScan(DataBlockEncoding encoding) {
+ this.encoding = encoding;
+ }
+
+ public static Stream<Arguments> parameters() {
+ return Arrays.stream(DataBlockEncoding.values()).map(Arguments::of);
+ }
+
+ @BeforeAll
+ public static void setUpBeforeClass() throws Exception {
+ UTIL.startMiniCluster(2);
+ }
+
+ @AfterAll
+ public static void tearDownAfterClass() throws Exception {
+ UTIL.shutdownMiniCluster();
+ }
+
+ /**
+ * Test that we can do reverse scans when writing tags and using
DataBlockEncoding. Fails with an
+ * exception for PREFIX, DIFF, and FAST_DIFF prior to HBASE-27580
+ */
+ @TestTemplate
+ public void testReverseScanWithDBE(TestInfo testInfo) throws IOException {
+ byte[] family = Bytes.toBytes("0");
+
+ Configuration conf = new Configuration(UTIL.getConfiguration());
+ conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
+
+ try (Connection connection = ConnectionFactory.createConnection(conf)) {
+ testReverseScanWithDBE(testInfo, connection, family,
HConstants.DEFAULT_BLOCKSIZE, 10);
+ }
+ }
+
+ /**
+ * Test that we can do reverse scans when writing tags and using
DataBlockEncoding. Fails with an
+ * exception for PREFIX, DIFF, and FAST_DIFF
+ */
+ @TestTemplate
+ public void testReverseScanWithDBEWhenCurrentBlockUpdates(TestInfo testInfo)
throws IOException {
+ byte[] family = Bytes.toBytes("0");
+
+ Configuration conf = new Configuration(UTIL.getConfiguration());
+ conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
+
+ try (Connection connection = ConnectionFactory.createConnection(conf)) {
+ testReverseScanWithDBE(testInfo, connection, family, 1024, 30000);
+ }
+ }
+
+ private void testReverseScanWithDBE(TestInfo testInfo, Connection conn,
byte[] family,
+ int blockSize, int maxRows) throws IOException {
+ LOG.info("Running test with DBE={}", encoding);
+ TableName tableName =
+ TableName.valueOf(testInfo.getTestMethod().get().getName() + "-" +
encoding);
+ UTIL.createTable(
+
TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder
+
.newBuilder(family).setDataBlockEncoding(encoding).setBlocksize(blockSize).build()).build(),
+ null);
+
+ Table table = conn.getTable(tableName);
+
+ byte[] val1 = new byte[10];
+ byte[] val2 = new byte[10];
+ Bytes.random(val1);
+ Bytes.random(val2);
+
+ for (int i = 0; i < maxRows; i++) {
+ table.put(new Put(Bytes.toBytes(i)).addColumn(family, Bytes.toBytes(1),
val1)
+ .addColumn(family, Bytes.toBytes(2), val2).setTTL(600_000));
+ }
+
+ UTIL.flush(table.getName());
+
+ Scan scan = new Scan();
+ scan.setReversed(true);
+
+ try (ResultScanner scanner = table.getScanner(scan)) {
+ for (int i = maxRows - 1; i >= 0; i--) {
+ Result row = scanner.next();
+ assertEquals(2, row.size());
+
+ Cell cell1 = row.getColumnLatestCell(family, Bytes.toBytes(1));
+ assertTrue(CellUtil.matchingRows(cell1, Bytes.toBytes(i)));
+ assertTrue(CellUtil.matchingValue(cell1, val1));
+
+ Cell cell2 = row.getColumnLatestCell(family, Bytes.toBytes(2));
+ assertTrue(CellUtil.matchingRows(cell2, Bytes.toBytes(i)));
+ assertTrue(CellUtil.matchingValue(cell2, val2));
+ }
+ }
+ }
+}
