This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch HBASE-28463
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 9572f7d8d87fde9b61ac2c7b54d75a0b3328bf4c
Author: vinayak hegde <vinayakph...@gmail.com>
AuthorDate: Fri Apr 12 14:54:37 2024 +0530
HBASE-28505 Implement enforcement to require Date Tiered Compaction for
Time Range Data Tiering (#5809)
Signed-off-by: Wellington Chevreuil <wchevre...@apache.org>
---
.../hbase/regionserver/DateTieredStoreEngine.java | 3 ++
.../hadoop/hbase/util/TableDescriptorChecker.java | 36 +++++++++++++++++
.../hbase/client/TestIllegalTableDescriptor.java | 45 ++++++++++++++++++++++
3 files changed, 84 insertions(+)
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
index d15a6c92ef0..8fdbb6035ae 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java
@@ -41,6 +41,9 @@ import org.apache.yetus.audience.InterfaceAudience;
@InterfaceAudience.Private
public class DateTieredStoreEngine extends StoreEngine<DefaultStoreFlusher,
DateTieredCompactionPolicy, DateTieredCompactor, DefaultStoreFileManager> {
+
+ public static final String DATE_TIERED_STORE_ENGINE =
DateTieredStoreEngine.class.getName();
+
@Override
public boolean needsCompaction(List<HStoreFile> filesCompacting) {
return compactionPolicy.needsCompaction(storeFileManager.getStorefiles(),
filesCompacting);
diff --git
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java
index 94e2e4bbfa0..471583b32b7 100644
---
a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java
+++
b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/TableDescriptorChecker.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.hbase.util;
+import static
org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine.DATE_TIERED_STORE_ENGINE;
+
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CompoundConfiguration;
@@ -28,10 +30,13 @@ import
org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.fs.ErasureCodingUtils;
+import org.apache.hadoop.hbase.regionserver.DataTieringManager;
+import org.apache.hadoop.hbase.regionserver.DataTieringType;
import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
import org.apache.hadoop.hbase.regionserver.RegionSplitPolicy;
+import org.apache.hadoop.hbase.regionserver.StoreEngine;
import
org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy;
import org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy;
import org.apache.yetus.audience.InterfaceAudience;
@@ -191,6 +196,8 @@ public final class TableDescriptorChecker {
// check in-memory compaction
warnOrThrowExceptionForFailure(logWarn, hcd::getInMemoryCompaction);
+
+ checkDateTieredCompactionForTimeRangeDataTiering(conf, td);
}
}
@@ -210,6 +217,35 @@ public final class TableDescriptorChecker {
});
}
+ private static void checkDateTieredCompactionForTimeRangeDataTiering(final
Configuration conf,
+ final TableDescriptor td) throws IOException {
+ // Table level configurations
+ checkDateTieredCompactionForTimeRangeDataTiering(conf);
+ for (ColumnFamilyDescriptor cfd : td.getColumnFamilies()) {
+ // Column family level configurations
+ Configuration cfdConf =
+ new
CompoundConfiguration().add(conf).addStringMap(cfd.getConfiguration());
+ checkDateTieredCompactionForTimeRangeDataTiering(cfdConf);
+ }
+ }
+
+ private static void checkDateTieredCompactionForTimeRangeDataTiering(final
Configuration conf)
+ throws IOException {
+ final String errorMessage =
+ "Time Range Data Tiering should be enabled with Date Tiered Compaction.";
+
+ warnOrThrowExceptionForFailure(false, () -> {
+
+ // Determine whether Date Tiered Compaction will be enabled when Time
Range Data Tiering is
+ // enabled after the configuration change.
+ if
(DataTieringType.TIME_RANGE.name().equals(conf.get(DataTieringManager.DATATIERING_KEY)))
{
+ if
(!DATE_TIERED_STORE_ENGINE.equals(conf.get(StoreEngine.STORE_ENGINE_CLASS_KEY)))
{
+ throw new IllegalArgumentException(errorMessage);
+ }
+ }
+ });
+ }
+
private static void checkCompactionPolicy(final Configuration conf, final
TableDescriptor td)
throws IOException {
warnOrThrowExceptionForFailure(false, () -> {
diff --git
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java
index a67c43d31fa..666cfbdf54a 100644
---
a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java
+++
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIllegalTableDescriptor.java
@@ -31,6 +31,9 @@ import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.regionserver.DataTieringManager;
+import org.apache.hadoop.hbase.regionserver.DataTieringType;
+import org.apache.hadoop.hbase.regionserver.StoreEngine;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -187,6 +190,48 @@ public class TestIllegalTableDescriptor {
+ "cause very frequent flushing."));
}
+ @Test
+ public void testIllegalTableDescriptorWithDataTiering() throws IOException {
+ // table level configuration changes
+ TableDescriptorBuilder builder =
+
TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
+ ColumnFamilyDescriptorBuilder cfBuilder =
ColumnFamilyDescriptorBuilder.newBuilder(FAMILY);
+ builder.setColumnFamily(cfBuilder.build());
+
+ // First scenario: DataTieringType set to TIME_RANGE without
DateTieredStoreEngine
+ builder.setValue(DataTieringManager.DATATIERING_KEY,
DataTieringType.TIME_RANGE.name());
+ checkTableIsIllegal(builder.build());
+
+ // Second scenario: DataTieringType set to TIME_RANGE with
DateTieredStoreEngine
+ builder.setValue(StoreEngine.STORE_ENGINE_CLASS_KEY,
+ "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine");
+ checkTableIsLegal(builder.build());
+
+ // Third scenario: Disabling DateTieredStoreEngine while Time Range
DataTiering is active
+ builder.setValue(StoreEngine.STORE_ENGINE_CLASS_KEY,
+ "org.apache.hadoop.hbase.regionserver.DefaultStoreEngine");
+ checkTableIsIllegal(builder.build());
+
+ // column family level configuration changes
+ builder =
TableDescriptorBuilder.newBuilder(TableName.valueOf(name.getMethodName()));
+ cfBuilder = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY);
+
+ // First scenario: DataTieringType set to TIME_RANGE without
DateTieredStoreEngine
+ cfBuilder.setConfiguration(DataTieringManager.DATATIERING_KEY,
+ DataTieringType.TIME_RANGE.name());
+ checkTableIsIllegal(builder.setColumnFamily(cfBuilder.build()).build());
+
+ // Second scenario: DataTieringType set to TIME_RANGE with
DateTieredStoreEngine
+ cfBuilder.setConfiguration(StoreEngine.STORE_ENGINE_CLASS_KEY,
+ "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine");
+ checkTableIsLegal(builder.modifyColumnFamily(cfBuilder.build()).build());
+
+ // Third scenario: Disabling DateTieredStoreEngine while Time Range
DataTiering is active
+ cfBuilder.setConfiguration(StoreEngine.STORE_ENGINE_CLASS_KEY,
+ "org.apache.hadoop.hbase.regionserver.DefaultStoreEngine");
+ checkTableIsIllegal(builder.modifyColumnFamily(cfBuilder.build()).build());
+ }
+
private void checkTableIsLegal(TableDescriptor tableDescriptor) throws
IOException {
Admin admin = TEST_UTIL.getAdmin();
admin.createTable(tableDescriptor);
