This is an automated email from the ASF dual-hosted git repository.
taklwu pushed a commit to branch HBASE-28957
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/HBASE-28957 by this push:
new 7eb6a765376 [HBASE-29520] Utilize Backed-up Bulkloaded Files in
Incremental Backup (#7246)
7eb6a765376 is described below
commit 7eb6a7653768e217b84ff9781908c8dfa3b23650
Author: asolomon <[email protected]>
AuthorDate: Tue Sep 9 03:18:01 2025 +0530
[HBASE-29520] Utilize Backed-up Bulkloaded Files in Incremental Backup
(#7246)
Signed-off-by: Tak Lon (Stephen) Wu <[email protected]>
---
.../backup/impl/AbstractPitrRestoreHandler.java | 2 +-
.../hadoop/hbase/backup/impl/BackupCommands.java | 2 +-
.../backup/impl/IncrementalTableBackupClient.java | 23 +++++++-
.../ContinuousBackupReplicationEndpoint.java | 24 ++------
.../hadoop/hbase/backup/util/BackupUtils.java | 13 +++++
.../apache/hadoop/hbase/backup/TestBackupBase.java | 13 +++++
.../hbase/backup/TestBackupDeleteWithCleanup.java | 2 +-
.../hadoop/hbase/backup/TestContinuousBackup.java | 6 --
.../TestIncrementalBackupWithContinuous.java | 67 ++++++++++++----------
.../hbase/backup/impl/TestBackupCommands.java | 2 +-
.../TestContinuousBackupReplicationEndpoint.java | 5 +-
11 files changed, 95 insertions(+), 64 deletions(-)
diff --git
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/AbstractPitrRestoreHandler.java
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/AbstractPitrRestoreHandler.java
index 8072277bf68..048ed882fe8 100644
---
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/AbstractPitrRestoreHandler.java
+++
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/AbstractPitrRestoreHandler.java
@@ -21,8 +21,8 @@ import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONF_CONTINU
import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONF_CONTINUOUS_BACKUP_WAL_DIR;
import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.DEFAULT_CONTINUOUS_BACKUP_PITR_WINDOW_DAYS;
import static
org.apache.hadoop.hbase.backup.replication.BackupFileSystemManager.WALS_DIR;
-import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.DATE_FORMAT;
import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.ONE_DAY_IN_MILLISECONDS;
+import static org.apache.hadoop.hbase.backup.util.BackupUtils.DATE_FORMAT;
import static org.apache.hadoop.hbase.mapreduce.WALPlayer.IGNORE_EMPTY_FILES;
import java.io.IOException;
diff --git
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
index 0b02a5edd89..a30530a98fc 100644
---
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
+++
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
@@ -49,8 +49,8 @@ import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKE
import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC;
import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME;
import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC;
-import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.DATE_FORMAT;
import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.ONE_DAY_IN_MILLISECONDS;
+import static org.apache.hadoop.hbase.backup.util.BackupUtils.DATE_FORMAT;
import java.io.IOException;
import java.net.URI;
diff --git
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java
index 956d10a656f..57d81f8f12b 100644
---
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java
+++
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalTableBackupClient.java
@@ -19,9 +19,10 @@ package org.apache.hadoop.hbase.backup.impl;
import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONF_CONTINUOUS_BACKUP_WAL_DIR;
import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.JOB_NAME_CONF_KEY;
+import static
org.apache.hadoop.hbase.backup.replication.BackupFileSystemManager.BULKLOAD_FILES_DIR;
import static
org.apache.hadoop.hbase.backup.replication.BackupFileSystemManager.WALS_DIR;
-import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.DATE_FORMAT;
import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.ONE_DAY_IN_MILLISECONDS;
+import static org.apache.hadoop.hbase.backup.util.BackupUtils.DATE_FORMAT;
import java.io.IOException;
import java.net.URI;
@@ -166,6 +167,26 @@ public class IncrementalTableBackupClient extends
TableBackupClient {
Path tblDir = CommonFSUtils.getTableDir(rootdir, srcTable);
Path p = new Path(tblDir, regionName + Path.SEPARATOR + fam +
Path.SEPARATOR + filename);
+ // For continuous backup: bulkload files are copied from backup
directory defined by
+ // CONF_CONTINUOUS_BACKUP_WAL_DIR instead of source cluster.
+ String backupRootDir = conf.get(CONF_CONTINUOUS_BACKUP_WAL_DIR);
+ if (backupInfo.isContinuousBackupEnabled() &&
!Strings.isNullOrEmpty(backupRootDir)) {
+ String dayDirectoryName =
BackupUtils.formatToDateString(bulkLoad.getTimestamp());
+ Path bulkLoadBackupPath =
+ new Path(backupRootDir, BULKLOAD_FILES_DIR + Path.SEPARATOR +
dayDirectoryName);
+ Path bulkLoadDir = new Path(bulkLoadBackupPath,
+ srcTable.getNamespaceAsString() + Path.SEPARATOR +
srcTable.getNameAsString());
+ FileSystem backupFs = FileSystem.get(bulkLoadDir.toUri(), conf);
+ Path fullBulkLoadBackupPath =
+ new Path(bulkLoadDir, regionName + Path.SEPARATOR + fam +
Path.SEPARATOR + filename);
+ if (backupFs.exists(fullBulkLoadBackupPath)) {
+ LOG.debug("Backup bulkload file found {}", fullBulkLoadBackupPath);
+ p = fullBulkLoadBackupPath;
+ } else {
+ LOG.warn("Backup bulkload file not found {}",
fullBulkLoadBackupPath);
+ }
+ }
+
String srcTableQualifier = srcTable.getQualifierAsString();
String srcTableNs = srcTable.getNamespaceAsString();
Path tgtFam = new Path(tgtRoot, srcTableNs + Path.SEPARATOR +
srcTableQualifier
diff --git
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/replication/ContinuousBackupReplicationEndpoint.java
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/replication/ContinuousBackupReplicationEndpoint.java
index 69c445c484d..19624d04c23 100644
---
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/replication/ContinuousBackupReplicationEndpoint.java
+++
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/replication/ContinuousBackupReplicationEndpoint.java
@@ -21,11 +21,8 @@ import com.google.errorprone.annotations.RestrictedApi;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UncheckedIOException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
import java.util.List;
import java.util.Map;
-import java.util.TimeZone;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
@@ -41,6 +38,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
+import org.apache.hadoop.hbase.backup.util.BackupUtils;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.io.asyncfs.monitor.StreamSlowMonitor;
@@ -94,7 +92,6 @@ public class ContinuousBackupReplicationEndpoint extends
BaseReplicationEndpoint
public static final long ONE_DAY_IN_MILLISECONDS = TimeUnit.DAYS.toMillis(1);
public static final String WAL_FILE_PREFIX = "wal_file.";
- public static final String DATE_FORMAT = "yyyy-MM-dd";
@Override
public void init(Context context) throws IOException {
@@ -330,7 +327,7 @@ public class ContinuousBackupReplicationEndpoint extends
BaseReplicationEndpoint
}
private FSHLogProvider.Writer createWalWriter(long dayInMillis) {
- String dayDirectoryName = formatToDateString(dayInMillis);
+ String dayDirectoryName = BackupUtils.formatToDateString(dayInMillis);
FileSystem fs = backupFileSystemManager.getBackupFs();
Path walsDir = backupFileSystemManager.getWalsDir();
@@ -408,7 +405,7 @@ public class ContinuousBackupReplicationEndpoint extends
BaseReplicationEndpoint
LOG.trace("{} Bulk load files to upload: {}", Utils.logPeerId(peerId),
bulkLoadFiles.stream().map(Path::toString).collect(Collectors.joining(", ")));
}
- String dayDirectoryName = formatToDateString(dayInMillis);
+ String dayDirectoryName = BackupUtils.formatToDateString(dayInMillis);
Path bulkloadDir = new Path(backupFileSystemManager.getBulkLoadFilesDir(),
dayDirectoryName);
try {
backupFileSystemManager.getBackupFs().mkdirs(bulkloadDir);
@@ -446,7 +443,7 @@ public class ContinuousBackupReplicationEndpoint extends
BaseReplicationEndpoint
} catch (IOException e) {
throw new BulkLoadUploadException(
String.format("%s Failed to copy bulk load file %s to %s on day %s",
- Utils.logPeerId(peerId), file, destPath,
formatToDateString(dayInMillis)),
+ Utils.logPeerId(peerId), file, destPath,
BackupUtils.formatToDateString(dayInMillis)),
e);
}
}
@@ -495,19 +492,6 @@ public class ContinuousBackupReplicationEndpoint extends
BaseReplicationEndpoint
}
}
- /**
- * Convert dayInMillis to "yyyy-MM-dd" format
- */
- @RestrictedApi(
- explanation = "Package-private for test visibility only. Do not use
outside tests.",
- link = "",
- allowedOnPath =
"(.*/src/test/.*|.*/org/apache/hadoop/hbase/backup/replication/ContinuousBackupReplicationEndpoint.java)")
- String formatToDateString(long dayInMillis) {
- SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
- dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- return dateFormat.format(new Date(dayInMillis));
- }
-
private Path getBulkLoadFileStagingPath(Path relativePathFromNamespace)
throws IOException {
FileSystem rootFs = CommonFSUtils.getRootDirFileSystem(conf);
Path rootDir = CommonFSUtils.getRootDir(conf);
diff --git
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java
index 34fa82ef45f..bf309104775 100644
---
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java
+++
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java
@@ -24,14 +24,17 @@ import static
org.apache.hadoop.hbase.replication.regionserver.ReplicationMarker
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URLDecoder;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
+import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.TimeZone;
import java.util.TreeMap;
import java.util.TreeSet;
import org.apache.hadoop.conf.Configuration;
@@ -86,6 +89,7 @@ public final class BackupUtils {
private static final Logger LOG = LoggerFactory.getLogger(BackupUtils.class);
public static final String LOGNAME_SEPARATOR = ".";
public static final int MILLISEC_IN_HOUR = 3600000;
+ public static final String DATE_FORMAT = "yyyy-MM-dd";
private BackupUtils() {
throw new AssertionError("Instantiating utility class...");
@@ -932,4 +936,13 @@ public final class BackupUtils {
return admin.listReplicationPeers().stream()
.anyMatch(peer ->
peer.getPeerId().equals(CONTINUOUS_BACKUP_REPLICATION_PEER));
}
+
+ /**
+ * Convert dayInMillis to "yyyy-MM-dd" format
+ */
+ public static String formatToDateString(long dayInMillis) {
+ SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
+ dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+ return dateFormat.format(new Date(dayInMillis));
+ }
}
diff --git
a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
index 131b6377345..159514bd45b 100644
---
a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
+++
b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java
@@ -17,10 +17,13 @@
*/
package org.apache.hadoop.hbase.backup;
+import static
org.apache.hadoop.hbase.HConstants.REPLICATION_BULKLOAD_ENABLE_KEY;
+import static org.apache.hadoop.hbase.HConstants.REPLICATION_CLUSTER_ID;
import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONTINUOUS_BACKUP_REPLICATION_PEER;
import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.CONF_BACKUP_MAX_WAL_SIZE;
import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.CONF_STAGED_WAL_FLUSH_INITIAL_DELAY;
import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.CONF_STAGED_WAL_FLUSH_INTERVAL;
+import static org.apache.hadoop.hbase.mapreduce.WALPlayer.IGNORE_EMPTY_FILES;
import java.io.IOException;
import java.util.ArrayList;
@@ -46,6 +49,7 @@ import org.apache.hadoop.hbase.backup.BackupInfo.BackupPhase;
import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
import org.apache.hadoop.hbase.backup.impl.BackupManager;
+import org.apache.hadoop.hbase.backup.impl.BackupManifest;
import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
import org.apache.hadoop.hbase.backup.impl.FullTableBackupClient;
import org.apache.hadoop.hbase.backup.impl.IncrementalBackupManager;
@@ -304,6 +308,9 @@ public class TestBackupBase {
conf1.set(CONF_BACKUP_MAX_WAL_SIZE, "10240");
conf1.set(CONF_STAGED_WAL_FLUSH_INITIAL_DELAY, "10");
conf1.set(CONF_STAGED_WAL_FLUSH_INTERVAL, "10");
+ conf1.setBoolean(REPLICATION_BULKLOAD_ENABLE_KEY, true);
+ conf1.set(REPLICATION_CLUSTER_ID, "clusterId1");
+ conf1.setBoolean(IGNORE_EMPTY_FILES, true);
if (secure) {
// set the always on security provider
@@ -571,6 +578,12 @@ public class TestBackupBase {
}
}
+ BackupManifest getLatestBackupManifest(List<BackupInfo> backups) throws
IOException {
+ BackupInfo newestBackup = backups.get(0);
+ return HBackupFileSystem.getManifest(conf1, new Path(BACKUP_ROOT_DIR),
+ newestBackup.getBackupId());
+ }
+
void deleteContinuousBackupReplicationPeerIfExists(Admin admin) throws
IOException {
if (
admin.listReplicationPeers().stream()
diff --git
a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteWithCleanup.java
b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteWithCleanup.java
index 07c9110072b..8bd2fe4cc78 100644
---
a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteWithCleanup.java
+++
b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDeleteWithCleanup.java
@@ -21,8 +21,8 @@ import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONF_CONTINU
import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONTINUOUS_BACKUP_REPLICATION_PEER;
import static
org.apache.hadoop.hbase.backup.replication.BackupFileSystemManager.BULKLOAD_FILES_DIR;
import static
org.apache.hadoop.hbase.backup.replication.BackupFileSystemManager.WALS_DIR;
-import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.DATE_FORMAT;
import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.ONE_DAY_IN_MILLISECONDS;
+import static org.apache.hadoop.hbase.backup.util.BackupUtils.DATE_FORMAT;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
diff --git
a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestContinuousBackup.java
b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestContinuousBackup.java
index 0cc34ed63eb..2fdfa8b73f8 100644
---
a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestContinuousBackup.java
+++
b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestContinuousBackup.java
@@ -271,12 +271,6 @@ public class TestContinuousBackup extends TestBackupBase {
}
}
- BackupManifest getLatestBackupManifest(List<BackupInfo> backups) throws
IOException {
- BackupInfo newestBackup = backups.get(0);
- return HBackupFileSystem.getManifest(conf1, new Path(BACKUP_ROOT_DIR),
- newestBackup.getBackupId());
- }
-
private void verifyTableInBackupSystemTable(TableName table) throws
IOException {
try (BackupSystemTable backupTable = new
BackupSystemTable(TEST_UTIL.getConnection())) {
Map<TableName, Long> tableBackupMap =
backupTable.getContinuousBackupTableSet();
diff --git
a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithContinuous.java
b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithContinuous.java
index 0978ff3ebef..e67e50ebee3 100644
---
a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithContinuous.java
+++
b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupWithContinuous.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.backup;
+import static
org.apache.hadoop.hbase.backup.BackupRestoreConstants.CONF_CONTINUOUS_BACKUP_WAL_DIR;
import static
org.apache.hadoop.hbase.replication.regionserver.ReplicationMarkerChore.REPLICATION_MARKER_ENABLED_DEFAULT;
import static
org.apache.hadoop.hbase.replication.regionserver.ReplicationMarkerChore.REPLICATION_MARKER_ENABLED_KEY;
import static org.junit.Assert.assertEquals;
@@ -48,6 +49,8 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.HFileTestUtil;
import org.apache.hadoop.util.ToolRunner;
+import org.junit.After;
+import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -57,7 +60,7 @@ import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
@Category(LargeTests.class)
-public class TestIncrementalBackupWithContinuous extends TestContinuousBackup {
+public class TestIncrementalBackupWithContinuous extends TestBackupBase {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
@@ -67,11 +70,31 @@ public class TestIncrementalBackupWithContinuous extends
TestContinuousBackup {
LoggerFactory.getLogger(TestIncrementalBackupWithContinuous.class);
private static final int ROWS_IN_BULK_LOAD = 100;
+ private static final String backupWalDirName = "TestContinuousBackupWalDir";
+
+ @Before
+ public void beforeTest() throws IOException {
+ Path root = TEST_UTIL.getDataTestDirOnTestFS();
+ Path backupWalDir = new Path(root, backupWalDirName);
+ conf1.set(CONF_CONTINUOUS_BACKUP_WAL_DIR, backupWalDir.toString());
+ conf1.setBoolean(REPLICATION_MARKER_ENABLED_KEY, true);
+ }
+
+ @After
+ public void afterTest() throws IOException {
+ Path root = TEST_UTIL.getDataTestDirOnTestFS();
+ Path backupWalDir = new Path(root, backupWalDirName);
+ FileSystem fs = FileSystem.get(conf1);
+ if (fs.exists(backupWalDir)) {
+ fs.delete(backupWalDir, true);
+ }
+ conf1.unset(CONF_CONTINUOUS_BACKUP_WAL_DIR);
+ conf1.setBoolean(REPLICATION_MARKER_ENABLED_KEY,
REPLICATION_MARKER_ENABLED_DEFAULT);
+ deleteContinuousBackupReplicationPeerIfExists(TEST_UTIL.getAdmin());
+ }
@Test
public void testContinuousBackupWithIncrementalBackupSuccess() throws
Exception {
- LOG.info("Testing incremental backup with continuous backup");
- conf1.setBoolean(REPLICATION_MARKER_ENABLED_KEY, true);
String methodName =
Thread.currentThread().getStackTrace()[1].getMethodName();
TableName tableName = TableName.valueOf("table_" + methodName);
Table t1 = TEST_UTIL.createTable(tableName, famName);
@@ -80,18 +103,13 @@ public class TestIncrementalBackupWithContinuous extends
TestContinuousBackup {
int before = table.getBackupHistory().size();
// Run continuous backup
- String[] args = buildBackupArgs("full", new TableName[] { tableName },
true);
- int ret = ToolRunner.run(conf1, new BackupDriver(), args);
- assertEquals("Full Backup should succeed", 0, ret);
+ String backup1 = backupTables(BackupType.FULL, List.of(tableName),
BACKUP_ROOT_DIR, true);
+ assertTrue(checkSucceeded(backup1));
// Verify backup history increased and all the backups are succeeded
LOG.info("Verify backup history increased and all the backups are
succeeded");
List<BackupInfo> backups = table.getBackupHistory();
assertEquals("Backup history should increase", before + 1,
backups.size());
- for (BackupInfo data : List.of(backups.get(0))) {
- String backupId = data.getBackupId();
- assertTrue(checkSucceeded(backupId));
- }
// Verify backup manifest contains the correct tables
LOG.info("Verify backup manifest contains the correct tables");
@@ -105,42 +123,34 @@ public class TestIncrementalBackupWithContinuous extends
TestContinuousBackup {
// Run incremental backup
LOG.info("Run incremental backup now");
before = table.getBackupHistory().size();
- args = buildBackupArgs("incremental", new TableName[] { tableName },
false);
- ret = ToolRunner.run(conf1, new BackupDriver(), args);
- assertEquals("Incremental Backup should succeed", 0, ret);
+ String backup2 =
+ backupTables(BackupType.INCREMENTAL, List.of(tableName),
BACKUP_ROOT_DIR, true);
+ assertTrue(checkSucceeded(backup2));
LOG.info("Incremental backup completed");
// Verify backup history increased and all the backups are succeeded
backups = table.getBackupHistory();
- String incrementalBackupid = null;
assertEquals("Backup history should increase", before + 1,
backups.size());
- for (BackupInfo data : List.of(backups.get(0))) {
- String backupId = data.getBackupId();
- incrementalBackupid = backupId;
- assertTrue(checkSucceeded(backupId));
- }
TEST_UTIL.truncateTable(tableName);
+
// Restore incremental backup
TableName[] tables = new TableName[] { tableName };
BackupAdminImpl client = new BackupAdminImpl(TEST_UTIL.getConnection());
- client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR,
incrementalBackupid, false,
- tables, tables, true));
+ client.restore(
+ BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backup2, false,
tables, tables, true));
assertEquals(NB_ROWS_IN_BATCH, TEST_UTIL.countRows(tableName));
- } finally {
- conf1.setBoolean(REPLICATION_MARKER_ENABLED_KEY,
REPLICATION_MARKER_ENABLED_DEFAULT);
}
}
@Test
public void testIncrementalBackupCopyingBulkloadTillIncrCommittedWalTs()
throws Exception {
- conf1.setBoolean(REPLICATION_MARKER_ENABLED_KEY, true);
String methodName =
Thread.currentThread().getStackTrace()[1].getMethodName();
TableName tableName1 = TableName.valueOf("table_" + methodName);
TEST_UTIL.createTable(tableName1, famName);
- try (BackupSystemTable systemTable = new
BackupSystemTable(TEST_UTIL.getConnection())) {
+ try (BackupSystemTable systemTable = new
BackupSystemTable(TEST_UTIL.getConnection())) {
// The test starts with no data, and no bulk loaded rows.
int expectedRowCount = 0;
assertEquals(expectedRowCount, TEST_UTIL.countRows(tableName1));
@@ -157,7 +167,7 @@ public class TestIncrementalBackupWithContinuous extends
TestContinuousBackup {
assertEquals(expectedRowCount, TEST_UTIL.countRows(tableName1));
assertEquals(1,
systemTable.readBulkloadRows(List.of(tableName1)).size());
loadTable(TEST_UTIL.getConnection().getTable(tableName1));
- Thread.sleep(10000);
+ Thread.sleep(15000);
performBulkLoad("bulkPostIncr", methodName, tableName1);
assertEquals(2,
systemTable.readBulkloadRows(List.of(tableName1)).size());
@@ -177,14 +187,11 @@ public class TestIncrementalBackupWithContinuous extends
TestContinuousBackup {
client.restore(
BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backup2, false,
tables, tables, true));
assertEquals(expectedRowCount, TEST_UTIL.countRows(tableName1));
- } finally {
- conf1.setBoolean(REPLICATION_MARKER_ENABLED_KEY,
REPLICATION_MARKER_ENABLED_DEFAULT);
}
}
@Test
public void testPitrFailureDueToMissingBackupPostBulkload() throws Exception
{
- conf1.setBoolean(REPLICATION_MARKER_ENABLED_KEY, true);
String methodName =
Thread.currentThread().getStackTrace()[1].getMethodName();
TableName tableName1 = TableName.valueOf("table_" + methodName);
TEST_UTIL.createTable(tableName1, famName);
@@ -228,8 +235,6 @@ public class TestIncrementalBackupWithContinuous extends
TestContinuousBackup {
new TableName[] { restoredTable }, restoreTs, null);
int ret = ToolRunner.run(conf1, new PointInTimeRestoreDriver(), args);
assertNotEquals("Restore should fail since there is one bulkload without
any backup", 0, ret);
- } finally {
- conf1.setBoolean(REPLICATION_MARKER_ENABLED_KEY,
REPLICATION_MARKER_ENABLED_DEFAULT);
}
}
diff --git
a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/impl/TestBackupCommands.java
b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/impl/TestBackupCommands.java
index be14227e4cc..15ab2b2bdbe 100644
---
a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/impl/TestBackupCommands.java
+++
b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/impl/TestBackupCommands.java
@@ -22,8 +22,8 @@ import static
org.apache.hadoop.hbase.backup.TestBackupDeleteWithCleanup.logDire
import static
org.apache.hadoop.hbase.backup.TestBackupDeleteWithCleanup.setupBackupFolders;
import static
org.apache.hadoop.hbase.backup.replication.BackupFileSystemManager.BULKLOAD_FILES_DIR;
import static
org.apache.hadoop.hbase.backup.replication.BackupFileSystemManager.WALS_DIR;
-import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.DATE_FORMAT;
import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.ONE_DAY_IN_MILLISECONDS;
+import static org.apache.hadoop.hbase.backup.util.BackupUtils.DATE_FORMAT;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
diff --git
a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/replication/TestContinuousBackupReplicationEndpoint.java
b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/replication/TestContinuousBackupReplicationEndpoint.java
index 8f8e83dbda6..cc9200882e3 100644
---
a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/replication/TestContinuousBackupReplicationEndpoint.java
+++
b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/replication/TestContinuousBackupReplicationEndpoint.java
@@ -26,10 +26,10 @@ import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplica
import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.CONF_PEER_UUID;
import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.CONF_STAGED_WAL_FLUSH_INITIAL_DELAY;
import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.CONF_STAGED_WAL_FLUSH_INTERVAL;
-import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.DATE_FORMAT;
import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.ONE_DAY_IN_MILLISECONDS;
import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.WAL_FILE_PREFIX;
import static
org.apache.hadoop.hbase.backup.replication.ContinuousBackupReplicationEndpoint.copyWithCleanup;
+import static org.apache.hadoop.hbase.backup.util.BackupUtils.DATE_FORMAT;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -65,6 +65,7 @@ import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtil;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.util.BackupUtils;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
@@ -497,7 +498,7 @@ public class TestContinuousBackupReplicationEndpoint {
firstAttempt = false;
try {
// Construct destination path and create a partial file
- String dayDirectoryName = formatToDateString(dayInMillis);
+ String dayDirectoryName =
BackupUtils.formatToDateString(dayInMillis);
BackupFileSystemManager backupFileSystemManager =
new BackupFileSystemManager("peer1", conf,
conf.get(CONF_BACKUP_ROOT_DIR));
Path bulkloadDir =
