This is an automated email from the ASF dual-hosted git repository.
taklwu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 96922fc01c5 HBASE-29984 Support separate old WAL directories in backup
(#8512)
96922fc01c5 is described below
commit 96922fc01c5c9ff712baf793eff7ad7674477635
Author: Ma Zhengxuan <[email protected]>
AuthorDate: Wed Aug 12 00:22:46 2026 +0800
HBASE-29984 Support separate old WAL directories in backup (#8512)
Signed-off-by: Tak Lon (Stephen) Wu <[email protected]>
Co-authored-by: mazhengxuan <[email protected]>
---
.../backup/impl/IncrementalBackupManager.java | 8 +-
.../hadoop/hbase/backup/util/BackupUtils.java | 4 +
.../hadoop/hbase/backup/TestBackupUtils.java | 5 +
.../hbase/backup/TestIncrementalBackupManager.java | 104 +++++++++++++++++++++
4 files changed, 117 insertions(+), 4 deletions(-)
diff --git
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
index 5f48bfc39ad..18be4c4f94a 100644
---
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
+++
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
@@ -184,10 +184,10 @@ public class IncrementalBackupManager extends
BackupManager {
}
// Include the .oldlogs files too.
- FileStatus[] oldlogs = fs.listStatus(oldLogDir);
- for (FileStatus oldlog : oldlogs) {
- p = oldlog.getPath();
- currentLogFile = p.toString();
+ List<String> oldlogs = BackupUtils.getFiles(fs, oldLogDir, new
ArrayList<>(), path -> true);
+ for (String oldlog : oldlogs) {
+ p = new Path(oldlog);
+ currentLogFile = oldlog;
if (AbstractFSWALProvider.isMetaFile(p)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skip .meta log file: " + currentLogFile);
diff --git
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java
index 138426d9079..b094b35e8e5 100644
---
a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java
+++
b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/util/BackupUtils.java
@@ -332,6 +332,10 @@ public final class BackupUtils {
if (p.getName().endsWith(MasterRegionFactory.ARCHIVED_WAL_SUFFIX)) {
return null;
}
+ Path parent = p.getParent();
+ if (parent != null && ServerName.isFullServerName(parent.getName())) {
+ return ServerName.valueOf(parent.getName()).getAddress().toString();
+ }
try {
String urlDecodedName = URLDecoder.decode(p.getName(), "UTF8");
Iterable<String> nameSplitsOnComma =
Splitter.on(",").split(urlDecodedName);
diff --git
a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupUtils.java
b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupUtils.java
index cf69476270c..07dec45b694 100644
---
a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupUtils.java
+++
b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupUtils.java
@@ -111,6 +111,11 @@ public class TestBackupUtils {
+ "regiongroup-0" + BackupUtils.LOGNAME_SEPARATOR +
EnvironmentEdgeManager.currentTime());
assertEquals(host + Addressing.HOSTNAME_PORT_SEPARATOR + port,
BackupUtils.parseHostFromOldLog(testOldWalWithRegionGroupingPath));
+
+ Path testOldWalInServerDirPath = new Path(new Path(oldLogDir,
serverName.toString()),
+ "wal" + BackupUtils.LOGNAME_SEPARATOR +
EnvironmentEdgeManager.currentTime());
+ assertEquals(host + Addressing.HOSTNAME_PORT_SEPARATOR + port,
+ BackupUtils.parseHostFromOldLog(testOldWalInServerDirPath));
}
}
diff --git
a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupManager.java
b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupManager.java
new file mode 100644
index 00000000000..6ea46e96202
--- /dev/null
+++
b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupManager.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
+import org.apache.hadoop.hbase.backup.impl.IncrementalBackupManager;
+import org.apache.hadoop.hbase.backup.util.BackupUtils;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+@Tag(LargeTests.TAG)
+public class TestIncrementalBackupManager extends TestBackupBase {
+
+ @BeforeAll
+ public static void setUp() throws Exception {
+ TEST_UTIL = new HBaseTestingUtil();
+ conf1 = TEST_UTIL.getConfiguration();
+ conf1.setBoolean(AbstractFSWALProvider.SEPARATE_OLDLOGDIR, true);
+ autoRestoreOnFailure = true;
+ useSecondCluster = false;
+ setUpHelper();
+ }
+
+ @Test
+ public void testCollectWALFilesFromRegionServerDirectories() throws
Exception {
+ testCollectWALFiles(true);
+ }
+
+ @Test
+ public void testCollectWALFilesFromFlatOldWALDirectory() throws Exception {
+ testCollectWALFiles(false);
+ }
+
+ private void testCollectWALFiles(boolean separateOldLogDir) throws Exception
{
+ Configuration testConf = new Configuration(conf1);
+ testConf.setBoolean(AbstractFSWALProvider.SEPARATE_OLDLOGDIR,
separateOldLogDir);
+ List<TableName> tables = List.of(table1);
+ try (Connection conn = ConnectionFactory.createConnection(testConf);
+ BackupAdminImpl backupAdmin = new BackupAdminImpl(conn)) {
+ String fullBackupId =
+ backupAdmin.backupTables(createBackupRequest(BackupType.FULL, tables,
BACKUP_ROOT_DIR));
+ assertTrue(checkSucceeded(fullBackupId));
+
+ try (IncrementalBackupManager manager = new
IncrementalBackupManager(conn, testConf)) {
+ BackupInfo backupInfo = manager.createBackupInfo("backup_test",
BackupType.INCREMENTAL,
+ tables, BACKUP_ROOT_DIR, -1, -1, false);
+ Map<String, Long> previousTimestamps =
+ BackupUtils.getRSLogTimestampMins(manager.readLogTimestampMap());
+ ServerName serverName =
TEST_UTIL.getMiniHBaseCluster().getRegionServer(0).getServerName();
+ Long previousTimestamp =
previousTimestamps.get(serverName.getAddress().toString());
+ assertNotNull(previousTimestamp);
+
+ TEST_UTIL.waitFor(30_000,
+ () -> EnvironmentEdgeManager.currentTime() > previousTimestamp + 1);
+ Path walRootDir = CommonFSUtils.getWALRootDir(conf1);
+ Path archiveDir = new Path(walRootDir,
+ AbstractFSWALProvider.getWALArchiveDirectoryName(testConf,
serverName.toString()));
+ String walName = (separateOldLogDir ? "wal" : serverName.toString())
+ + BackupUtils.LOGNAME_SEPARATOR + (previousTimestamp + 1);
+ Path archivedWAL = new Path(archiveDir, walName);
+ FileSystem fs = walRootDir.getFileSystem(conf1);
+ fs.mkdirs(archiveDir);
+ fs.create(archivedWAL).close();
+
+ manager.getIncrBackupLogFileMap();
+
+
assertTrue(backupInfo.getIncrBackupFileList().contains(archivedWAL.toString()));
+ }
+ }
+ }
+}
