Repository: hbase Updated Branches: refs/heads/master a42556553 -> f62c8201b
HBASE-21077 MR job launched by hbase incremental backup command failed with FileNotFoundException Signed-off-by: tedyu <yuzhih...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f62c8201 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f62c8201 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f62c8201 Branch: refs/heads/master Commit: f62c8201b62eb440ac0eeb7f61229ba0a343af6d Parents: a425565 Author: Vladimir Rodionov <vrodio...@hortonworks.com> Authored: Tue Aug 21 12:35:00 2018 -0700 Committer: tedyu <yuzhih...@gmail.com> Committed: Tue Aug 21 14:27:32 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/backup/impl/IncrementalBackupManager.java | 2 +- .../java/org/apache/hadoop/hbase/backup/TestBackupBase.java | 7 +++++++ .../org/apache/hadoop/hbase/backup/TestIncrementalBackup.java | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/f62c8201/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java index 81652d8..3eebf42 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java @@ -178,7 +178,7 @@ public class IncrementalBackupManager extends BackupManager { Set<String> set = new HashSet<>(); for (int i=0; i < logFromSystemTable.size(); i++) { WALItem item = logFromSystemTable.get(i); - set.add(item.walFile); + set.add((new Path(item.walFile)).getName()); } return set; } http://git-wip-us.apache.org/repos/asf/hbase/blob/f62c8201/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java index 08ecd63..94c2271 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java @@ -57,6 +57,8 @@ import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.master.cleaner.LogCleaner; +import org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner; import org.apache.hadoop.hbase.security.HadoopSecurityEnabledUserProviderForTesting; import org.apache.hadoop.hbase.security.UserProvider; import org.apache.hadoop.hbase.security.access.SecureTestUtil; @@ -288,6 +290,11 @@ public class TestBackupBase { BackupManager.decorateMasterConfiguration(conf1); BackupManager.decorateRegionServerConfiguration(conf1); conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); + // Set TTL for old WALs to 1 sec to enforce fast cleaning of an archived + // WAL files + conf1.setLong(TimeToLiveLogCleaner.TTL_CONF_KEY, 1000); + conf1.setLong(LogCleaner.OLD_WALS_CLEANER_THREAD_TIMEOUT_MSEC, 1000); + // Set MultiWAL (with 2 default WAL files per RS) conf1.set(WALFactory.WAL_PROVIDER, provider); TEST_UTIL.startMiniCluster(); http://git-wip-us.apache.org/repos/asf/hbase/blob/f62c8201/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java index b74f42f..48e2c5e 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java @@ -157,6 +157,8 @@ public class TestIncrementalBackup extends TestBackupBase { int NB_ROWS_FAM2 = 7; HTable t3 = insertIntoTable(conn, table1, fam2Name, 2, NB_ROWS_FAM2); t3.close(); + // Wait for 5 sec to make sure that old WALs were deleted + Thread.sleep(5000); // #3 - incremental backup for multiple tables request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);