Repository: hbase Updated Branches: refs/heads/master 423a0ab71 -> 9e9db3245
HBASE-20630 B&R: Delete command enhancements Signed-off-by: tedyu <yuzhih...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9e9db324 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9e9db324 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9e9db324 Branch: refs/heads/master Commit: 9e9db3245f2d825837f231c004eac99f83829a95 Parents: 423a0ab Author: Vladimir Rodionov <vrodio...@hortonworks.com> Authored: Wed Jun 6 16:49:15 2018 -0700 Committer: tedyu <yuzhih...@gmail.com> Committed: Wed Jun 13 18:46:48 2018 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/backup/BackupDriver.java | 9 ++- .../hbase/backup/BackupRestoreConstants.java | 7 +- .../hbase/backup/impl/BackupCommands.java | 85 +++++++++++++++++--- .../hadoop/hbase/backup/TestBackupDelete.java | 58 ++++++++++++- 4 files changed, 146 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/9e9db324/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java index 8baf2f0..6644d89 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupDriver.java @@ -17,10 +17,14 @@ */ package org.apache.hadoop.hbase.backup; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BACKUP_LIST_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_LIST; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER; @@ -46,14 +50,13 @@ import org.apache.hadoop.hbase.backup.impl.BackupManager; import org.apache.hadoop.hbase.util.AbstractHBaseTool; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.util.ToolRunner; +import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; import org.apache.log4j.Level; import org.apache.log4j.LogManager; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; - /** * * Command-line entry point for backup operation @@ -152,10 +155,12 @@ public class BackupDriver extends AbstractHBaseTool { addOptNoArg(OPTION_DEBUG, OPTION_DEBUG_DESC); addOptWithArg(OPTION_TABLE, OPTION_TABLE_DESC); addOptWithArg(OPTION_BANDWIDTH, OPTION_BANDWIDTH_DESC); + addOptWithArg(OPTION_LIST, OPTION_BACKUP_LIST_DESC); addOptWithArg(OPTION_WORKERS, OPTION_WORKERS_DESC); addOptWithArg(OPTION_RECORD_NUMBER, OPTION_RECORD_NUMBER_DESC); addOptWithArg(OPTION_SET, OPTION_SET_DESC); addOptWithArg(OPTION_PATH, OPTION_PATH_DESC); + addOptWithArg(OPTION_KEEP, OPTION_KEEP_DESC); addOptWithArg(OPTION_YARN_QUEUE_NAME, OPTION_YARN_QUEUE_NAME_DESC); } http://git-wip-us.apache.org/repos/asf/hbase/blob/9e9db324/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java index eaeef22..16ec3d2 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java @@ -65,8 +65,9 @@ public interface BackupRestoreConstants { String OPTION_TABLE_DESC = "Table name. If specified, only backup images," + " which contain this table will be listed."; - String OPTION_TABLE_LIST = "l"; + String OPTION_LIST = "l"; String OPTION_TABLE_LIST_DESC = "Table name list, comma-separated."; + String OPTION_BACKUP_LIST_DESC = "Backup ids list, comma-separated."; String OPTION_BANDWIDTH = "b"; String OPTION_BANDWIDTH_DESC = "Bandwidth per task (MapReduce task) in MB/s"; @@ -80,6 +81,10 @@ public interface BackupRestoreConstants { String OPTION_PATH = "p"; String OPTION_PATH_DESC = "Backup destination root directory path"; + String OPTION_KEEP = "k"; + String OPTION_KEEP_DESC = "Specifies maximum age of backup (in days) to keep during bulk delete"; + + String OPTION_TABLE_MAPPING = "m"; String OPTION_TABLE_MAPPING_DESC = "A comma separated list of target tables. " http://git-wip-us.apache.org/repos/asf/hbase/blob/9e9db324/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java index 3e28f51..fc2f1a2 100644 --- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java +++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java @@ -18,10 +18,14 @@ package org.apache.hadoop.hbase.backup.impl; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BACKUP_LIST_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_KEEP_DESC; +import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_LIST; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC; import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER; @@ -61,11 +65,11 @@ import org.apache.hadoop.hbase.backup.util.BackupUtils; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; -import org.apache.yetus.audience.InterfaceAudience; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine; import org.apache.hbase.thirdparty.org.apache.commons.cli.HelpFormatter; import org.apache.hbase.thirdparty.org.apache.commons.cli.Options; +import org.apache.yetus.audience.InterfaceAudience; /** * General backup commands, options and usage messages @@ -105,8 +109,7 @@ public final class BackupCommands { public static final String HISTORY_CMD_USAGE = "Usage: hbase backup history [options]"; - public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete <backup_id>\n" - + " backup_id Backup image id\n"; + public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete [options]"; public static final String REPAIR_CMD_USAGE = "Usage: hbase backup repair\n"; @@ -560,29 +563,93 @@ public final class BackupCommands { @Override public void execute() throws IOException { - if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { + + if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 1) { printUsage(); throw new IOException(INCORRECT_USAGE); } + if (!cmdline.hasOption(OPTION_KEEP) && !cmdline.hasOption(OPTION_LIST)) { + printUsage(); + throw new IOException(INCORRECT_USAGE); + } super.execute(); + if (cmdline.hasOption(OPTION_KEEP)) { + executeDeleteOlderThan(cmdline); + } else if (cmdline.hasOption(OPTION_LIST)) { + executeDeleteListOfBackups(cmdline); + } + } + + private void executeDeleteOlderThan(CommandLine cmdline) throws IOException { + String value = cmdline.getOptionValue(OPTION_KEEP); + int days = 0; + try { + days = Integer.parseInt(value); + } catch (NumberFormatException e) { + throw new IOException(value + " is not an integer number"); + } + final long fdays = days; + BackupInfo.Filter dateFilter = new BackupInfo.Filter() { + @Override + public boolean apply(BackupInfo info) { + long currentTime = EnvironmentEdgeManager.currentTime(); + long maxTsToDelete = currentTime - fdays * 24 * 3600 * 1000; + return info.getCompleteTs() <= maxTsToDelete; + } + }; + List<BackupInfo> history = null; + try (final BackupSystemTable sysTable = new BackupSystemTable(conn); + BackupAdminImpl admin = new BackupAdminImpl(conn)) { + history = sysTable.getBackupHistory(-1, dateFilter); + String[] backupIds = convertToBackupIds(history); + int deleted = admin.deleteBackups(backupIds); + System.out.println("Deleted " + deleted + " backups. Total older than " + days + " days: " + + backupIds.length); + } catch (IOException e) { + System.err.println("Delete command FAILED. Please run backup repair tool to restore backup " + + "system integrity"); + throw e; + } + } + + private String[] convertToBackupIds(List<BackupInfo> history) { + String[] ids = new String[history.size()]; + for (int i = 0; i < ids.length; i++) { + ids[i] = history.get(i).getBackupId(); + } + return ids; + } + + private void executeDeleteListOfBackups(CommandLine cmdline) throws IOException { + String value = cmdline.getOptionValue(OPTION_LIST); + String[] backupIds = value.split(","); - String[] args = cmdline.getArgs(); - String[] backupIds = new String[args.length - 1]; - System.arraycopy(args, 1, backupIds, 0, backupIds.length); try (BackupAdminImpl admin = new BackupAdminImpl(conn)) { int deleted = admin.deleteBackups(backupIds); - System.out.println("Deleted " + deleted + " backups. Total requested: " + (args.length -1)); + System.out.println("Deleted " + deleted + " backups. Total requested: " + backupIds.length); } catch (IOException e) { System.err.println("Delete command FAILED. Please run backup repair tool to restore backup " - + "system integrity"); + + "system integrity"); throw e; } + } @Override protected void printUsage() { System.out.println(DELETE_CMD_USAGE); + Options options = new Options(); + options.addOption(OPTION_KEEP, true, OPTION_KEEP_DESC); + options.addOption(OPTION_LIST, true, OPTION_BACKUP_LIST_DESC); + + HelpFormatter helpFormatter = new HelpFormatter(); + helpFormatter.setLeftPadding(2); + helpFormatter.setDescPadding(8); + helpFormatter.setWidth(100); + helpFormatter.setSyntaxPrefix("Options:"); + helpFormatter.printHelp(" ", null, options, USAGE_FOOTER); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/9e9db324/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java ---------------------------------------------------------------------- diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java index 9e67ff7..bc8b346 100644 --- a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java +++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestBackupDelete.java @@ -28,7 +28,10 @@ import org.apache.hadoop.hbase.HBaseClassTestRule; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.backup.impl.BackupSystemTable; import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.EnvironmentEdge; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.util.ToolRunner; +import org.junit.Assert; import org.junit.ClassRule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -90,7 +93,7 @@ public class TestBackupDelete extends TestBackupBase { ByteArrayOutputStream baos = new ByteArrayOutputStream(); System.setOut(new PrintStream(baos)); - String[] args = new String[] { "delete", backupId }; + String[] args = new String[] { "delete", "-l", backupId }; // Run backup try { @@ -104,4 +107,57 @@ public class TestBackupDelete extends TestBackupBase { LOG.info(baos.toString()); assertTrue(output.indexOf("Deleted 1 backups") >= 0); } + + @Test + public void testBackupPurgeOldBackupsCommand() throws Exception { + LOG.info("test backup delete (purge old backups) on a single table with data: command-line"); + List<TableName> tableList = Lists.newArrayList(table1); + EnvironmentEdgeManager.injectEdge(new EnvironmentEdge() { + // time - 2 days + @Override + public long currentTime() { + return System.currentTimeMillis() - 2 * 24 * 3600 * 1000 ; + } + }); + String backupId = fullTableBackup(tableList); + assertTrue(checkSucceeded(backupId)); + + EnvironmentEdgeManager.reset(); + + LOG.info("backup complete"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + System.setOut(new PrintStream(baos)); + + // Purge all backups which are older than 3 days + // Must return 0 (no backups were purged) + String[] args = new String[] { "delete", "-k", "3" }; + // Run backup + + try { + int ret = ToolRunner.run(conf1, new BackupDriver(), args); + assertTrue(ret == 0); + } catch (Exception e) { + LOG.error("failed", e); + Assert.fail(e.getMessage()); + } + String output = baos.toString(); + LOG.info(baos.toString()); + assertTrue(output.indexOf("Deleted 0 backups") >= 0); + + // Purge all backups which are older than 1 days + // Must return 1 deleted backup + args = new String[] { "delete", "-k", "1" }; + // Run backup + baos.reset(); + try { + int ret = ToolRunner.run(conf1, new BackupDriver(), args); + assertTrue(ret == 0); + } catch (Exception e) { + LOG.error("failed", e); + Assert.fail(e.getMessage()); + } + output = baos.toString(); + LOG.info(baos.toString()); + assertTrue(output.indexOf("Deleted 1 backups") >= 0); + } }
