http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java new file mode 100644 index 0000000..c4227f5 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java @@ -0,0 +1,754 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup.impl; + +import java.io.IOException; +import java.net.URI; +import java.util.List; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupInfo; +import org.apache.hadoop.hbase.backup.BackupRequest; +import org.apache.hadoop.hbase.backup.BackupRestoreConstants; +import org.apache.hadoop.hbase.backup.BackupType; +import org.apache.hadoop.hbase.backup.util.BackupClientUtil; +import org.apache.hadoop.hbase.backup.util.BackupSet; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; + +import com.google.common.collect.Lists; + +/** + * General backup commands, options and usage messages + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public final class BackupCommands implements BackupRestoreConstants { + + public final static String INCORRECT_USAGE = "Incorrect usage"; + + public static final String USAGE = + "Usage: bin/hbase backup COMMAND [command-specific arguments]\n" + + "where COMMAND is one of:\n" + + " create create a new backup image\n" + + " delete delete an existing backup image\n" + + " describe show the detailed information of a backup image\n" + + " history show history of all successful backups\n" + + " progress show the progress of the latest backup request\n" + + " set backup set management\n" + + "Run \'bin/hbase backup COMMAND -h\' to see help message for each command\n"; + + public static final String CREATE_CMD_USAGE = + "Usage: bin/hbase backup create <type> <backup_root> [tables] [options]\n" + + " type \"full\" to create a full backup image\n" + + " \"incremental\" to create an incremental backup image\n" + + " backup_root Full path to store the backup image\n" + + " tables If no tables (\"\") are specified, all tables are backed up.\n" + + " otherwise it is a comma separated list of tables."; + + + public static final String PROGRESS_CMD_USAGE = "Usage: bin/hbase backup progress <backupId>\n" + + " backupId Backup image id\n"; + public static final String NO_INFO_FOUND = "No info was found for backup id: "; + + public static final String DESCRIBE_CMD_USAGE = "Usage: bin/hbase backup describe <backupId>\n" + + " backupId Backup image id\n"; + + public static final String HISTORY_CMD_USAGE = + "Usage: bin/hbase backup history [options]"; + + + + public static final String DELETE_CMD_USAGE = "Usage: bin/hbase backup delete <backupId>\n" + + " backupId Backup image id\n"; + + public static final String CANCEL_CMD_USAGE = "Usage: bin/hbase backup cancel <backupId>\n" + + " backupId Backup image id\n"; + + public static final String SET_CMD_USAGE = "Usage: bin/hbase backup set COMMAND [name] [tables]\n" + + " name Backup set name\n" + + " tables If no tables (\"\") are specified, all tables will belong to the set.\n" + + " Otherwise it is a comma separated list of tables.\n" + + "COMMAND is one of:\n" + + " add add tables to a set, create a set if needed\n" + + " remove remove tables from a set\n" + + " list list all backup sets in the system\n" + + " describe describe set\n" + + " delete delete backup set\n"; + + public static final String USAGE_FOOTER = ""; + + public static abstract class Command extends Configured { + CommandLine cmdline; + + Command(Configuration conf) { + super(conf); + } + + public void execute() throws IOException + { + if (cmdline.hasOption("h") || cmdline.hasOption("help")) { + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + } + + protected abstract void printUsage(); + } + + private BackupCommands() { + throw new AssertionError("Instantiating utility class..."); + } + + public static Command createCommand(Configuration conf, BackupCommand type, CommandLine cmdline) { + Command cmd = null; + switch (type) { + case CREATE: + cmd = new CreateCommand(conf, cmdline); + break; + case DESCRIBE: + cmd = new DescribeCommand(conf, cmdline); + break; + case PROGRESS: + cmd = new ProgressCommand(conf, cmdline); + break; + case DELETE: + cmd = new DeleteCommand(conf, cmdline); + break; + case CANCEL: + cmd = new CancelCommand(conf, cmdline); + break; + case HISTORY: + cmd = new HistoryCommand(conf, cmdline); + break; + case SET: + cmd = new BackupSetCommand(conf, cmdline); + break; + case HELP: + default: + cmd = new HelpCommand(conf, cmdline); + break; + } + return cmd; + } + + static int numOfArgs(String[] args) { + if (args == null) return 0; + return args.length; + } + + public static class CreateCommand extends Command { + + CreateCommand(Configuration conf, CommandLine cmdline) { + super(conf); + this.cmdline = cmdline; + } + + @Override + public void execute() throws IOException { + super.execute(); + if (cmdline == null || cmdline.getArgs() == null) { + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + String[] args = cmdline.getArgs(); + if (args.length < 3 || args.length > 4) { + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + + if (!BackupType.FULL.toString().equalsIgnoreCase(args[1]) + && !BackupType.INCREMENTAL.toString().equalsIgnoreCase(args[1])) { + System.out.println("ERROR: invalid backup type: "+ args[1]); + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + if(!verifyPath(args[2])) { + System.out.println("ERROR: invalid backup destination: "+ args[2]); + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + + String tables = null; + Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); + + // Check if we have both: backup set and list of tables + if(args.length == 4 && cmdline.hasOption(OPTION_SET)) { + System.out.println("ERROR: You can specify either backup set or list"+ + " of tables, but not both"); + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + + // Check backup set + String setName = null; + if (cmdline.hasOption(OPTION_SET)) { + setName = cmdline.getOptionValue(OPTION_SET); + tables = getTablesForSet(setName, conf); + + if (tables == null) { + System.out.println("ERROR: Backup set '" + setName+ "' is either empty or does not exist"); + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + } else { + tables = (args.length == 4) ? args[3] : null; + } + int bandwidth = cmdline.hasOption(OPTION_BANDWIDTH) ? + Integer.parseInt(cmdline.getOptionValue(OPTION_BANDWIDTH)) : -1; + int workers = cmdline.hasOption(OPTION_WORKERS) ? + Integer.parseInt(cmdline.getOptionValue(OPTION_WORKERS)) : -1; + + try (Connection conn = ConnectionFactory.createConnection(getConf()); + HBaseBackupAdmin admin = new HBaseBackupAdmin(conn);) { + BackupRequest request = new BackupRequest(); + request.setBackupType(BackupType.valueOf(args[1].toUpperCase())) + .setTableList(tables != null?Lists.newArrayList(BackupClientUtil.parseTableNames(tables)): null) + .setTargetRootDir(args[2]).setWorkers(workers).setBandwidth(bandwidth) + .setBackupSetName(setName); + + String backupId = admin.backupTables(request); + System.out.println("Backup session "+ backupId+" finished. Status: SUCCESS"); + } catch (IOException e) { + System.out.println("Backup session finished. Status: FAILURE"); + throw e; + } + } + + private boolean verifyPath(String path) { + try{ + Path p = new Path(path); + Configuration conf = getConf() != null? getConf(): + HBaseConfiguration.create(); + URI uri = p.toUri(); + if(uri.getScheme() == null) return false; + FileSystem.get(uri, conf); + return true; + } catch(Exception e){ + return false; + } + } + + private String getTablesForSet(String name, Configuration conf) + throws IOException { + try (final Connection conn = ConnectionFactory.createConnection(conf); + final BackupSystemTable table = new BackupSystemTable(conn)) { + List<TableName> tables = table.describeBackupSet(name); + if (tables == null) return null; + return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); + } + } + + @Override + protected void printUsage() { + System.out.println(CREATE_CMD_USAGE); + Options options = new Options(); + options.addOption(OPTION_WORKERS, true, OPTION_WORKERS_DESC); + options.addOption(OPTION_BANDWIDTH, true, OPTION_BANDWIDTH_DESC); + options.addOption(OPTION_SET, true, OPTION_SET_RESTORE_DESC); + + HelpFormatter helpFormatter = new HelpFormatter(); + helpFormatter.setLeftPadding(2); + helpFormatter.setDescPadding(8); + helpFormatter.setWidth(100); + helpFormatter.setSyntaxPrefix("Options:"); + helpFormatter.printHelp(" ", null, options, USAGE_FOOTER); + + } + } + + private static class HelpCommand extends Command { + + HelpCommand(Configuration conf, CommandLine cmdline) { + super(conf); + this.cmdline = cmdline; + } + + @Override + public void execute() throws IOException { + super.execute(); + if (cmdline == null) { + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + + String[] args = cmdline.getArgs(); + if (args == null || args.length == 0) { + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + + if (args.length != 2) { + System.out.println("ERROR: Only supports help message of a single command type"); + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + + String type = args[1]; + + if (BackupCommand.CREATE.name().equalsIgnoreCase(type)) { + System.out.println(CREATE_CMD_USAGE); + } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(type)) { + System.out.println(DESCRIBE_CMD_USAGE); + } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(type)) { + System.out.println(HISTORY_CMD_USAGE); + } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(type)) { + System.out.println(PROGRESS_CMD_USAGE); + } else if (BackupCommand.DELETE.name().equalsIgnoreCase(type)) { + System.out.println(DELETE_CMD_USAGE); + } else if (BackupCommand.CANCEL.name().equalsIgnoreCase(type)) { + System.out.println(CANCEL_CMD_USAGE); + } else if (BackupCommand.SET.name().equalsIgnoreCase(type)) { + System.out.println(SET_CMD_USAGE); + } else { + System.out.println("Unknown command : " + type); + printUsage(); + } + } + + @Override + protected void printUsage() { + System.out.println(USAGE); + } + } + + private static class DescribeCommand extends Command { + + DescribeCommand(Configuration conf, CommandLine cmdline) { + super(conf); + this.cmdline = cmdline; + } + + @Override + public void execute() throws IOException { + super.execute(); + if (cmdline == null || cmdline.getArgs() == null) { + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + String[] args = cmdline.getArgs(); + if (args.length != 2) { + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + + String backupId = args[1]; + Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); + try (final Connection conn = ConnectionFactory.createConnection(conf); + final BackupSystemTable sysTable = new BackupSystemTable(conn);) { + BackupInfo info = sysTable.readBackupInfo(backupId); + if (info == null) { + System.out.println("ERROR: " + backupId + " does not exist"); + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + System.out.println(info.getShortDescription()); + } + } + + @Override + protected void printUsage() { + System.out.println(DESCRIBE_CMD_USAGE); + } + } + + private static class ProgressCommand extends Command { + + ProgressCommand(Configuration conf, CommandLine cmdline) { + super(conf); + this.cmdline = cmdline; + } + + @Override + public void execute() throws IOException { + super.execute(); + + if (cmdline == null || cmdline.getArgs() == null || + cmdline.getArgs().length == 1) { + System.err.println("No backup id was specified, " + + "will retrieve the most recent (ongoing) sessions"); + } + String[] args = cmdline == null ? null : cmdline.getArgs(); + if (args != null && args.length > 2) { + System.err.println("ERROR: wrong number of arguments: " + args.length); + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + + String backupId = (args == null || args.length <= 1) ? null : args[1]; + Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); + try(final Connection conn = ConnectionFactory.createConnection(conf); + final BackupSystemTable sysTable = new BackupSystemTable(conn);){ + BackupInfo info = sysTable.readBackupInfo(backupId); + int progress = info == null? -1: info.getProgress(); + if(progress < 0){ + System.out.println(NO_INFO_FOUND + backupId); + } else{ + System.out.println(backupId+" progress=" + progress+"%"); + } + } + } + + @Override + protected void printUsage() { + System.out.println(PROGRESS_CMD_USAGE); + } + } + + private static class DeleteCommand extends Command { + + DeleteCommand(Configuration conf, CommandLine cmdline) { + super(conf); + this.cmdline = cmdline; + } + + @Override + public void execute() throws IOException { + super.execute(); + if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + + String[] args = cmdline.getArgs(); + + String[] backupIds = new String[args.length - 1]; + System.arraycopy(args, 1, backupIds, 0, backupIds.length); + Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); + try (final Connection conn = ConnectionFactory.createConnection(conf); + HBaseBackupAdmin admin = new HBaseBackupAdmin(conn);) { + int deleted = admin.deleteBackups(args); + System.out.println("Deleted " + deleted + " backups. Total requested: " + args.length); + } + + } + + @Override + protected void printUsage() { + System.out.println(DELETE_CMD_USAGE); + } + } + +// TODO Cancel command + + private static class CancelCommand extends Command { + + CancelCommand(Configuration conf, CommandLine cmdline) { + super(conf); + this.cmdline = cmdline; + } + + @Override + public void execute() throws IOException { + super.execute(); + if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { + System.out.println("No backup id(s) was specified, will use the most recent one"); + } + Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); + try (final Connection conn = ConnectionFactory.createConnection(conf); + HBaseBackupAdmin admin = new HBaseBackupAdmin(conn);) { + // TODO cancel backup + } + } + + @Override + protected void printUsage() { + } + } + + private static class HistoryCommand extends Command { + + private final static int DEFAULT_HISTORY_LENGTH = 10; + + HistoryCommand(Configuration conf, CommandLine cmdline) { + super(conf); + this.cmdline = cmdline; + } + + @Override + public void execute() throws IOException { + + super.execute(); + + int n = parseHistoryLength(); + final TableName tableName = getTableName(); + final String setName = getTableSetName(); + BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() { + @Override + public boolean apply(BackupInfo info) { + if (tableName == null) return true; + List<TableName> names = info.getTableNames(); + return names.contains(tableName); + } + }; + BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() { + @Override + public boolean apply(BackupInfo info) { + if (setName == null) return true; + String backupId = info.getBackupId(); + return backupId.startsWith(setName); + } + }; + Path backupRootPath = getBackupRootPath(); + List<BackupInfo> history = null; + Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); + if (backupRootPath == null) { + // Load from hbase:backup + try (final Connection conn = ConnectionFactory.createConnection(conf); + final BackupSystemTable sysTable = new BackupSystemTable(conn);) { + + history = sysTable.getBackupHistory(n, tableNameFilter, tableSetFilter); + } + } else { + // load from backup FS + history = BackupClientUtil.getHistory(conf, n, backupRootPath, + tableNameFilter, tableSetFilter); + } + for (BackupInfo info : history) { + System.out.println(info.getShortDescription()); + } + } + + private Path getBackupRootPath() throws IOException { + String value = null; + try{ + value = cmdline.getOptionValue(OPTION_PATH); + if (value == null) return null; + return new Path(value); + } catch (IllegalArgumentException e) { + System.out.println("ERROR: Illegal argument for backup root path: "+ value); + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + } + + private TableName getTableName() throws IOException { + String value = cmdline.getOptionValue(OPTION_TABLE); + if (value == null) return null; + try{ + return TableName.valueOf(value); + } catch (IllegalArgumentException e){ + System.out.println("Illegal argument for table name: "+ value); + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + } + + private String getTableSetName() throws IOException { + String value = cmdline.getOptionValue(OPTION_SET); + return value; + } + + private int parseHistoryLength() throws IOException { + String value = cmdline.getOptionValue(OPTION_RECORD_NUMBER); + try{ + if (value == null) return DEFAULT_HISTORY_LENGTH; + return Integer.parseInt(value); + } catch(NumberFormatException e) { + System.out.println("Illegal argument for history length: "+ value); + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + } + + @Override + protected void printUsage() { + System.out.println(HISTORY_CMD_USAGE); + Options options = new Options(); + options.addOption(OPTION_RECORD_NUMBER, true, OPTION_RECORD_NUMBER_DESC); + options.addOption(OPTION_PATH, true, OPTION_PATH_DESC); + options.addOption(OPTION_TABLE, true, OPTION_TABLE_DESC); + options.addOption(OPTION_SET, true, OPTION_SET_DESC); + + HelpFormatter helpFormatter = new HelpFormatter(); + helpFormatter.setLeftPadding(2); + helpFormatter.setDescPadding(8); + helpFormatter.setWidth(100); + helpFormatter.setSyntaxPrefix("Options:"); + helpFormatter.printHelp(" ", null, options, USAGE_FOOTER); + } + } + + private static class BackupSetCommand extends Command { + private final static String SET_ADD_CMD = "add"; + private final static String SET_REMOVE_CMD = "remove"; + private final static String SET_DELETE_CMD = "delete"; + private final static String SET_DESCRIBE_CMD = "describe"; + private final static String SET_LIST_CMD = "list"; + + BackupSetCommand(Configuration conf, CommandLine cmdline) { + super(conf); + this.cmdline = cmdline; + } + + @Override + public void execute() throws IOException { + super.execute(); + // Command-line must have at least one element + if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + + String[] args = cmdline.getArgs(); + String cmdStr = args[1]; + BackupCommand cmd = getCommand(cmdStr); + + switch (cmd) { + case SET_ADD: + processSetAdd(args); + break; + case SET_REMOVE: + processSetRemove(args); + break; + case SET_DELETE: + processSetDelete(args); + break; + case SET_DESCRIBE: + processSetDescribe(args); + break; + case SET_LIST: + processSetList(args); + break; + default: + break; + + } + } + + private void processSetList(String[] args) throws IOException { + // List all backup set names + // does not expect any args + Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); + try(final Connection conn = ConnectionFactory.createConnection(conf); + HBaseBackupAdmin admin = new HBaseBackupAdmin(conn);){ + List<BackupSet> list = admin.listBackupSets(); + for(BackupSet bs: list){ + System.out.println(bs); + } + } + } + + private void processSetDescribe(String[] args) throws IOException { + if (args == null || args.length != 3) { + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + String setName = args[2]; + Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); + try(final Connection conn = ConnectionFactory.createConnection(conf); + final BackupSystemTable sysTable = new BackupSystemTable(conn);){ + List<TableName> tables = sysTable.describeBackupSet(setName); + BackupSet set = tables == null? null : new BackupSet(setName, tables); + if(set == null) { + System.out.println("Set '"+setName+"' does not exist."); + } else{ + System.out.println(set); + } + } + } + + private void processSetDelete(String[] args) throws IOException { + if (args == null || args.length != 3) { + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + String setName = args[2]; + Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); + try(final Connection conn = ConnectionFactory.createConnection(conf); + final HBaseBackupAdmin admin = new HBaseBackupAdmin(conn);){ + boolean result = admin.deleteBackupSet(setName); + if(result){ + System.out.println("Delete set "+setName+" OK."); + } else{ + System.out.println("Set "+setName+" does not exist"); + } + } + } + + private void processSetRemove(String[] args) throws IOException { + if (args == null || args.length != 4) { + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + + String setName = args[2]; + String[] tables = args[3].split(","); + Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); + try(final Connection conn = ConnectionFactory.createConnection(conf); + final HBaseBackupAdmin admin = new HBaseBackupAdmin(conn);){ + admin.removeFromBackupSet(setName, tables); + } + } + + private void processSetAdd(String[] args) throws IOException { + if (args == null || args.length != 4) { + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + String setName = args[2]; + String[] tables = args[3].split(","); + TableName[] tableNames = new TableName[tables.length]; + for(int i=0; i < tables.length; i++){ + tableNames[i] = TableName.valueOf(tables[i]); + } + Configuration conf = getConf() != null? getConf():HBaseConfiguration.create(); + try(final Connection conn = ConnectionFactory.createConnection(conf); + final HBaseBackupAdmin admin = new HBaseBackupAdmin(conn);){ + admin.addToBackupSet(setName, tableNames); + } + + } + + private BackupCommand getCommand(String cmdStr) throws IOException { + if (cmdStr.equals(SET_ADD_CMD)) { + return BackupCommand.SET_ADD; + } else if (cmdStr.equals(SET_REMOVE_CMD)) { + return BackupCommand.SET_REMOVE; + } else if (cmdStr.equals(SET_DELETE_CMD)) { + return BackupCommand.SET_DELETE; + } else if (cmdStr.equals(SET_DESCRIBE_CMD)) { + return BackupCommand.SET_DESCRIBE; + } else if (cmdStr.equals(SET_LIST_CMD)) { + return BackupCommand.SET_LIST; + } else { + System.out.println("ERROR: Unknown command for 'set' :" + cmdStr); + printUsage(); + throw new IOException(INCORRECT_USAGE); + } + } + + @Override + protected void printUsage() { + System.out.println(SET_CMD_USAGE); + } + + } +}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java new file mode 100644 index 0000000..ca204b4 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup.impl; + +import org.apache.hadoop.hbase.HBaseIOException; +import org.apache.hadoop.hbase.backup.BackupInfo; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; + +/** + * Backup exception + */ +@SuppressWarnings("serial") +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class BackupException extends HBaseIOException { + private BackupInfo description; + + /** + * Some exception happened for a backup and don't even know the backup that it was about + * @param msg Full description of the failure + */ + public BackupException(String msg) { + super(msg); + } + + /** + * Some exception happened for a backup with a cause + * @param cause the cause + */ + public BackupException(Throwable cause) { + super(cause); + } + + /** + * Exception for the given backup that has no previous root cause + * @param msg reason why the backup failed + * @param desc description of the backup that is being failed + */ + public BackupException(String msg, BackupInfo desc) { + super(msg); + this.description = desc; + } + + /** + * Exception for the given backup due to another exception + * @param msg reason why the backup failed + * @param cause root cause of the failure + * @param desc description of the backup that is being failed + */ + public BackupException(String msg, Throwable cause, BackupInfo desc) { + super(msg, cause); + this.description = desc; + } + + /** + * Exception when the description of the backup cannot be determined, due to some other root + * cause + * @param message description of what caused the failure + * @param e root cause + */ + public BackupException(String message, Exception e) { + super(message, e); + } + + public BackupInfo getBackupContext() { + return this.description; + } + +} http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java new file mode 100644 index 0000000..6fb7cfd --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java @@ -0,0 +1,502 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup.impl; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupInfo; +import org.apache.hadoop.hbase.backup.BackupInfo.BackupState; +import org.apache.hadoop.hbase.backup.BackupRestoreConstants; +import org.apache.hadoop.hbase.backup.BackupType; +import org.apache.hadoop.hbase.backup.HBackupFileSystem; +import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage; +import org.apache.hadoop.hbase.backup.master.BackupController; +import org.apache.hadoop.hbase.backup.master.BackupLogCleaner; +import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager; +import org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * Handles backup requests on server-side, creates backup context records in hbase:backup + * to keep track backup. The timestamps kept in hbase:backup table will be used for future + * incremental backup. Creates BackupContext and DispatchRequest. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class BackupManager implements Closeable { + private static final Log LOG = LogFactory.getLog(BackupManager.class); + + private Configuration conf = null; + private BackupInfo backupContext = null; + private ExecutorService pool = null; + private BackupSystemTable systemTable; + private final Connection conn; + + /** + * Backup manager constructor. + * @param conf configuration + * @throws IOException exception + */ + public BackupManager(Connection conn, Configuration conf) throws IOException { + if (!conf.getBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, + BackupRestoreConstants.BACKUP_ENABLE_DEFAULT)) { + throw new BackupException("HBase backup is not enabled. Check your " + + BackupRestoreConstants.BACKUP_ENABLE_KEY + " setting."); + } + this.conf = conf; + this.conn = conn; + this.systemTable = new BackupSystemTable(conn); + + } + + /** + * Return backup context + */ + protected BackupInfo getBackupContext() + { + return backupContext; + } + /** + * This method modifies the master's configuration in order to inject backup-related features + * @param conf configuration + */ + public static void decorateMasterConfiguration(Configuration conf) { + if (!isBackupEnabled(conf)) { + return; + } + // Add WAL archive cleaner plug-in + String plugins = conf.get(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS); + String cleanerClass = BackupLogCleaner.class.getCanonicalName(); + if (!plugins.contains(cleanerClass)) { + conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, plugins + "," + cleanerClass); + } + + String classes = conf.get("hbase.procedure.master.classes"); + String masterProcedureClass = LogRollMasterProcedureManager.class.getName(); + if(classes == null){ + conf.set("hbase.procedure.master.classes", masterProcedureClass); + } else if(!classes.contains(masterProcedureClass)){ + conf.set("hbase.procedure.master.classes", classes +","+masterProcedureClass); + } + + // Set Master Observer - Backup Controller + classes = conf.get("hbase.coprocessor.master.classes"); + String observerClass = BackupController.class.getName(); + if(classes == null){ + conf.set("hbase.coprocessor.master.classes", observerClass); + } else if(!classes.contains(observerClass)){ + conf.set("hbase.coprocessor.master.classes", classes +","+observerClass); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Added log cleaner: " + cleanerClass); + LOG.debug("Added master procedure manager: "+masterProcedureClass); + LOG.debug("Added master observer: "+observerClass); + } + + } + + /** + * This method modifies the RS configuration in order to inject backup-related features + * @param conf configuration + */ + public static void decorateRSConfiguration(Configuration conf) { + if (!isBackupEnabled(conf)) { + return; + } + + String classes = conf.get("hbase.procedure.regionserver.classes"); + String regionProcedureClass = LogRollRegionServerProcedureManager.class.getName(); + if(classes == null){ + conf.set("hbase.procedure.regionserver.classes", regionProcedureClass); + } else if(!classes.contains(regionProcedureClass)){ + conf.set("hbase.procedure.regionserver.classes", classes +","+regionProcedureClass); + } + if (LOG.isDebugEnabled()) { + LOG.debug("Added region procedure manager: "+regionProcedureClass); + } + + } + + + public static boolean isBackupEnabled(Configuration conf) { + return conf.getBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, + BackupRestoreConstants.BACKUP_ENABLE_DEFAULT); + } + + /** + * Get configuration + * @return configuration + */ + Configuration getConf() { + return conf; + } + + /** + * Stop all the work of backup. + */ + @Override + public void close() { + // currently, we shutdown now for all ongoing back handlers, we may need to do something like + // record the failed list somewhere later + if (this.pool != null) { + this.pool.shutdownNow(); + } + if (systemTable != null) { + try { + systemTable.close(); + } catch (Exception e) { + LOG.error(e); + } + } + if (conn != null) { + try { + conn.close(); + } catch (IOException e) { + LOG.error(e); + } + } + } + + /** + * Create a BackupContext based on input backup request. + * @param backupId backup id + * @param type type + * @param tableList table list + * @param targetRootDir root dir + * @param workers number of parallel workers + * @param bandwidth bandwidth per worker in MB per sec + * @return BackupInfo + * @throws BackupException exception + */ + public BackupInfo createBackupContext(String backupId, BackupType type, + List<TableName> tableList, String targetRootDir, int workers, long bandwidth) + throws BackupException { + if (targetRootDir == null) { + throw new BackupException("Wrong backup request parameter: target backup root directory"); + } + + if (type == BackupType.FULL && (tableList == null || tableList.isEmpty())) { + // If table list is null for full backup, which means backup all tables. Then fill the table + // list with all user tables from meta. It no table available, throw the request exception. + + HTableDescriptor[] htds = null; + try (Admin admin = conn.getAdmin()) { + htds = admin.listTables(); + } catch (Exception e) { + throw new BackupException(e); + } + + if (htds == null) { + throw new BackupException("No table exists for full backup of all tables."); + } else { + tableList = new ArrayList<>(); + for (HTableDescriptor hTableDescriptor : htds) { + tableList.add(hTableDescriptor.getTableName()); + } + + LOG.info("Full backup all the tables available in the cluster: " + tableList); + } + } + + // there are one or more tables in the table list + backupContext = new BackupInfo(backupId, type, + tableList.toArray(new TableName[tableList.size()]), + targetRootDir); + backupContext.setBandwidth(bandwidth); + backupContext.setWorkers(workers); + return backupContext; + } + + /** + * Check if any ongoing backup. Currently, we only reply on checking status in hbase:backup. We + * need to consider to handle the case of orphan records in the future. Otherwise, all the coming + * request will fail. + * @return the ongoing backup id if on going backup exists, otherwise null + * @throws IOException exception + */ + private String getOngoingBackupId() throws IOException { + + ArrayList<BackupInfo> sessions = systemTable.getBackupContexts(BackupState.RUNNING); + if (sessions.size() == 0) { + return null; + } + return sessions.get(0).getBackupId(); + } + + /** + * Start the backup manager service. + * @throws IOException exception + */ + public void initialize() throws IOException { + String ongoingBackupId = this.getOngoingBackupId(); + if (ongoingBackupId != null) { + LOG.info("There is a ongoing backup " + ongoingBackupId + + ". Can not launch new backup until no ongoing backup remains."); + throw new BackupException("There is ongoing backup."); + } + + // Initialize thread pools + int nrThreads = this.conf.getInt("hbase.backup.threads.max", 1); + ThreadFactoryBuilder builder = new ThreadFactoryBuilder(); + builder.setNameFormat("BackupHandler-%1$d"); + this.pool = + new ThreadPoolExecutor(nrThreads, nrThreads, 60, TimeUnit.SECONDS, + new LinkedBlockingQueue<Runnable>(), builder.build()); + ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true); + } + + public void setBackupContext(BackupInfo backupContext) { + this.backupContext = backupContext; + } + + /** + * Get direct ancestors of the current backup. + * @param backupCtx The backup context for the current backup + * @return The ancestors for the current backup + * @throws IOException exception + * @throws BackupException exception + */ + public ArrayList<BackupImage> getAncestors(BackupInfo backupCtx) throws IOException, + BackupException { + LOG.debug("Getting the direct ancestors of the current backup "+ + backupCtx.getBackupId()); + + ArrayList<BackupImage> ancestors = new ArrayList<BackupImage>(); + + // full backup does not have ancestor + if (backupCtx.getType() == BackupType.FULL) { + LOG.debug("Current backup is a full backup, no direct ancestor for it."); + return ancestors; + } + + // get all backup history list in descending order + + ArrayList<BackupInfo> allHistoryList = getBackupHistory(true); + for (BackupInfo backup : allHistoryList) { + BackupImage image = + new BackupImage(backup.getBackupId(), backup.getType(), + backup.getTargetRootDir(), + backup.getTableNames(), backup.getStartTs(), backup + .getEndTs()); + // add the full backup image as an ancestor until the last incremental backup + if (backup.getType().equals(BackupType.FULL)) { + // check the backup image coverage, if previous image could be covered by the newer ones, + // then no need to add + if (!BackupManifest.canCoverImage(ancestors, image)) { + ancestors.add(image); + } + } else { + // found last incremental backup, if previously added full backup ancestor images can cover + // it, then this incremental ancestor is not the dependent of the current incremental + // backup, that is to say, this is the backup scope boundary of current table set. + // Otherwise, this incremental backup ancestor is the dependent ancestor of the ongoing + // incremental backup + if (BackupManifest.canCoverImage(ancestors, image)) { + LOG.debug("Met the backup boundary of the current table set. " + + "The root full backup images for the current backup scope:"); + for (BackupImage image1 : ancestors) { + LOG.debug(" BackupId: " + image1.getBackupId() + ", Backup directory: " + + image1.getRootDir()); + } + } else { + Path logBackupPath = + HBackupFileSystem.getLogBackupPath(backup.getTargetRootDir(), + backup.getBackupId()); + LOG.debug("Current backup has an incremental backup ancestor, " + + "touching its image manifest in " + logBackupPath.toString() + + " to construct the dependency."); + BackupManifest lastIncrImgManifest = new BackupManifest(conf, logBackupPath); + BackupImage lastIncrImage = lastIncrImgManifest.getBackupImage(); + ancestors.add(lastIncrImage); + + LOG.debug("Last dependent incremental backup image information:"); + LOG.debug(" Token: " + lastIncrImage.getBackupId()); + LOG.debug(" Backup directory: " + lastIncrImage.getRootDir()); + } + } + } + LOG.debug("Got " + ancestors.size() + " ancestors for the current backup."); + return ancestors; + } + + /** + * Get the direct ancestors of this backup for one table involved. + * @param backupContext backup context + * @param table table + * @return backupImages on the dependency list + * @throws BackupException exception + * @throws IOException exception + */ + public ArrayList<BackupImage> getAncestors(BackupInfo backupContext, TableName table) + throws BackupException, IOException { + ArrayList<BackupImage> ancestors = getAncestors(backupContext); + ArrayList<BackupImage> tableAncestors = new ArrayList<BackupImage>(); + for (BackupImage image : ancestors) { + if (image.hasTable(table)) { + tableAncestors.add(image); + if (image.getType() == BackupType.FULL) { + break; + } + } + } + return tableAncestors; + } + + /* + * hbase:backup operations + */ + + /** + * Updates status (state) of a backup session in a persistent store + * @param context context + * @throws IOException exception + */ + public void updateBackupInfo(BackupInfo context) throws IOException { + systemTable.updateBackupInfo(context); + } + + /** + * Read the last backup start code (timestamp) of last successful backup. Will return null + * if there is no startcode stored in hbase:backup or the value is of length 0. These two + * cases indicate there is no successful backup completed so far. + * @return the timestamp of a last successful backup + * @throws IOException exception + */ + public String readBackupStartCode() throws IOException { + return systemTable.readBackupStartCode(backupContext.getTargetRootDir()); + } + + /** + * Write the start code (timestamp) to hbase:backup. If passed in null, then write 0 byte. + * @param startCode start code + * @throws IOException exception + */ + public void writeBackupStartCode(Long startCode) throws IOException { + systemTable.writeBackupStartCode(startCode, backupContext.getTargetRootDir()); + } + + /** + * Get the RS log information after the last log roll from hbase:backup. + * @return RS log info + * @throws IOException exception + */ + public HashMap<String, Long> readRegionServerLastLogRollResult() throws IOException { + return systemTable.readRegionServerLastLogRollResult(backupContext.getTargetRootDir()); + } + + /** + * Get all completed backup information (in desc order by time) + * @return history info of BackupCompleteData + * @throws IOException exception + */ + public List<BackupInfo> getBackupHistory() throws IOException { + return systemTable.getBackupHistory(); + } + + public ArrayList<BackupInfo> getBackupHistory(boolean completed) throws IOException { + return systemTable.getBackupHistory(completed); + } + /** + * Write the current timestamps for each regionserver to hbase:backup after a successful full or + * incremental backup. Each table may have a different set of log timestamps. The saved timestamp + * is of the last log file that was backed up already. + * @param tables tables + * @throws IOException exception + */ + public void writeRegionServerLogTimestamp(Set<TableName> tables, + HashMap<String, Long> newTimestamps) throws IOException { + systemTable.writeRegionServerLogTimestamp(tables, newTimestamps, + backupContext.getTargetRootDir()); + } + + /** + * Read the timestamp for each region server log after the last successful backup. Each table has + * its own set of the timestamps. + * @return the timestamp for each region server. key: tableName value: + * RegionServer,PreviousTimeStamp + * @throws IOException exception + */ + public HashMap<TableName, HashMap<String, Long>> readLogTimestampMap() throws IOException { + return systemTable.readLogTimestampMap(backupContext.getTargetRootDir()); + } + + /** + * Return the current tables covered by incremental backup. + * @return set of tableNames + * @throws IOException exception + */ + public Set<TableName> getIncrementalBackupTableSet() throws IOException { + return systemTable.getIncrementalBackupTableSet(backupContext.getTargetRootDir()); + } + + /** + * Adds set of tables to overall incremental backup table set + * @param tables tables + * @throws IOException exception + */ + public void addIncrementalBackupTableSet(Set<TableName> tables) throws IOException { + systemTable.addIncrementalBackupTableSet(tables, backupContext.getTargetRootDir()); + } + + /** + * Saves list of WAL files after incremental backup operation. These files will be stored until + * TTL expiration and are used by Backup Log Cleaner plugin to determine which WAL files can be + * safely purged. + */ + public void recordWALFiles(List<String> files) throws IOException { + systemTable.addWALFiles(files, + backupContext.getBackupId(), backupContext.getTargetRootDir()); + } + + /** + * Get WAL files iterator + * @return WAL files iterator from hbase:backup + * @throws IOException + */ + public Iterator<BackupSystemTable.WALItem> getWALFilesFromBackupSystem() throws IOException { + return systemTable.getWALFilesIterator(backupContext.getTargetRootDir()); + } + + public Connection getConnection() { + return conn; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java new file mode 100644 index 0000000..c3dc539 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java @@ -0,0 +1,711 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup.impl; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.backup.BackupInfo; +import org.apache.hadoop.hbase.backup.BackupType; +import org.apache.hadoop.hbase.backup.util.BackupClientUtil; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos; + + +/** + * Backup manifest Contains all the meta data of a backup image. The manifest info will be bundled + * as manifest file together with data. So that each backup image will contain all the info needed + * for restore. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class BackupManifest { + + private static final Log LOG = LogFactory.getLog(BackupManifest.class); + + // manifest file name + public static final String MANIFEST_FILE_NAME = ".backup.manifest"; + + // backup image, the dependency graph is made up by series of backup images + + public static class BackupImage implements Comparable<BackupImage> { + + private String backupId; + private BackupType type; + private String rootDir; + private List<TableName> tableList; + private long startTs; + private long completeTs; + private ArrayList<BackupImage> ancestors; + private HashMap<TableName, HashMap<String, Long>> incrTimeRanges; + + public BackupImage() { + super(); + } + + public BackupImage(String backupId, BackupType type, String rootDir, + List<TableName> tableList, long startTs, long completeTs) { + this.backupId = backupId; + this.type = type; + this.rootDir = rootDir; + this.tableList = tableList; + this.startTs = startTs; + this.completeTs = completeTs; + } + + static BackupImage fromProto(BackupProtos.BackupImage im) { + String backupId = im.getBackupId(); + String rootDir = im.getRootDir(); + long startTs = im.getStartTs(); + long completeTs = im.getCompleteTs(); + List<HBaseProtos.TableName> tableListList = im.getTableListList(); + List<TableName> tableList = new ArrayList<TableName>(); + for(HBaseProtos.TableName tn : tableListList) { + tableList.add(ProtobufUtil.toTableName(tn)); + } + + List<BackupProtos.BackupImage> ancestorList = im.getAncestorsList(); + + BackupType type = + im.getBackupType() == BackupProtos.BackupType.FULL ? BackupType.FULL: + BackupType.INCREMENTAL; + + BackupImage image = new BackupImage(backupId, type, rootDir, tableList, startTs, completeTs); + for(BackupProtos.BackupImage img: ancestorList) { + image.addAncestor(fromProto(img)); + } + image.setIncrTimeRanges(loadIncrementalTimestampMap(im)); + return image; + } + + BackupProtos.BackupImage toProto() { + BackupProtos.BackupImage.Builder builder = BackupProtos.BackupImage.newBuilder(); + builder.setBackupId(backupId); + builder.setCompleteTs(completeTs); + builder.setStartTs(startTs); + builder.setRootDir(rootDir); + if (type == BackupType.FULL) { + builder.setBackupType(BackupProtos.BackupType.FULL); + } else{ + builder.setBackupType(BackupProtos.BackupType.INCREMENTAL); + } + + for (TableName name: tableList) { + builder.addTableList(ProtobufUtil.toProtoTableNameShaded(name)); + } + + if (ancestors != null){ + for (BackupImage im: ancestors){ + builder.addAncestors(im.toProto()); + } + } + + setIncrementalTimestampMap(builder); + return builder.build(); + } + + + private static HashMap<TableName, HashMap<String, Long>> + loadIncrementalTimestampMap(BackupProtos.BackupImage proto) { + List<BackupProtos.TableServerTimestamp> list = proto.getTstMapList(); + + HashMap<TableName, HashMap<String, Long>> incrTimeRanges = + new HashMap<TableName, HashMap<String, Long>>(); + if(list == null || list.size() == 0) return incrTimeRanges; + for(BackupProtos.TableServerTimestamp tst: list){ + TableName tn = ProtobufUtil.toTableName(tst.getTable()); + HashMap<String, Long> map = incrTimeRanges.get(tn); + if(map == null){ + map = new HashMap<String, Long>(); + incrTimeRanges.put(tn, map); + } + List<BackupProtos.ServerTimestamp> listSt = tst.getServerTimestampList(); + for(BackupProtos.ServerTimestamp stm: listSt) { + ServerName sn = ProtobufUtil.toServerNameShaded(stm.getServer()); + map.put(sn.getHostname() +":" + sn.getPort(), stm.getTimestamp()); + } + } + return incrTimeRanges; + } + + + private void setIncrementalTimestampMap(BackupProtos.BackupImage.Builder builder) { + if (this.incrTimeRanges == null) { + return; + } + for (Entry<TableName, HashMap<String,Long>> entry: this.incrTimeRanges.entrySet()) { + TableName key = entry.getKey(); + HashMap<String, Long> value = entry.getValue(); + BackupProtos.TableServerTimestamp.Builder tstBuilder = + BackupProtos.TableServerTimestamp.newBuilder(); + tstBuilder.setTable(ProtobufUtil.toProtoTableNameShaded(key)); + + for (Map.Entry<String, Long> entry2 : value.entrySet()) { + String s = entry2.getKey(); + BackupProtos.ServerTimestamp.Builder stBuilder = BackupProtos.ServerTimestamp.newBuilder(); + HBaseProtos.ServerName.Builder snBuilder = HBaseProtos.ServerName.newBuilder(); + ServerName sn = ServerName.parseServerName(s); + snBuilder.setHostName(sn.getHostname()); + snBuilder.setPort(sn.getPort()); + stBuilder.setServer(snBuilder.build()); + stBuilder.setTimestamp(entry2.getValue()); + tstBuilder.addServerTimestamp(stBuilder.build()); + } + builder.addTstMap(tstBuilder.build()); + } + } + + public String getBackupId() { + return backupId; + } + + public void setBackupId(String backupId) { + this.backupId = backupId; + } + + public BackupType getType() { + return type; + } + + public void setType(BackupType type) { + this.type = type; + } + + public String getRootDir() { + return rootDir; + } + + public void setRootDir(String rootDir) { + this.rootDir = rootDir; + } + + public List<TableName> getTableNames() { + return tableList; + } + + public void setTableList(List<TableName> tableList) { + this.tableList = tableList; + } + + public long getStartTs() { + return startTs; + } + + public void setStartTs(long startTs) { + this.startTs = startTs; + } + + public long getCompleteTs() { + return completeTs; + } + + public void setCompleteTs(long completeTs) { + this.completeTs = completeTs; + } + + public ArrayList<BackupImage> getAncestors() { + if (this.ancestors == null) { + this.ancestors = new ArrayList<BackupImage>(); + } + return this.ancestors; + } + + public void addAncestor(BackupImage backupImage) { + this.getAncestors().add(backupImage); + } + + public boolean hasAncestor(String token) { + for (BackupImage image : this.getAncestors()) { + if (image.getBackupId().equals(token)) { + return true; + } + } + return false; + } + + public boolean hasTable(TableName table) { + for (TableName t : tableList) { + if (t.equals(table)) { + return true; + } + } + return false; + } + + @Override + public int compareTo(BackupImage other) { + String thisBackupId = this.getBackupId(); + String otherBackupId = other.getBackupId(); + int index1 = thisBackupId.lastIndexOf("_"); + int index2 = otherBackupId.lastIndexOf("_"); + String name1 = thisBackupId.substring(0, index1); + String name2 = otherBackupId.substring(0, index2); + if(name1.equals(name2)) { + Long thisTS = Long.valueOf(thisBackupId.substring(index1 + 1)); + Long otherTS = Long.valueOf(otherBackupId.substring(index2 + 1)); + return thisTS.compareTo(otherTS); + } else { + return name1.compareTo(name2); + } + } + @Override + public boolean equals(Object obj) { + if (obj instanceof BackupImage) { + return this.compareTo((BackupImage)obj) == 0; + } + return false; + } + @Override + public int hashCode() { + int hash = 33 * this.getBackupId().hashCode() + type.hashCode(); + hash = 33 * hash + rootDir.hashCode(); + hash = 33 * hash + Long.valueOf(startTs).hashCode(); + hash = 33 * hash + Long.valueOf(completeTs).hashCode(); + for (TableName table : tableList) { + hash = 33 * hash + table.hashCode(); + } + return hash; + } + + public HashMap<TableName, HashMap<String, Long>> getIncrTimeRanges() { + return incrTimeRanges; + } + + public void setIncrTimeRanges(HashMap<TableName, HashMap<String, Long>> incrTimeRanges) { + this.incrTimeRanges = incrTimeRanges; + } + } + + // hadoop hbase configuration + protected Configuration config = null; + + // backup root directory + private String rootDir = null; + + // backup image directory + private String tableBackupDir = null; + + // backup log directory if this is an incremental backup + private String logBackupDir = null; + + // backup token + private String backupId; + + // backup type, full or incremental + private BackupType type; + + // the table list for the backup + private ArrayList<TableName> tableList; + + // actual start timestamp of the backup process + private long startTs; + + // actual complete timestamp of the backup process + private long completeTs; + + // the region server timestamp for tables: + // <table, <rs, timestamp>> + private Map<TableName, HashMap<String, Long>> incrTimeRanges; + + // dependency of this backup, including all the dependent images to do PIT recovery + //private Map<String, BackupImage> dependency; + private BackupImage backupImage; + + /** + * Construct manifest for a ongoing backup. + * @param backupCtx The ongoing backup context + */ + public BackupManifest(BackupInfo backupCtx) { + this.backupId = backupCtx.getBackupId(); + this.type = backupCtx.getType(); + this.rootDir = backupCtx.getTargetRootDir(); + if (this.type == BackupType.INCREMENTAL) { + this.logBackupDir = backupCtx.getHLogTargetDir(); + } + this.startTs = backupCtx.getStartTs(); + this.completeTs = backupCtx.getEndTs(); + this.loadTableList(backupCtx.getTableNames()); + this.backupImage = new BackupImage(this.backupId, this.type, this.rootDir, tableList, this.startTs, + this.completeTs); + } + + + /** + * Construct a table level manifest for a backup of the named table. + * @param backupCtx The ongoing backup context + */ + public BackupManifest(BackupInfo backupCtx, TableName table) { + this.backupId = backupCtx.getBackupId(); + this.type = backupCtx.getType(); + this.rootDir = backupCtx.getTargetRootDir(); + this.tableBackupDir = backupCtx.getBackupStatus(table).getTargetDir(); + if (this.type == BackupType.INCREMENTAL) { + this.logBackupDir = backupCtx.getHLogTargetDir(); + } + this.startTs = backupCtx.getStartTs(); + this.completeTs = backupCtx.getEndTs(); + List<TableName> tables = new ArrayList<TableName>(); + tables.add(table); + this.loadTableList(tables); + this.backupImage = new BackupImage(this.backupId, this.type, this.rootDir, tableList, this.startTs, + this.completeTs); + } + + /** + * Construct manifest from a backup directory. + * @param conf configuration + * @param backupPath backup path + * @throws IOException + */ + + public BackupManifest(Configuration conf, Path backupPath) throws IOException { + this(backupPath.getFileSystem(conf), backupPath); + } + + /** + * Construct manifest from a backup directory. + * @param fs the FileSystem + * @param backupPath backup path + * @throws BackupException exception + */ + + public BackupManifest(FileSystem fs, Path backupPath) throws BackupException { + if (LOG.isDebugEnabled()) { + LOG.debug("Loading manifest from: " + backupPath.toString()); + } + // The input backupDir may not exactly be the backup table dir. + // It could be the backup log dir where there is also a manifest file stored. + // This variable's purpose is to keep the correct and original location so + // that we can store/persist it. + this.tableBackupDir = backupPath.toString(); + this.config = fs.getConf(); + try { + + FileStatus[] subFiles = BackupClientUtil.listStatus(fs, backupPath, null); + if (subFiles == null) { + String errorMsg = backupPath.toString() + " does not exist"; + LOG.error(errorMsg); + throw new IOException(errorMsg); + } + for (FileStatus subFile : subFiles) { + if (subFile.getPath().getName().equals(MANIFEST_FILE_NAME)) { + + // load and set manifest field from file content + FSDataInputStream in = fs.open(subFile.getPath()); + long len = subFile.getLen(); + byte[] pbBytes = new byte[(int) len]; + in.readFully(pbBytes); + BackupProtos.BackupImage proto = null; + try{ + proto = BackupProtos.BackupImage.parseFrom(pbBytes); + } catch(Exception e){ + throw new BackupException(e); + } + this.backupImage = BackupImage.fromProto(proto); + // Here the parameter backupDir is where the manifest file is. + // There should always be a manifest file under: + // backupRootDir/namespace/table/backupId/.backup.manifest + this.rootDir = backupPath.getParent().getParent().getParent().toString(); + + Path p = backupPath.getParent(); + if (p.getName().equals(HConstants.HREGION_LOGDIR_NAME)) { + this.rootDir = p.getParent().toString(); + } else { + this.rootDir = p.getParent().getParent().toString(); + } + this.backupId = this.backupImage.getBackupId(); + this.startTs = this.backupImage.getStartTs(); + this.completeTs = this.backupImage.getCompleteTs(); + this.type = this.backupImage.getType(); + this.tableList = (ArrayList<TableName>)this.backupImage.getTableNames(); + this.incrTimeRanges = this.backupImage.getIncrTimeRanges(); + LOG.debug("Loaded manifest instance from manifest file: " + + BackupClientUtil.getPath(subFile.getPath())); + return; + } + } + String errorMsg = "No manifest file found in: " + backupPath.toString(); + throw new IOException(errorMsg); + + } catch (IOException e) { + throw new BackupException(e.getMessage()); + } + } + + public BackupType getType() { + return type; + } + + public void setType(BackupType type) { + this.type = type; + } + + /** + * Loads table list. + * @param tableList Table list + */ + private void loadTableList(List<TableName> tableList) { + + this.tableList = this.getTableList(); + if (this.tableList.size() > 0) { + this.tableList.clear(); + } + for (int i = 0; i < tableList.size(); i++) { + this.tableList.add(tableList.get(i)); + } + + LOG.debug(tableList.size() + " tables exist in table set."); + } + + /** + * Get the table set of this image. + * @return The table set list + */ + public ArrayList<TableName> getTableList() { + if (this.tableList == null) { + this.tableList = new ArrayList<TableName>(); + } + return this.tableList; + } + + /** + * Persist the manifest file. + * @throws IOException IOException when storing the manifest file. + */ + + public void store(Configuration conf) throws BackupException { + byte[] data = backupImage.toProto().toByteArray(); + // write the file, overwrite if already exist + Path manifestFilePath = + new Path(new Path((this.tableBackupDir != null ? this.tableBackupDir : this.logBackupDir)) + ,MANIFEST_FILE_NAME); + try { + FSDataOutputStream out = + manifestFilePath.getFileSystem(conf).create(manifestFilePath, true); + out.write(data); + out.close(); + } catch (IOException e) { + throw new BackupException(e.getMessage()); + } + + LOG.info("Manifest file stored to " + manifestFilePath); + } + + + /** + * Get this backup image. + * @return the backup image. + */ + public BackupImage getBackupImage() { + return backupImage; + } + + /** + * Add dependent backup image for this backup. + * @param image The direct dependent backup image + */ + public void addDependentImage(BackupImage image) { + this.backupImage.addAncestor(image); + } + + /** + * Set the incremental timestamp map directly. + * @param incrTimestampMap timestamp map + */ + public void setIncrTimestampMap(HashMap<TableName, HashMap<String, Long>> incrTimestampMap) { + this.incrTimeRanges = incrTimestampMap; + this.backupImage.setIncrTimeRanges(incrTimestampMap); + } + + public Map<TableName, HashMap<String, Long>> getIncrTimestampMap() { + if (this.incrTimeRanges == null) { + this.incrTimeRanges = new HashMap<TableName, HashMap<String, Long>>(); + } + return this.incrTimeRanges; + } + + /** + * Get the image list of this backup for restore in time order. + * @param reverse If true, then output in reverse order, otherwise in time order from old to new + * @return the backup image list for restore in time order + */ + public ArrayList<BackupImage> getRestoreDependentList(boolean reverse) { + TreeMap<Long, BackupImage> restoreImages = new TreeMap<Long, BackupImage>(); + restoreImages.put(backupImage.startTs, backupImage); + for (BackupImage image : backupImage.getAncestors()) { + restoreImages.put(Long.valueOf(image.startTs), image); + } + return new ArrayList<BackupImage>(reverse ? (restoreImages.descendingMap().values()) + : (restoreImages.values())); + } + + /** + * Get the dependent image list for a specific table of this backup in time order from old to new + * if want to restore to this backup image level. + * @param table table + * @return the backup image list for a table in time order + */ + public ArrayList<BackupImage> getDependentListByTable(TableName table) { + ArrayList<BackupImage> tableImageList = new ArrayList<BackupImage>(); + ArrayList<BackupImage> imageList = getRestoreDependentList(true); + for (BackupImage image : imageList) { + if (image.hasTable(table)) { + tableImageList.add(image); + if (image.getType() == BackupType.FULL) { + break; + } + } + } + Collections.reverse(tableImageList); + return tableImageList; + } + + /** + * Get the full dependent image list in the whole dependency scope for a specific table of this + * backup in time order from old to new. + * @param table table + * @return the full backup image list for a table in time order in the whole scope of the + * dependency of this image + */ + public ArrayList<BackupImage> getAllDependentListByTable(TableName table) { + ArrayList<BackupImage> tableImageList = new ArrayList<BackupImage>(); + ArrayList<BackupImage> imageList = getRestoreDependentList(false); + for (BackupImage image : imageList) { + if (image.hasTable(table)) { + tableImageList.add(image); + } + } + return tableImageList; + } + + /** + * Check whether backup image1 could cover backup image2 or not. + * @param image1 backup image 1 + * @param image2 backup image 2 + * @return true if image1 can cover image2, otherwise false + */ + public static boolean canCoverImage(BackupImage image1, BackupImage image2) { + // image1 can cover image2 only when the following conditions are satisfied: + // - image1 must not be an incremental image; + // - image1 must be taken after image2 has been taken; + // - table set of image1 must cover the table set of image2. + if (image1.getType() == BackupType.INCREMENTAL) { + return false; + } + if (image1.getStartTs() < image2.getStartTs()) { + return false; + } + List<TableName> image1TableList = image1.getTableNames(); + List<TableName> image2TableList = image2.getTableNames(); + boolean found = false; + for (int i = 0; i < image2TableList.size(); i++) { + found = false; + for (int j = 0; j < image1TableList.size(); j++) { + if (image2TableList.get(i).equals(image1TableList.get(j))) { + found = true; + break; + } + } + if (!found) { + return false; + } + } + + LOG.debug("Backup image " + image1.getBackupId() + " can cover " + image2.getBackupId()); + return true; + } + + /** + * Check whether backup image set could cover a backup image or not. + * @param fullImages The backup image set + * @param image The target backup image + * @return true if fullImages can cover image, otherwise false + */ + public static boolean canCoverImage(ArrayList<BackupImage> fullImages, BackupImage image) { + // fullImages can cover image only when the following conditions are satisfied: + // - each image of fullImages must not be an incremental image; + // - each image of fullImages must be taken after image has been taken; + // - sum table set of fullImages must cover the table set of image. + for (BackupImage image1 : fullImages) { + if (image1.getType() == BackupType.INCREMENTAL) { + return false; + } + if (image1.getStartTs() < image.getStartTs()) { + return false; + } + } + + ArrayList<String> image1TableList = new ArrayList<String>(); + for (BackupImage image1 : fullImages) { + List<TableName> tableList = image1.getTableNames(); + for (TableName table : tableList) { + image1TableList.add(table.getNameAsString()); + } + } + ArrayList<String> image2TableList = new ArrayList<String>(); + List<TableName> tableList = image.getTableNames(); + for (TableName table : tableList) { + image2TableList.add(table.getNameAsString()); + } + + for (int i = 0; i < image2TableList.size(); i++) { + if (image1TableList.contains(image2TableList.get(i)) == false) { + return false; + } + } + + LOG.debug("Full image set can cover image " + image.getBackupId()); + return true; + } + + public BackupInfo toBackupInfo() + { + BackupInfo info = new BackupInfo(); + info.setType(type); + TableName[] tables = new TableName[tableList.size()]; + info.addTables(getTableList().toArray(tables)); + info.setBackupId(backupId); + info.setStartTs(startTs); + info.setTargetRootDir(rootDir); + if(type == BackupType.INCREMENTAL) { + info.setHlogTargetDir(logBackupDir); + } + return info; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSnapshotCopy.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSnapshotCopy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSnapshotCopy.java new file mode 100644 index 0000000..731ccd3 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSnapshotCopy.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.backup.impl; + +import org.apache.hadoop.hbase.snapshot.ExportSnapshot; + +/* this class will be extended in future jira to support progress report */ +public class BackupSnapshotCopy extends ExportSnapshot { + private String table; + + public BackupSnapshotCopy(String table) { + super(); + this.table = table; + } + + public String getTable() { + return this.table; + } + +}
