HBASE-16727 Backup refactoring: remove MR dependencies from HMaster (Vladimir Rodionov)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b14e2ab1 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b14e2ab1 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b14e2ab1 Branch: refs/heads/HBASE-7912 Commit: b14e2ab1c24e65ff88dd4c579acf83cb4ed0605e Parents: e35f7b9 Author: tedyu <yuzhih...@gmail.com> Authored: Wed Oct 5 16:29:40 2016 -0700 Committer: tedyu <yuzhih...@gmail.com> Committed: Wed Oct 5 16:29:40 2016 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hbase/backup/BackupInfo.java | 504 - .../hadoop/hbase/backup/BackupRequest.java | 91 - .../hadoop/hbase/backup/BackupStatus.java | 104 - .../hadoop/hbase/backup/RestoreRequest.java | 94 - .../hbase/backup/impl/BackupCommands.java | 717 - .../hbase/backup/impl/BackupException.java | 86 - .../hbase/backup/impl/BackupManifest.java | 791 - .../backup/impl/BackupRestoreConstants.java | 47 - .../hbase/backup/impl/BackupSystemTable.java | 873 - .../backup/impl/BackupSystemTableHelper.java | 433 - .../hbase/backup/util/BackupClientUtil.java | 437 - .../hadoop/hbase/backup/util/BackupSet.java | 62 - .../org/apache/hadoop/hbase/client/Admin.java | 9 - .../apache/hadoop/hbase/client/BackupAdmin.java | 174 - .../hbase/client/ConnectionImplementation.java | 13 - .../apache/hadoop/hbase/client/HBaseAdmin.java | 231 +- .../hadoop/hbase/client/HBaseBackupAdmin.java | 439 - .../hadoop/hbase/protobuf/RequestConverter.java | 44 - .../ClientSnapshotDescriptionUtils.java | 3 +- .../hbase/IntegrationTestBackupRestore.java | 10 +- .../hbase/protobuf/generated/BackupProtos.java | 15632 ++++++++--------- .../hbase/protobuf/generated/MasterProtos.java | 5893 +------ hbase-protocol/src/main/protobuf/Backup.proto | 8 +- hbase-protocol/src/main/protobuf/Master.proto | 44 - .../apache/hadoop/hbase/backup/BackupAdmin.java | 171 + .../apache/hadoop/hbase/backup/BackupInfo.java | 504 + .../hadoop/hbase/backup/BackupRequest.java | 91 + .../hadoop/hbase/backup/BackupStatus.java | 104 + .../hadoop/hbase/backup/RestoreDriver.java | 4 +- .../hadoop/hbase/backup/RestoreRequest.java | 94 + .../hbase/backup/impl/BackupCommands.java | 720 + .../hbase/backup/impl/BackupException.java | 86 + .../hbase/backup/impl/BackupManifest.java | 791 + .../backup/impl/BackupRestoreConstants.java | 47 + .../hbase/backup/impl/BackupSystemTable.java | 926 + .../backup/impl/BackupSystemTableHelper.java | 433 + .../backup/impl/FullTableBackupClient.java | 540 + .../hbase/backup/impl/HBaseBackupAdmin.java | 555 + .../backup/impl/IncrementalBackupManager.java | 27 +- .../impl/IncrementalTableBackupClient.java | 235 + .../hbase/backup/impl/RestoreTablesClient.java | 236 + .../backup/master/FullTableBackupProcedure.java | 777 - .../master/IncrementalTableBackupProcedure.java | 400 - .../backup/master/RestoreTablesProcedure.java | 387 - .../hbase/backup/util/BackupClientUtil.java | 437 + .../hbase/backup/util/BackupServerUtil.java | 3 +- .../hadoop/hbase/backup/util/BackupSet.java | 62 + .../hbase/backup/util/RestoreServerUtil.java | 85 +- .../org/apache/hadoop/hbase/master/HMaster.java | 119 +- .../hadoop/hbase/master/MasterRpcServices.java | 42 - .../hadoop/hbase/master/MasterServices.java | 30 - .../hadoop/hbase/backup/TestBackupBase.java | 24 +- .../hbase/backup/TestBackupBoundaryTests.java | 8 +- .../hbase/backup/TestBackupDeleteRestore.java | 1 - .../hbase/backup/TestBackupMultipleDeletes.java | 20 +- .../hadoop/hbase/backup/TestFullRestore.java | 1 - .../hbase/backup/TestIncrementalBackup.java | 61 +- .../TestIncrementalBackupDeleteTable.java | 8 +- .../hadoop/hbase/backup/TestRemoteBackup.java | 1 - .../hadoop/hbase/master/TestCatalogJanitor.java | 17 - 60 files changed, 13746 insertions(+), 21040 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java deleted file mode 100644 index be5ffea..0000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java +++ /dev/null @@ -1,504 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.util.BackupClientUtil; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.BackupProtos; -import org.apache.hadoop.hbase.protobuf.generated.BackupProtos.BackupInfo.Builder; -import org.apache.hadoop.hbase.protobuf.generated.BackupProtos.TableBackupStatus; -import org.apache.hadoop.hbase.util.Bytes; - - -/** - * An object to encapsulate the information for each backup request - */ -@InterfaceAudience.Private -@InterfaceStability.Evolving -public class BackupInfo implements Comparable<BackupInfo> { - private static final Log LOG = LogFactory.getLog(BackupInfo.class); - - public static interface Filter { - - /** - * Filter interface - * @param info: backup info - * @return true if info passes filter, false otherwise - */ - public boolean apply(BackupInfo info); - } - // backup status flag - public static enum BackupState { - WAITING, RUNNING, COMPLETE, FAILED, ANY; - } - - // backup phase - public static enum BackupPhase { - SNAPSHOTCOPY, INCREMENTAL_COPY, STORE_MANIFEST; - } - - // backup id: a timestamp when we request the backup - private String backupId; - - // backup type, full or incremental - private BackupType type; - - // target root directory for storing the backup files - private String targetRootDir; - - // overall backup state - private BackupState state; - - // overall backup phase - private BackupPhase phase; - - // overall backup failure message - private String failedMsg; - - // backup status map for all tables - private Map<TableName, BackupStatus> backupStatusMap; - - // actual start timestamp of the backup process - private long startTs; - - // actual end timestamp of the backup process, could be fail or complete - private long endTs; - - // the total bytes of incremental logs copied - private long totalBytesCopied; - - // for incremental backup, the location of the backed-up hlogs - private String hlogTargetDir = null; - - // incremental backup file list - transient private List<String> incrBackupFileList; - - // new region server log timestamps for table set after distributed log roll - // key - table name, value - map of RegionServer hostname -> last log rolled timestamp - transient private HashMap<TableName, HashMap<String, Long>> tableSetTimestampMap; - - // backup progress in %% (0-100) - private int progress; - - // distributed job id - private String jobId; - - // Number of parallel workers. -1 - system defined - private int workers = -1; - - // Bandwidth per worker in MB per sec. -1 - unlimited - private long bandwidth = -1; - - public BackupInfo() { - backupStatusMap = new HashMap<TableName, BackupStatus>(); - } - - public BackupInfo(String backupId, BackupType type, TableName[] tables, String targetRootDir) { - this(); - this.backupId = backupId; - this.type = type; - this.targetRootDir = targetRootDir; - if (LOG.isDebugEnabled()) { - LOG.debug("CreateBackupContext: " + tables.length + " " + tables[0]); - } - this.addTables(tables); - - if (type == BackupType.INCREMENTAL) { - setHlogTargetDir(BackupClientUtil.getLogBackupDir(targetRootDir, backupId)); - } - - this.startTs = 0; - this.endTs = 0; - } - - public String getJobId() { - return jobId; - } - - public void setJobId(String jobId) { - this.jobId = jobId; - } - - public int getWorkers() { - return workers; - } - - public void setWorkers(int workers) { - this.workers = workers; - } - - public long getBandwidth() { - return bandwidth; - } - - public void setBandwidth(long bandwidth) { - this.bandwidth = bandwidth; - } - - public void setBackupStatusMap(Map<TableName, BackupStatus> backupStatusMap) { - this.backupStatusMap = backupStatusMap; - } - - public HashMap<TableName, HashMap<String, Long>> getTableSetTimestampMap() { - return tableSetTimestampMap; - } - - public void - setTableSetTimestampMap(HashMap<TableName, HashMap<String, Long>> tableSetTimestampMap) { - this.tableSetTimestampMap = tableSetTimestampMap; - } - - public String getHlogTargetDir() { - return hlogTargetDir; - } - - public void setType(BackupType type) { - this.type = type; - } - - public void setTargetRootDir(String targetRootDir) { - this.targetRootDir = targetRootDir; - } - - public void setTotalBytesCopied(long totalBytesCopied) { - this.totalBytesCopied = totalBytesCopied; - } - - /** - * Set progress (0-100%) - * @param msg progress value - */ - - public void setProgress(int p) { - this.progress = p; - } - - /** - * Get current progress - */ - public int getProgress() { - return progress; - } - - public String getBackupId() { - return backupId; - } - - public void setBackupId(String backupId) { - this.backupId = backupId; - } - - public BackupStatus getBackupStatus(TableName table) { - return this.backupStatusMap.get(table); - } - - public String getFailedMsg() { - return failedMsg; - } - - public void setFailedMsg(String failedMsg) { - this.failedMsg = failedMsg; - } - - public long getStartTs() { - return startTs; - } - - public void setStartTs(long startTs) { - this.startTs = startTs; - } - - public long getEndTs() { - return endTs; - } - - public void setEndTs(long endTs) { - this.endTs = endTs; - } - - public long getTotalBytesCopied() { - return totalBytesCopied; - } - - public BackupState getState() { - return state; - } - - public void setState(BackupState flag) { - this.state = flag; - } - - public BackupPhase getPhase() { - return phase; - } - - public void setPhase(BackupPhase phase) { - this.phase = phase; - } - - public BackupType getType() { - return type; - } - - public void setSnapshotName(TableName table, String snapshotName) { - this.backupStatusMap.get(table).setSnapshotName(snapshotName); - } - - public String getSnapshotName(TableName table) { - return this.backupStatusMap.get(table).getSnapshotName(); - } - - public List<String> getSnapshotNames() { - List<String> snapshotNames = new ArrayList<String>(); - for (BackupStatus backupStatus : this.backupStatusMap.values()) { - snapshotNames.add(backupStatus.getSnapshotName()); - } - return snapshotNames; - } - - public Set<TableName> getTables() { - return this.backupStatusMap.keySet(); - } - - public List<TableName> getTableNames() { - return new ArrayList<TableName>(backupStatusMap.keySet()); - } - - public void addTables(TableName[] tables) { - for (TableName table : tables) { - BackupStatus backupStatus = new BackupStatus(table, this.targetRootDir, this.backupId); - this.backupStatusMap.put(table, backupStatus); - } - } - - public void setTables(List<TableName> tables) { - this.backupStatusMap.clear(); - for (TableName table : tables) { - BackupStatus backupStatus = new BackupStatus(table, this.targetRootDir, this.backupId); - this.backupStatusMap.put(table, backupStatus); - } - } - - public String getTargetRootDir() { - return targetRootDir; - } - - public void setHlogTargetDir(String hlogTagetDir) { - this.hlogTargetDir = hlogTagetDir; - } - - public String getHLogTargetDir() { - return hlogTargetDir; - } - - public List<String> getIncrBackupFileList() { - return incrBackupFileList; - } - - public void setIncrBackupFileList(List<String> incrBackupFileList) { - this.incrBackupFileList = incrBackupFileList; - } - - /** - * Set the new region server log timestamps after distributed log roll - * @param newTableSetTimestampMap table timestamp map - */ - public void - setIncrTimestampMap(HashMap<TableName, HashMap<String, Long>> newTableSetTimestampMap) { - this.tableSetTimestampMap = newTableSetTimestampMap; - } - - /** - * Get new region server log timestamps after distributed log roll - * @return new region server log timestamps - */ - public HashMap<TableName, HashMap<String, Long>> getIncrTimestampMap() { - return this.tableSetTimestampMap; - } - - public TableName getTableBySnapshot(String snapshotName) { - for (Entry<TableName, BackupStatus> entry : this.backupStatusMap.entrySet()) { - if (snapshotName.equals(entry.getValue().getSnapshotName())) { - return entry.getKey(); - } - } - return null; - } - - public BackupProtos.BackupInfo toProtosBackupInfo() { - BackupProtos.BackupInfo.Builder builder = BackupProtos.BackupInfo.newBuilder(); - builder.setBackupId(getBackupId()); - setBackupStatusMap(builder); - builder.setEndTs(getEndTs()); - if (getFailedMsg() != null) { - builder.setFailedMessage(getFailedMsg()); - } - if (getState() != null) { - builder.setState(BackupProtos.BackupInfo.BackupState.valueOf(getState().name())); - } - if (getPhase() != null) { - builder.setPhase(BackupProtos.BackupInfo.BackupPhase.valueOf(getPhase().name())); - } - - builder.setProgress(getProgress()); - builder.setStartTs(getStartTs()); - builder.setTargetRootDir(getTargetRootDir()); - builder.setType(BackupProtos.BackupType.valueOf(getType().name())); - builder.setWorkersNumber(workers); - builder.setBandwidth(bandwidth); - if (jobId != null) { - builder.setJobId(jobId); - } - return builder.build(); - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof BackupInfo) { - BackupInfo other = (BackupInfo) obj; - try { - return Bytes.equals(toByteArray(), other.toByteArray()); - } catch (IOException e) { - LOG.error(e); - return false; - } - } else { - return false; - } - } - - public byte[] toByteArray() throws IOException { - return toProtosBackupInfo().toByteArray(); - } - - private void setBackupStatusMap(Builder builder) { - for (Entry<TableName, BackupStatus> entry : backupStatusMap.entrySet()) { - builder.addTableBackupStatus(entry.getValue().toProto()); - } - } - - public static BackupInfo fromByteArray(byte[] data) throws IOException { - return fromProto(BackupProtos.BackupInfo.parseFrom(data)); - } - - public static BackupInfo fromStream(final InputStream stream) throws IOException { - return fromProto(BackupProtos.BackupInfo.parseDelimitedFrom(stream)); - } - - public static BackupInfo fromProto(BackupProtos.BackupInfo proto) { - BackupInfo context = new BackupInfo(); - context.setBackupId(proto.getBackupId()); - context.setBackupStatusMap(toMap(proto.getTableBackupStatusList())); - context.setEndTs(proto.getEndTs()); - if (proto.hasFailedMessage()) { - context.setFailedMsg(proto.getFailedMessage()); - } - if (proto.hasState()) { - context.setState(BackupInfo.BackupState.valueOf(proto.getState().name())); - } - - context.setHlogTargetDir(BackupClientUtil.getLogBackupDir(proto.getTargetRootDir(), - proto.getBackupId())); - - if (proto.hasPhase()) { - context.setPhase(BackupPhase.valueOf(proto.getPhase().name())); - } - if (proto.hasProgress()) { - context.setProgress(proto.getProgress()); - } - context.setStartTs(proto.getStartTs()); - context.setTargetRootDir(proto.getTargetRootDir()); - context.setType(BackupType.valueOf(proto.getType().name())); - context.setWorkers(proto.getWorkersNumber()); - context.setBandwidth(proto.getBandwidth()); - if (proto.hasJobId()) { - context.setJobId(proto.getJobId()); - } - return context; - } - - private static Map<TableName, BackupStatus> toMap(List<TableBackupStatus> list) { - HashMap<TableName, BackupStatus> map = new HashMap<>(); - for (TableBackupStatus tbs : list) { - map.put(ProtobufUtil.toTableName(tbs.getTable()), BackupStatus.convert(tbs)); - } - return map; - } - - public String getShortDescription() { - StringBuilder sb = new StringBuilder(); - sb.append("ID : " + backupId).append("\n"); - sb.append("Type : " + getType()).append("\n"); - sb.append("Tables : " + getTableListAsString()).append("\n"); - sb.append("State : " + getState()).append("\n"); - Date date = null; - Calendar cal = Calendar.getInstance(); - cal.setTimeInMillis(getStartTs()); - date = cal.getTime(); - sb.append("Start time : " + date).append("\n"); - if (state == BackupState.FAILED) { - sb.append("Failed message : " + getFailedMsg()).append("\n"); - } else if (state == BackupState.RUNNING) { - sb.append("Phase : " + getPhase()).append("\n"); - } else if (state == BackupState.COMPLETE) { - cal = Calendar.getInstance(); - cal.setTimeInMillis(getEndTs()); - date = cal.getTime(); - sb.append("End time : " + date).append("\n"); - } - sb.append("Progress : " + getProgress()).append("\n"); - return sb.toString(); - } - - public String getStatusAndProgressAsString() { - StringBuilder sb = new StringBuilder(); - sb.append("id: ").append(getBackupId()).append(" state: ").append(getState()) - .append(" progress: ").append(getProgress()); - return sb.toString(); - } - - public String getTableListAsString() { - return StringUtils.join(backupStatusMap.keySet(), ","); - } - - @Override - public int compareTo(BackupInfo o) { - Long thisTS = new Long(this.getBackupId().substring(this.getBackupId().lastIndexOf("_") + 1)); - Long otherTS = new Long(o.getBackupId().substring(o.getBackupId().lastIndexOf("_") + 1)); - return thisTS.compareTo(otherTS); - } - -} http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java deleted file mode 100644 index d141239..0000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import java.util.List; - -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; - -/** - * POJO class for backup request - */ -@InterfaceAudience.Public -@InterfaceStability.Evolving -public final class BackupRequest { - private BackupType type; - private List<TableName> tableList; - private String targetRootDir; - private int workers = -1; - private long bandwidth = -1L; - private String backupSetName; - - public BackupRequest() { - } - - public BackupRequest setBackupType(BackupType type) { - this.type = type; - return this; - } - public BackupType getBackupType() { - return this.type; - } - - public BackupRequest setTableList(List<TableName> tableList) { - this.tableList = tableList; - return this; - } - public List<TableName> getTableList() { - return this.tableList; - } - - public BackupRequest setTargetRootDir(String targetRootDir) { - this.targetRootDir = targetRootDir; - return this; - } - public String getTargetRootDir() { - return this.targetRootDir; - } - - public BackupRequest setWorkers(int workers) { - this.workers = workers; - return this; - } - public int getWorkers() { - return this.workers; - } - - public BackupRequest setBandwidth(long bandwidth) { - this.bandwidth = bandwidth; - return this; - } - public long getBandwidth() { - return this.bandwidth; - } - - public String getBackupSetName() { - return backupSetName; - } - - public void setBackupSetName(String backupSetName) { - this.backupSetName = backupSetName; - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java deleted file mode 100644 index c82e05a..0000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java +++ /dev/null @@ -1,104 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import java.io.Serializable; - -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.util.BackupClientUtil; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.BackupProtos; - -/** - * Backup status and related information encapsulated for a table. - * At this moment only TargetDir and SnapshotName is encapsulated here. - */ - -@InterfaceAudience.Private -@InterfaceStability.Evolving -public class BackupStatus implements Serializable { - - private static final long serialVersionUID = -5968397963548535982L; - - // table name for backup - private TableName table; - - // target directory of the backup image for this table - private String targetDir; - - // snapshot name for offline/online snapshot - private String snapshotName = null; - - public BackupStatus() { - - } - - public BackupStatus(TableName table, String targetRootDir, String backupId) { - this.table = table; - this.targetDir = BackupClientUtil.getTableBackupDir(targetRootDir, backupId, table); - } - - public String getSnapshotName() { - return snapshotName; - } - - public void setSnapshotName(String snapshotName) { - this.snapshotName = snapshotName; - } - - public String getTargetDir() { - return targetDir; - } - - public TableName getTable() { - return table; - } - - public void setTable(TableName table) { - this.table = table; - } - - public void setTargetDir(String targetDir) { - this.targetDir = targetDir; - } - - public static BackupStatus convert(BackupProtos.TableBackupStatus proto) - { - BackupStatus bs = new BackupStatus(); - bs.setTable(ProtobufUtil.toTableName(proto.getTable())); - bs.setTargetDir(proto.getTargetDir()); - if(proto.hasSnapshot()){ - bs.setSnapshotName(proto.getSnapshot()); - } - return bs; - } - - public BackupProtos.TableBackupStatus toProto() { - BackupProtos.TableBackupStatus.Builder builder = - BackupProtos.TableBackupStatus.newBuilder(); - if(snapshotName != null) { - builder.setSnapshot(snapshotName); - } - builder.setTable(ProtobufUtil.toProtoTableName(table)); - builder.setTargetDir(targetDir); - return builder.build(); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java deleted file mode 100644 index 7490d20..0000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java +++ /dev/null @@ -1,94 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; - -/** - * POJO class for restore request - */ -@InterfaceAudience.Public -@InterfaceStability.Evolving -public class RestoreRequest { - - private String backupRootDir; - private String backupId; - private boolean check = false; - private TableName[] fromTables; - private TableName[] toTables; - private boolean overwrite = false; - - public RestoreRequest() { - } - - public String getBackupRootDir() { - return backupRootDir; - } - - public RestoreRequest setBackupRootDir(String backupRootDir) { - this.backupRootDir = backupRootDir; - return this; - } - - public String getBackupId() { - return backupId; - } - - public RestoreRequest setBackupId(String backupId) { - this.backupId = backupId; - return this; - } - - public boolean isCheck() { - return check; - } - - public RestoreRequest setCheck(boolean check) { - this.check = check; - return this; - } - - public TableName[] getFromTables() { - return fromTables; - } - - public RestoreRequest setFromTables(TableName[] fromTables) { - this.fromTables = fromTables; - return this; - } - - public TableName[] getToTables() { - return toTables; - } - - public RestoreRequest setToTables(TableName[] toTables) { - this.toTables = toTables; - return this; - } - - public boolean isOverwrite() { - return overwrite; - } - - public RestoreRequest setOverwrite(boolean overwrite) { - this.overwrite = overwrite; - return this; - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java deleted file mode 100644 index 2ff5756..0000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java +++ /dev/null @@ -1,717 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import java.io.IOException; -import java.util.List; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupRequest; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants.BackupCommand; -import org.apache.hadoop.hbase.backup.util.BackupClientUtil; -import org.apache.hadoop.hbase.backup.util.BackupSet; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.BackupAdmin; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; - -import com.google.common.collect.Lists; - -/** - * General backup commands, options and usage messages - */ -@InterfaceAudience.Private -@InterfaceStability.Evolving -public final class BackupCommands { - - public final static String INCORRECT_USAGE = "Incorrect usage"; - - public static final String USAGE = "Usage: hbase backup COMMAND [command-specific arguments]\n" - + "where COMMAND is one of:\n" - + " create create a new backup image\n" - + " delete delete an existing backup image\n" - + " describe show the detailed information of a backup image\n" - + " history show history of all successful backups\n" - + " progress show the progress of the latest backup request\n" - + " set backup set management\n" - + "Run \'hbase backup COMMAND -h\' to see help message for each command\n"; - - public static final String CREATE_CMD_USAGE = - "Usage: hbase backup create <type> <BACKUP_ROOT> [tables] [-set name] " - + "[-w workers][-b bandwith]\n" - + " type \"full\" to create a full backup image\n" - + " \"incremental\" to create an incremental backup image\n" - + " BACKUP_ROOT The full root path to store the backup image,\n" - + " the prefix can be hdfs, webhdfs or gpfs\n" - + "Options:\n" - + " tables If no tables (\"\") are specified, all tables are backed up.\n" - + " Otherwise it is a comma separated list of tables.\n" - + " -w number of parallel workers (MapReduce tasks).\n" - + " -b bandwith per one worker (MapReduce task) in MBs per sec\n" - + " -set name of backup set to use (mutually exclusive with [tables])" ; - - public static final String PROGRESS_CMD_USAGE = "Usage: hbase backup progress <backupId>\n" - + " backupId backup image id\n"; - public static final String NO_INFO_FOUND = "No info was found for backup id: "; - - public static final String DESCRIBE_CMD_USAGE = "Usage: hbase backup decsribe <backupId>\n" - + " backupId backup image id\n"; - - public static final String HISTORY_CMD_USAGE = - "Usage: hbase backup history [-path BACKUP_ROOT] [-n N] [-t table]\n" - + " -n N show up to N last backup sessions, default - 10\n" - + " -path backup root path\n" - + " -t table table name. If specified, only backup images which contain this table\n" - + " will be listed." ; - - - public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete <backupId>\n" - + " backupId backup image id\n"; - - public static final String CANCEL_CMD_USAGE = "Usage: hbase backup cancel <backupId>\n" - + " backupId backup image id\n"; - - public static final String SET_CMD_USAGE = "Usage: hbase backup set COMMAND [name] [tables]\n" - + " name Backup set name\n" - + " tables If no tables (\"\") are specified, all tables will belong to the set.\n" - + " Otherwise it is a comma separated list of tables.\n" - + "COMMAND is one of:\n" - + " add add tables to a set, create a set if needed\n" - + " remove remove tables from a set\n" - + " list list all backup sets in the system\n" - + " describe describe set\n" - + " delete delete backup set\n"; - - public static abstract class Command extends Configured { - CommandLine cmdline; - - Command(Configuration conf) { - super(conf); - } - - public void execute() throws IOException - { - if (cmdline.hasOption("h") || cmdline.hasOption("help")) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - } - - protected abstract void printUsage(); - } - - private BackupCommands() { - throw new AssertionError("Instantiating utility class..."); - } - - public static Command createCommand(Configuration conf, BackupCommand type, CommandLine cmdline) { - Command cmd = null; - switch (type) { - case CREATE: - cmd = new CreateCommand(conf, cmdline); - break; - case DESCRIBE: - cmd = new DescribeCommand(conf, cmdline); - break; - case PROGRESS: - cmd = new ProgressCommand(conf, cmdline); - break; - case DELETE: - cmd = new DeleteCommand(conf, cmdline); - break; - case CANCEL: - cmd = new CancelCommand(conf, cmdline); - break; - case HISTORY: - cmd = new HistoryCommand(conf, cmdline); - break; - case SET: - cmd = new BackupSetCommand(conf, cmdline); - break; - case HELP: - default: - cmd = new HelpCommand(conf, cmdline); - break; - } - return cmd; - } - - static int numOfArgs(String[] args) { - if (args == null) return 0; - return args.length; - } - - public static class CreateCommand extends Command { - - CreateCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - super.execute(); - if (cmdline == null || cmdline.getArgs() == null) { - System.err.println("ERROR: missing arguments"); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - String[] args = cmdline.getArgs(); - if (args.length < 3 || args.length > 4) { - System.err.println("ERROR: wrong number of arguments: "+ args.length); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - if (!BackupType.FULL.toString().equalsIgnoreCase(args[1]) - && !BackupType.INCREMENTAL.toString().equalsIgnoreCase(args[1])) { - System.err.println("ERROR: invalid backup type: "+ args[1]); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String tables = null; - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - - // Check backup set - String setName = null; - if (cmdline.hasOption("set")) { - setName = cmdline.getOptionValue("set"); - tables = getTablesForSet(setName, conf); - - if (tables == null) { - System.err.println("ERROR: Backup set '" + setName+ "' is either empty or does not exist"); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - } else { - tables = (args.length == 4) ? args[3] : null; - } - int bandwidth = cmdline.hasOption('b') ? Integer.parseInt(cmdline.getOptionValue('b')) : -1; - int workers = cmdline.hasOption('w') ? Integer.parseInt(cmdline.getOptionValue('w')) : -1; - - try (Connection conn = ConnectionFactory.createConnection(getConf()); - Admin admin = conn.getAdmin(); - BackupAdmin backupAdmin = admin.getBackupAdmin();) { - BackupRequest request = new BackupRequest(); - request.setBackupType(BackupType.valueOf(args[1].toUpperCase())) - .setTableList(tables != null?Lists.newArrayList(BackupClientUtil.parseTableNames(tables)): null) - .setTargetRootDir(args[2]).setWorkers(workers).setBandwidth(bandwidth) - .setBackupSetName(setName); - String backupId = backupAdmin.backupTables(request); - System.out.println("Backup session "+ backupId+" finished. Status: SUCCESS"); - } catch (IOException e) { - System.err.println("Backup session finished. Status: FAILURE"); - throw e; - } - } - private String getTablesForSet(String name, Configuration conf) - throws IOException { - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupSystemTable table = new BackupSystemTable(conn)) { - List<TableName> tables = table.describeBackupSet(name); - if (tables == null) return null; - return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND); - } - } - - @Override - protected void printUsage() { - System.err.println(CREATE_CMD_USAGE); - } - } - - private static class HelpCommand extends Command { - - HelpCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - super.execute(); - if (cmdline == null) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String[] args = cmdline.getArgs(); - if (args == null || args.length == 0) { - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - if (args.length != 2) { - System.err.println("Only supports help message of a single command type"); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String type = args[1]; - - if (BackupCommand.CREATE.name().equalsIgnoreCase(type)) { - System.out.println(CREATE_CMD_USAGE); - } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(type)) { - System.out.println(DESCRIBE_CMD_USAGE); - } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(type)) { - System.out.println(HISTORY_CMD_USAGE); - } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(type)) { - System.out.println(PROGRESS_CMD_USAGE); - } else if (BackupCommand.DELETE.name().equalsIgnoreCase(type)) { - System.out.println(DELETE_CMD_USAGE); - } else if (BackupCommand.CANCEL.name().equalsIgnoreCase(type)) { - System.out.println(CANCEL_CMD_USAGE); - } else if (BackupCommand.SET.name().equalsIgnoreCase(type)) { - System.out.println(SET_CMD_USAGE); - } else { - System.out.println("Unknown command : " + type); - printUsage(); - } - } - - @Override - protected void printUsage() { - System.err.println(USAGE); - } - } - - private static class DescribeCommand extends Command { - - DescribeCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - super.execute(); - if (cmdline == null || cmdline.getArgs() == null) { - System.err.println("ERROR: missing arguments"); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - String[] args = cmdline.getArgs(); - if (args.length != 2) { - System.err.println("ERROR: wrong number of arguments"); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String backupId = args[1]; - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupAdmin admin = conn.getAdmin().getBackupAdmin();) { - BackupInfo info = admin.getBackupInfo(backupId); - if (info == null) { - System.err.println("ERROR: " + backupId + " does not exist"); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - System.out.println(info.getShortDescription()); - } - } - - @Override - protected void printUsage() { - System.err.println(DESCRIBE_CMD_USAGE); - } - } - - private static class ProgressCommand extends Command { - - ProgressCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - super.execute(); - - if (cmdline == null || cmdline.getArgs() == null || - cmdline.getArgs().length == 1) { - System.err.println("No backup id was specified, " - + "will retrieve the most recent (ongoing) sessions"); - } - String[] args = cmdline.getArgs(); - if (args.length > 2) { - System.err.println("ERROR: wrong number of arguments: " + args.length); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String backupId = (args == null || args.length <= 1) ? null : args[1]; - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - try(final Connection conn = ConnectionFactory.createConnection(conf); - final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){ - int progress = admin.getProgress(backupId); - if(progress < 0){ - System.err.println(NO_INFO_FOUND + backupId); - } else{ - System.out.println(backupId+" progress=" + progress+"%"); - } - } - } - - @Override - protected void printUsage() { - System.err.println(PROGRESS_CMD_USAGE); - } - } - - private static class DeleteCommand extends Command { - - DeleteCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - super.execute(); - if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { - System.err.println("No backup id(s) was specified"); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String[] args = cmdline.getArgs(); - - String[] backupIds = new String[args.length - 1]; - System.arraycopy(args, 1, backupIds, 0, backupIds.length); - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupAdmin admin = conn.getAdmin().getBackupAdmin();) { - int deleted = admin.deleteBackups(args); - System.out.println("Deleted " + deleted + " backups. Total requested: " + args.length); - } - - } - - @Override - protected void printUsage() { - System.err.println(DELETE_CMD_USAGE); - } - } - -// TODO Cancel command - - private static class CancelCommand extends Command { - - CancelCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - super.execute(); - if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { - System.out.println("No backup id(s) was specified, will use the most recent one"); - } - String[] args = cmdline.getArgs(); - String backupId = args == null || args.length == 0 ? null : args[1]; - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupAdmin admin = conn.getAdmin().getBackupAdmin();) { - // TODO cancel backup - } - } - - @Override - protected void printUsage() { - } - } - - private static class HistoryCommand extends Command { - - private final static int DEFAULT_HISTORY_LENGTH = 10; - - HistoryCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - - super.execute(); - - int n = parseHistoryLength(); - final TableName tableName = getTableName(); - final String setName = getTableSetName(); - BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo info) { - if (tableName == null) return true; - List<TableName> names = info.getTableNames(); - return names.contains(tableName); - } - }; - BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() { - @Override - public boolean apply(BackupInfo info) { - if (setName == null) return true; - String backupId = info.getBackupId(); - return backupId.startsWith(setName); - } - }; - Path backupRootPath = getBackupRootPath(); - List<BackupInfo> history = null; - Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create(); - if (backupRootPath == null) { - // Load from hbase:backup - try (final Connection conn = ConnectionFactory.createConnection(conf); - final BackupAdmin admin = conn.getAdmin().getBackupAdmin();) { - - history = admin.getHistory(n, tableNameFilter, tableSetFilter); - } - } else { - // load from backup FS - history = BackupClientUtil.getHistory(conf, n, backupRootPath, - tableNameFilter, tableSetFilter); - } - for (BackupInfo info : history) { - System.out.println(info.getShortDescription()); - } - } - - private Path getBackupRootPath() throws IOException { - String value = null; - try{ - value = cmdline.getOptionValue("path"); - if (value == null) return null; - return new Path(value); - } catch (IllegalArgumentException e) { - System.err.println("ERROR: Illegal argument for backup root path: "+ value); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - } - - private TableName getTableName() throws IOException { - String value = cmdline.getOptionValue("t"); - if (value == null) return null; - try{ - return TableName.valueOf(value); - } catch (IllegalArgumentException e){ - System.err.println("Illegal argument for table name: "+ value); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - } - - private String getTableSetName() throws IOException { - String value = cmdline.getOptionValue("set"); - return value; - } - - private int parseHistoryLength() throws IOException { - String value = cmdline.getOptionValue("n"); - try{ - if (value == null) return DEFAULT_HISTORY_LENGTH; - return Integer.parseInt(value); - } catch(NumberFormatException e) { - System.err.println("Illegal argument for history length: "+ value); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - } - - @Override - protected void printUsage() { - System.err.println(HISTORY_CMD_USAGE); - } - } - - private static class BackupSetCommand extends Command { - private final static String SET_ADD_CMD = "add"; - private final static String SET_REMOVE_CMD = "remove"; - private final static String SET_DELETE_CMD = "delete"; - private final static String SET_DESCRIBE_CMD = "describe"; - private final static String SET_LIST_CMD = "list"; - - BackupSetCommand(Configuration conf, CommandLine cmdline) { - super(conf); - this.cmdline = cmdline; - } - - @Override - public void execute() throws IOException { - super.execute(); - // Command-line must have at least one element - if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) { - System.err.println("ERROR: Command line format"); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String[] args = cmdline.getArgs(); - String cmdStr = args[1]; - BackupCommand cmd = getCommand(cmdStr); - - switch (cmd) { - case SET_ADD: - processSetAdd(args); - break; - case SET_REMOVE: - processSetRemove(args); - break; - case SET_DELETE: - processSetDelete(args); - break; - case SET_DESCRIBE: - processSetDescribe(args); - break; - case SET_LIST: - processSetList(args); - break; - default: - break; - - } - } - - private void processSetList(String[] args) throws IOException { - // List all backup set names - // does not expect any args - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - try(final Connection conn = ConnectionFactory.createConnection(conf); - final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){ - List<BackupSet> list = admin.listBackupSets(); - for(BackupSet bs: list){ - System.out.println(bs); - } - } - } - - private void processSetDescribe(String[] args) throws IOException { - if (args == null || args.length != 3) { - System.err.println("ERROR: Wrong number of args for 'set describe' command: " - + numOfArgs(args)); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - String setName = args[2]; - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - try(final Connection conn = ConnectionFactory.createConnection(conf); - final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){ - BackupSet set = admin.getBackupSet(setName); - if(set == null) { - System.out.println("Set '"+setName+"' does not exist."); - } else{ - System.out.println(set); - } - } - } - - private void processSetDelete(String[] args) throws IOException { - if (args == null || args.length != 3) { - System.err.println("ERROR: Wrong number of args for 'set delete' command: " - + numOfArgs(args)); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - String setName = args[2]; - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - try(final Connection conn = ConnectionFactory.createConnection(conf); - final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){ - boolean result = admin.deleteBackupSet(setName); - if(result){ - System.out.println("Delete set "+setName+" OK."); - } else{ - System.out.println("Set "+setName+" does not exist"); - } - } - } - - private void processSetRemove(String[] args) throws IOException { - if (args == null || args.length != 4) { - System.err.println("ERROR: Wrong number of args for 'set remove' command: " - + numOfArgs(args)); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - - String setName = args[2]; - String[] tables = args[3].split(","); - Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create(); - try(final Connection conn = ConnectionFactory.createConnection(conf); - final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){ - admin.removeFromBackupSet(setName, tables); - } - } - - private void processSetAdd(String[] args) throws IOException { - if (args == null || args.length != 4) { - System.err.println("ERROR: Wrong number of args for 'set add' command: " - + numOfArgs(args)); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - String setName = args[2]; - String[] tables = args[3].split(","); - TableName[] tableNames = new TableName[tables.length]; - for(int i=0; i < tables.length; i++){ - tableNames[i] = TableName.valueOf(tables[i]); - } - Configuration conf = getConf() != null? getConf():HBaseConfiguration.create(); - try(final Connection conn = ConnectionFactory.createConnection(conf); - final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){ - admin.addToBackupSet(setName, tableNames); - } - - } - - private BackupCommand getCommand(String cmdStr) throws IOException { - if (cmdStr.equals(SET_ADD_CMD)) { - return BackupCommand.SET_ADD; - } else if (cmdStr.equals(SET_REMOVE_CMD)) { - return BackupCommand.SET_REMOVE; - } else if (cmdStr.equals(SET_DELETE_CMD)) { - return BackupCommand.SET_DELETE; - } else if (cmdStr.equals(SET_DESCRIBE_CMD)) { - return BackupCommand.SET_DESCRIBE; - } else if (cmdStr.equals(SET_LIST_CMD)) { - return BackupCommand.SET_LIST; - } else { - System.err.println("ERROR: Unknown command for 'set' :" + cmdStr); - printUsage(); - throw new IOException(INCORRECT_USAGE); - } - } - - @Override - protected void printUsage() { - System.err.println(SET_CMD_USAGE); - } - - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java deleted file mode 100644 index ca204b4..0000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java +++ /dev/null @@ -1,86 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import org.apache.hadoop.hbase.HBaseIOException; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; - -/** - * Backup exception - */ -@SuppressWarnings("serial") -@InterfaceAudience.Private -@InterfaceStability.Evolving -public class BackupException extends HBaseIOException { - private BackupInfo description; - - /** - * Some exception happened for a backup and don't even know the backup that it was about - * @param msg Full description of the failure - */ - public BackupException(String msg) { - super(msg); - } - - /** - * Some exception happened for a backup with a cause - * @param cause the cause - */ - public BackupException(Throwable cause) { - super(cause); - } - - /** - * Exception for the given backup that has no previous root cause - * @param msg reason why the backup failed - * @param desc description of the backup that is being failed - */ - public BackupException(String msg, BackupInfo desc) { - super(msg); - this.description = desc; - } - - /** - * Exception for the given backup due to another exception - * @param msg reason why the backup failed - * @param cause root cause of the failure - * @param desc description of the backup that is being failed - */ - public BackupException(String msg, Throwable cause, BackupInfo desc) { - super(msg, cause); - this.description = desc; - } - - /** - * Exception when the description of the backup cannot be determined, due to some other root - * cause - * @param message description of what caused the failure - * @param e root cause - */ - public BackupException(String message, Exception e) { - super(message, e); - } - - public BackupInfo getBackupContext() { - return this.description; - } - -} http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java deleted file mode 100644 index d10713d..0000000 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java +++ /dev/null @@ -1,791 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup.impl; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.TreeMap; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataInputStream; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupInfo; -import org.apache.hadoop.hbase.backup.BackupType; -import org.apache.hadoop.hbase.backup.util.BackupClientUtil; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.exceptions.DeserializationException; -import org.apache.hadoop.hbase.protobuf.ProtobufUtil; -import org.apache.hadoop.hbase.protobuf.generated.BackupProtos; -import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos; - -import com.google.protobuf.InvalidProtocolBufferException; - - -/** - * Backup manifest Contains all the meta data of a backup image. The manifest info will be bundled - * as manifest file together with data. So that each backup image will contain all the info needed - * for restore. - */ -@InterfaceAudience.Private -@InterfaceStability.Evolving -public class BackupManifest { - - private static final Log LOG = LogFactory.getLog(BackupManifest.class); - - // manifest file name - public static final String MANIFEST_FILE_NAME = ".backup.manifest"; - - // manifest file version, current is 1.0 - public static final String MANIFEST_VERSION = "1.0"; - - // backup image, the dependency graph is made up by series of backup images - - public static class BackupImage implements Comparable<BackupImage> { - - private String backupId; - private BackupType type; - private String rootDir; - private List<TableName> tableList; - private long startTs; - private long completeTs; - private ArrayList<BackupImage> ancestors; - - public BackupImage() { - super(); - } - - public BackupImage(String backupId, BackupType type, String rootDir, - List<TableName> tableList, long startTs, long completeTs) { - this.backupId = backupId; - this.type = type; - this.rootDir = rootDir; - this.tableList = tableList; - this.startTs = startTs; - this.completeTs = completeTs; - } - - static BackupImage fromProto(BackupProtos.BackupImage im) { - String backupId = im.getBackupId(); - String rootDir = im.getRootDir(); - long startTs = im.getStartTs(); - long completeTs = im.getCompleteTs(); - List<HBaseProtos.TableName> tableListList = im.getTableListList(); - List<TableName> tableList = new ArrayList<TableName>(); - for(HBaseProtos.TableName tn : tableListList) { - tableList.add(ProtobufUtil.toTableName(tn)); - } - - List<BackupProtos.BackupImage> ancestorList = im.getAncestorsList(); - - BackupType type = - im.getBackupType() == BackupProtos.BackupType.FULL ? BackupType.FULL: - BackupType.INCREMENTAL; - - BackupImage image = new BackupImage(backupId, type, rootDir, tableList, startTs, completeTs); - for(BackupProtos.BackupImage img: ancestorList) { - image.addAncestor(fromProto(img)); - } - return image; - } - - BackupProtos.BackupImage toProto() { - BackupProtos.BackupImage.Builder builder = BackupProtos.BackupImage.newBuilder(); - builder.setBackupId(backupId); - builder.setCompleteTs(completeTs); - builder.setStartTs(startTs); - builder.setRootDir(rootDir); - if (type == BackupType.FULL) { - builder.setBackupType(BackupProtos.BackupType.FULL); - } else{ - builder.setBackupType(BackupProtos.BackupType.INCREMENTAL); - } - - for (TableName name: tableList) { - builder.addTableList(ProtobufUtil.toProtoTableName(name)); - } - - if (ancestors != null){ - for (BackupImage im: ancestors){ - builder.addAncestors(im.toProto()); - } - } - - return builder.build(); - } - - public String getBackupId() { - return backupId; - } - - public void setBackupId(String backupId) { - this.backupId = backupId; - } - - public BackupType getType() { - return type; - } - - public void setType(BackupType type) { - this.type = type; - } - - public String getRootDir() { - return rootDir; - } - - public void setRootDir(String rootDir) { - this.rootDir = rootDir; - } - - public List<TableName> getTableNames() { - return tableList; - } - - public void setTableList(List<TableName> tableList) { - this.tableList = tableList; - } - - public long getStartTs() { - return startTs; - } - - public void setStartTs(long startTs) { - this.startTs = startTs; - } - - public long getCompleteTs() { - return completeTs; - } - - public void setCompleteTs(long completeTs) { - this.completeTs = completeTs; - } - - public ArrayList<BackupImage> getAncestors() { - if (this.ancestors == null) { - this.ancestors = new ArrayList<BackupImage>(); - } - return this.ancestors; - } - - public void addAncestor(BackupImage backupImage) { - this.getAncestors().add(backupImage); - } - - public boolean hasAncestor(String token) { - for (BackupImage image : this.getAncestors()) { - if (image.getBackupId().equals(token)) { - return true; - } - } - return false; - } - - public boolean hasTable(TableName table) { - for (TableName t : tableList) { - if (t.equals(table)) { - return true; - } - } - return false; - } - - @Override - public int compareTo(BackupImage other) { - String thisBackupId = this.getBackupId(); - String otherBackupId = other.getBackupId(); - int index1 = thisBackupId.lastIndexOf("_"); - int index2 = otherBackupId.lastIndexOf("_"); - String name1 = thisBackupId.substring(0, index1); - String name2 = otherBackupId.substring(0, index2); - if(name1.equals(name2)) { - Long thisTS = new Long(thisBackupId.substring(index1 + 1)); - Long otherTS = new Long(otherBackupId.substring(index2 + 1)); - return thisTS.compareTo(otherTS); - } else { - return name1.compareTo(name2); - } - } - } - - // manifest version - private String version = MANIFEST_VERSION; - - // hadoop hbase configuration - protected Configuration config = null; - - // backup root directory - private String rootDir = null; - - // backup image directory - private String tableBackupDir = null; - - // backup log directory if this is an incremental backup - private String logBackupDir = null; - - // backup token - private String backupId; - - // backup type, full or incremental - private BackupType type; - - // the table list for the backup - private ArrayList<TableName> tableList; - - // actual start timestamp of the backup process - private long startTs; - - // actual complete timestamp of the backup process - private long completeTs; - - // the region server timestamp for tables: - // <table, <rs, timestamp>> - private Map<TableName, HashMap<String, Long>> incrTimeRanges; - - // dependency of this backup, including all the dependent images to do PIT recovery - private Map<String, BackupImage> dependency; - - /** - * Construct manifest for a ongoing backup. - * @param backupCtx The ongoing backup context - */ - public BackupManifest(BackupInfo backupCtx) { - this.backupId = backupCtx.getBackupId(); - this.type = backupCtx.getType(); - this.rootDir = backupCtx.getTargetRootDir(); - if (this.type == BackupType.INCREMENTAL) { - this.logBackupDir = backupCtx.getHLogTargetDir(); - } - this.startTs = backupCtx.getStartTs(); - this.completeTs = backupCtx.getEndTs(); - this.loadTableList(backupCtx.getTableNames()); - } - - - /** - * Construct a table level manifest for a backup of the named table. - * @param backupCtx The ongoing backup context - */ - public BackupManifest(BackupInfo backupCtx, TableName table) { - this.backupId = backupCtx.getBackupId(); - this.type = backupCtx.getType(); - this.rootDir = backupCtx.getTargetRootDir(); - this.tableBackupDir = backupCtx.getBackupStatus(table).getTargetDir(); - if (this.type == BackupType.INCREMENTAL) { - this.logBackupDir = backupCtx.getHLogTargetDir(); - } - this.startTs = backupCtx.getStartTs(); - this.completeTs = backupCtx.getEndTs(); - List<TableName> tables = new ArrayList<TableName>(); - tables.add(table); - this.loadTableList(tables); - } - - /** - * Construct manifest from a backup directory. - * @param conf configuration - * @param backupPath backup path - * @throws IOException - */ - - public BackupManifest(Configuration conf, Path backupPath) throws IOException { - this(backupPath.getFileSystem(conf), backupPath); - } - - /** - * Construct manifest from a backup directory. - * @param conf configuration - * @param backupPath backup path - * @throws BackupException exception - */ - - public BackupManifest(FileSystem fs, Path backupPath) throws BackupException { - if (LOG.isDebugEnabled()) { - LOG.debug("Loading manifest from: " + backupPath.toString()); - } - // The input backupDir may not exactly be the backup table dir. - // It could be the backup log dir where there is also a manifest file stored. - // This variable's purpose is to keep the correct and original location so - // that we can store/persist it. - this.tableBackupDir = backupPath.toString(); - this.config = fs.getConf(); - try { - - FileStatus[] subFiles = BackupClientUtil.listStatus(fs, backupPath, null); - if (subFiles == null) { - String errorMsg = backupPath.toString() + " does not exist"; - LOG.error(errorMsg); - throw new IOException(errorMsg); - } - for (FileStatus subFile : subFiles) { - if (subFile.getPath().getName().equals(MANIFEST_FILE_NAME)) { - - // load and set manifest field from file content - FSDataInputStream in = fs.open(subFile.getPath()); - long len = subFile.getLen(); - byte[] pbBytes = new byte[(int) len]; - in.readFully(pbBytes); - BackupProtos.BackupManifest proto = null; - try{ - proto = parseFrom(pbBytes); - } catch(Exception e){ - throw new BackupException(e); - } - this.version = proto.getVersion(); - this.backupId = proto.getBackupId(); - this.type = BackupType.valueOf(proto.getType().name()); - // Here the parameter backupDir is where the manifest file is. - // There should always be a manifest file under: - // backupRootDir/namespace/table/backupId/.backup.manifest - this.rootDir = backupPath.getParent().getParent().getParent().toString(); - - Path p = backupPath.getParent(); - if (p.getName().equals(HConstants.HREGION_LOGDIR_NAME)) { - this.rootDir = p.getParent().toString(); - } else { - this.rootDir = p.getParent().getParent().toString(); - } - - loadTableList(proto); - this.startTs = proto.getStartTs(); - this.completeTs = proto.getCompleteTs(); - loadIncrementalTimestampMap(proto); - loadDependency(proto); - //TODO: merge will be implemented by future jira - LOG.debug("Loaded manifest instance from manifest file: " - + BackupClientUtil.getPath(subFile.getPath())); - return; - } - } - String errorMsg = "No manifest file found in: " + backupPath.toString(); - throw new IOException(errorMsg); - - } catch (IOException e) { - throw new BackupException(e.getMessage()); - } - } - - private void loadIncrementalTimestampMap(BackupProtos.BackupManifest proto) { - List<BackupProtos.TableServerTimestamp> list = proto.getTstMapList(); - if(list == null || list.size() == 0) return; - this.incrTimeRanges = new HashMap<TableName, HashMap<String, Long>>(); - for(BackupProtos.TableServerTimestamp tst: list){ - TableName tn = ProtobufUtil.toTableName(tst.getTable()); - HashMap<String, Long> map = this.incrTimeRanges.get(tn); - if(map == null){ - map = new HashMap<String, Long>(); - this.incrTimeRanges.put(tn, map); - } - List<BackupProtos.ServerTimestamp> listSt = tst.getServerTimestampList(); - for(BackupProtos.ServerTimestamp stm: listSt) { - map.put(stm.getServer(), stm.getTimestamp()); - } - } - } - - private void loadDependency(BackupProtos.BackupManifest proto) { - if(LOG.isDebugEnabled()) { - LOG.debug("load dependency for: "+proto.getBackupId()); - } - - dependency = new HashMap<String, BackupImage>(); - List<BackupProtos.BackupImage> list = proto.getDependentBackupImageList(); - for (BackupProtos.BackupImage im : list) { - BackupImage bim = BackupImage.fromProto(im); - if(im.getBackupId() != null){ - dependency.put(im.getBackupId(), bim); - } else{ - LOG.warn("Load dependency for backup manifest: "+ backupId+ - ". Null backup id in dependent image"); - } - } - } - - private void loadTableList(BackupProtos.BackupManifest proto) { - this.tableList = new ArrayList<TableName>(); - List<HBaseProtos.TableName> list = proto.getTableListList(); - for (HBaseProtos.TableName name: list) { - this.tableList.add(ProtobufUtil.toTableName(name)); - } - } - - public BackupType getType() { - return type; - } - - public void setType(BackupType type) { - this.type = type; - } - - /** - * Loads table list. - * @param tableList Table list - */ - private void loadTableList(List<TableName> tableList) { - - this.tableList = this.getTableList(); - if (this.tableList.size() > 0) { - this.tableList.clear(); - } - for (int i = 0; i < tableList.size(); i++) { - this.tableList.add(tableList.get(i)); - } - - LOG.debug(tableList.size() + " tables exist in table set."); - } - - /** - * Get the table set of this image. - * @return The table set list - */ - public ArrayList<TableName> getTableList() { - if (this.tableList == null) { - this.tableList = new ArrayList<TableName>(); - } - return this.tableList; - } - - /** - * Persist the manifest file. - * @throws IOException IOException when storing the manifest file. - */ - - public void store(Configuration conf) throws BackupException { - byte[] data = toByteArray(); - - // write the file, overwrite if already exist - Path manifestFilePath = - new Path(new Path((this.tableBackupDir != null ? this.tableBackupDir : this.logBackupDir)) - ,MANIFEST_FILE_NAME); - try { - FSDataOutputStream out = - manifestFilePath.getFileSystem(conf).create(manifestFilePath, true); - out.write(data); - out.close(); - } catch (IOException e) { - throw new BackupException(e.getMessage()); - } - - LOG.info("Manifest file stored to " + manifestFilePath); - } - - /** - * Protobuf serialization - * @return The filter serialized using pb - */ - public byte[] toByteArray() { - BackupProtos.BackupManifest.Builder builder = BackupProtos.BackupManifest.newBuilder(); - builder.setVersion(this.version); - builder.setBackupId(this.backupId); - builder.setType(BackupProtos.BackupType.valueOf(this.type.name())); - setTableList(builder); - builder.setStartTs(this.startTs); - builder.setCompleteTs(this.completeTs); - setIncrementalTimestampMap(builder); - setDependencyMap(builder); - return builder.build().toByteArray(); - } - - private void setIncrementalTimestampMap(BackupProtos.BackupManifest.Builder builder) { - if (this.incrTimeRanges == null) { - return; - } - for (Entry<TableName, HashMap<String,Long>> entry: this.incrTimeRanges.entrySet()) { - TableName key = entry.getKey(); - HashMap<String, Long> value = entry.getValue(); - BackupProtos.TableServerTimestamp.Builder tstBuilder = - BackupProtos.TableServerTimestamp.newBuilder(); - tstBuilder.setTable(ProtobufUtil.toProtoTableName(key)); - - for (String s : value.keySet()) { - BackupProtos.ServerTimestamp.Builder stBuilder = BackupProtos.ServerTimestamp.newBuilder(); - stBuilder.setServer(s); - stBuilder.setTimestamp(value.get(s)); - tstBuilder.addServerTimestamp(stBuilder.build()); - } - builder.addTstMap(tstBuilder.build()); - } - } - - private void setDependencyMap(BackupProtos.BackupManifest.Builder builder) { - for (BackupImage image: getDependency().values()) { - builder.addDependentBackupImage(image.toProto()); - } - } - - private void setTableList(BackupProtos.BackupManifest.Builder builder) { - for(TableName name: tableList){ - builder.addTableList(ProtobufUtil.toProtoTableName(name)); - } - } - - /** - * Parse protobuf from byte array - * @param pbBytes A pb serialized BackupManifest instance - * @return An instance of made from <code>bytes</code> - * @throws DeserializationException - */ - private static BackupProtos.BackupManifest parseFrom(final byte[] pbBytes) - throws DeserializationException { - BackupProtos.BackupManifest proto; - try { - proto = BackupProtos.BackupManifest.parseFrom(pbBytes); - } catch (InvalidProtocolBufferException e) { - throw new DeserializationException(e); - } - return proto; - } - - /** - * Get manifest file version - * @return version - */ - public String getVersion() { - return version; - } - - /** - * Get this backup image. - * @return the backup image. - */ - public BackupImage getBackupImage() { - return this.getDependency().get(this.backupId); - } - - /** - * Add dependent backup image for this backup. - * @param image The direct dependent backup image - */ - public void addDependentImage(BackupImage image) { - this.getDependency().get(this.backupId).addAncestor(image); - this.setDependencyMap(this.getDependency(), image); - } - - - - /** - * Get all dependent backup images. The image of this backup is also contained. - * @return The dependent backup images map - */ - public Map<String, BackupImage> getDependency() { - if (this.dependency == null) { - this.dependency = new HashMap<String, BackupImage>(); - LOG.debug(this.rootDir + " " + this.backupId + " " + this.type); - this.dependency.put(this.backupId, - new BackupImage(this.backupId, this.type, this.rootDir, tableList, this.startTs, - this.completeTs)); - } - return this.dependency; - } - - /** - * Set the incremental timestamp map directly. - * @param incrTimestampMap timestamp map - */ - public void setIncrTimestampMap(HashMap<TableName, HashMap<String, Long>> incrTimestampMap) { - this.incrTimeRanges = incrTimestampMap; - } - - - public Map<TableName, HashMap<String, Long>> getIncrTimestampMap() { - if (this.incrTimeRanges == null) { - this.incrTimeRanges = new HashMap<TableName, HashMap<String, Long>>(); - } - return this.incrTimeRanges; - } - - - /** - * Get the image list of this backup for restore in time order. - * @param reverse If true, then output in reverse order, otherwise in time order from old to new - * @return the backup image list for restore in time order - */ - public ArrayList<BackupImage> getRestoreDependentList(boolean reverse) { - TreeMap<Long, BackupImage> restoreImages = new TreeMap<Long, BackupImage>(); - for (BackupImage image : this.getDependency().values()) { - restoreImages.put(Long.valueOf(image.startTs), image); - } - return new ArrayList<BackupImage>(reverse ? (restoreImages.descendingMap().values()) - : (restoreImages.values())); - } - - /** - * Get the dependent image list for a specific table of this backup in time order from old to new - * if want to restore to this backup image level. - * @param table table - * @return the backup image list for a table in time order - */ - public ArrayList<BackupImage> getDependentListByTable(TableName table) { - ArrayList<BackupImage> tableImageList = new ArrayList<BackupImage>(); - ArrayList<BackupImage> imageList = getRestoreDependentList(true); - for (BackupImage image : imageList) { - if (image.hasTable(table)) { - tableImageList.add(image); - if (image.getType() == BackupType.FULL) { - break; - } - } - } - Collections.reverse(tableImageList); - return tableImageList; - } - - /** - * Get the full dependent image list in the whole dependency scope for a specific table of this - * backup in time order from old to new. - * @param table table - * @return the full backup image list for a table in time order in the whole scope of the - * dependency of this image - */ - public ArrayList<BackupImage> getAllDependentListByTable(TableName table) { - ArrayList<BackupImage> tableImageList = new ArrayList<BackupImage>(); - ArrayList<BackupImage> imageList = getRestoreDependentList(false); - for (BackupImage image : imageList) { - if (image.hasTable(table)) { - tableImageList.add(image); - } - } - return tableImageList; - } - - - /** - * Recursively set the dependency map of the backup images. - * @param map The dependency map - * @param image The backup image - */ - private void setDependencyMap(Map<String, BackupImage> map, BackupImage image) { - if (image == null) { - return; - } else { - map.put(image.getBackupId(), image); - for (BackupImage img : image.getAncestors()) { - setDependencyMap(map, img); - } - } - } - - /** - * Check whether backup image1 could cover backup image2 or not. - * @param image1 backup image 1 - * @param image2 backup image 2 - * @return true if image1 can cover image2, otherwise false - */ - public static boolean canCoverImage(BackupImage image1, BackupImage image2) { - // image1 can cover image2 only when the following conditions are satisfied: - // - image1 must not be an incremental image; - // - image1 must be taken after image2 has been taken; - // - table set of image1 must cover the table set of image2. - if (image1.getType() == BackupType.INCREMENTAL) { - return false; - } - if (image1.getStartTs() < image2.getStartTs()) { - return false; - } - List<TableName> image1TableList = image1.getTableNames(); - List<TableName> image2TableList = image2.getTableNames(); - boolean found = false; - for (int i = 0; i < image2TableList.size(); i++) { - found = false; - for (int j = 0; j < image1TableList.size(); j++) { - if (image2TableList.get(i).equals(image1TableList.get(j))) { - found = true; - break; - } - } - if (!found) { - return false; - } - } - - LOG.debug("Backup image " + image1.getBackupId() + " can cover " + image2.getBackupId()); - return true; - } - - /** - * Check whether backup image set could cover a backup image or not. - * @param fullImages The backup image set - * @param image The target backup image - * @return true if fullImages can cover image, otherwise false - */ - public static boolean canCoverImage(ArrayList<BackupImage> fullImages, BackupImage image) { - // fullImages can cover image only when the following conditions are satisfied: - // - each image of fullImages must not be an incremental image; - // - each image of fullImages must be taken after image has been taken; - // - sum table set of fullImages must cover the table set of image. - for (BackupImage image1 : fullImages) { - if (image1.getType() == BackupType.INCREMENTAL) { - return false; - } - if (image1.getStartTs() < image.getStartTs()) { - return false; - } - } - - ArrayList<String> image1TableList = new ArrayList<String>(); - for (BackupImage image1 : fullImages) { - List<TableName> tableList = image1.getTableNames(); - for (TableName table : tableList) { - image1TableList.add(table.getNameAsString()); - } - } - ArrayList<String> image2TableList = new ArrayList<String>(); - List<TableName> tableList = image.getTableNames(); - for (TableName table : tableList) { - image2TableList.add(table.getNameAsString()); - } - - for (int i = 0; i < image2TableList.size(); i++) { - if (image1TableList.contains(image2TableList.get(i)) == false) { - return false; - } - } - - LOG.debug("Full image set can cover image " + image.getBackupId()); - return true; - } - - public BackupInfo toBackupInfo() - { - BackupInfo info = new BackupInfo(); - info.setType(type); - TableName[] tables = new TableName[tableList.size()]; - info.addTables(getTableList().toArray(tables)); - info.setBackupId(backupId); - info.setStartTs(startTs); - info.setTargetRootDir(rootDir); - if(type == BackupType.INCREMENTAL) { - info.setHlogTargetDir(logBackupDir); - } - return info; - } -}