http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java index 2ceeda5..9d9cee0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/WALPlayer.java @@ -85,9 +85,6 @@ public class WALPlayer extends Configured implements Tool { private final static String JOB_NAME_CONF_KEY = "mapreduce.job.name"; - public WALPlayer(){ - } - protected WALPlayer(final Configuration c) { super(c); } @@ -97,7 +94,7 @@ public class WALPlayer extends Configured implements Tool { * This one can be used together with {@link KeyValueSortReducer} */ static class WALKeyValueMapper - extends Mapper<WALKey, WALEdit, ImmutableBytesWritable, KeyValue> { + extends Mapper<WALKey, WALEdit, ImmutableBytesWritable, KeyValue> { private byte[] table; @Override @@ -109,9 +106,7 @@ public class WALPlayer extends Configured implements Tool { if (Bytes.equals(table, key.getTablename().getName())) { for (Cell cell : value.getCells()) { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); - if (WALEdit.isMetaEditFamily(kv)) { - continue; - } + if (WALEdit.isMetaEditFamily(kv)) continue; context.write(new ImmutableBytesWritable(CellUtil.cloneRow(kv)), kv); } } @@ -137,7 +132,7 @@ public class WALPlayer extends Configured implements Tool { * a running HBase instance. */ protected static class WALMapper - extends Mapper<WALKey, WALEdit, ImmutableBytesWritable, Mutation> { + extends Mapper<WALKey, WALEdit, ImmutableBytesWritable, Mutation> { private Map<TableName, TableName> tables = new TreeMap<TableName, TableName>(); @Override @@ -154,9 +149,7 @@ public class WALPlayer extends Configured implements Tool { Cell lastCell = null; for (Cell cell : value.getCells()) { // filtering WAL meta entries - if (WALEdit.isMetaEditFamily(cell)) { - continue; - } + if (WALEdit.isMetaEditFamily(cell)) continue; // Allow a subclass filter out this cell. if (filter(context, cell)) { @@ -167,12 +160,8 @@ public class WALPlayer extends Configured implements Tool { if (lastCell == null || lastCell.getTypeByte() != cell.getTypeByte() || !CellUtil.matchingRow(lastCell, cell)) { // row or type changed, write out aggregate KVs. - if (put != null) { - context.write(tableOut, put); - } - if (del != null) { - context.write(tableOut, del); - } + if (put != null) context.write(tableOut, put); + if (del != null) context.write(tableOut, del); if (CellUtil.isDelete(cell)) { del = new Delete(CellUtil.cloneRow(cell)); } else { @@ -188,12 +177,8 @@ public class WALPlayer extends Configured implements Tool { lastCell = cell; } // write residual KVs - if (put != null) { - context.write(tableOut, put); - } - if (del != null) { - context.write(tableOut, del); - } + if (put != null) context.write(tableOut, put); + if (del != null) context.write(tableOut, del); } } catch (InterruptedException e) { e.printStackTrace(); @@ -201,8 +186,7 @@ public class WALPlayer extends Configured implements Tool { } /** - * Filter cell - * @param cell cell + * @param cell * @return Return true if we are to emit this cell. */ protected boolean filter(Context context, final Cell cell) { @@ -213,7 +197,9 @@ public class WALPlayer extends Configured implements Tool { public void setup(Context context) throws IOException { String[] tableMap = context.getConfiguration().getStrings(TABLE_MAP_KEY); String[] tablesToUse = context.getConfiguration().getStrings(TABLES_KEY); - if (tablesToUse == null || tableMap == null || tablesToUse.length != tableMap.length) { + if (tablesToUse == null && tableMap == null) { + // Then user wants all tables. + } else if (tablesToUse == null || tableMap == null || tablesToUse.length != tableMap.length) { // this can only happen when WALMapper is used directly by a class other than WALPlayer throw new IOException("No tables or incorrect table mapping specified."); } @@ -229,9 +215,7 @@ public class WALPlayer extends Configured implements Tool { void setupTime(Configuration conf, String option) throws IOException { String val = conf.get(option); - if (null == val) { - return; - } + if (null == val) return; long ms; try { // first try to parse in user friendly form @@ -311,8 +295,7 @@ public class WALPlayer extends Configured implements Tool { return job; } - /** - * Print usage + /* * @param errorMsg Error message. Can be null. */ private void usage(final String errorMsg) { @@ -322,8 +305,7 @@ public class WALPlayer extends Configured implements Tool { System.err.println("Usage: " + NAME + " [options] <wal inputdir> <tables> [<tableMappings>]"); System.err.println("Read all WAL entries for <tables>."); System.err.println("If no tables (\"\") are specific, all tables are imported."); - System.err.println("(Careful, even -ROOT- and hbase:meta entries will be imported"+ - " in that case.)"); + System.err.println("(Careful, even -ROOT- and hbase:meta entries will be imported in that case.)"); System.err.println("Otherwise <tables> is a comma separated list of tables.\n"); System.err.println("The WAL entries can be mapped to new set of tables via <tableMapping>."); System.err.println("<tableMapping> is a command separated list of targettables."); @@ -336,10 +318,10 @@ public class WALPlayer extends Configured implements Tool { System.err.println(" -D" + WALInputFormat.START_TIME_KEY + "=[date|ms]"); System.err.println(" -D" + WALInputFormat.END_TIME_KEY + "=[date|ms]"); System.err.println(" -D " + JOB_NAME_CONF_KEY - + "=jobName - use the specified mapreduce job name for the wal player"); + + "=jobName - use the specified mapreduce job name for the wal player"); System.err.println("For performance also consider the following options:\n" - + " -Dmapreduce.map.speculative=false\n" - + " -Dmapreduce.reduce.speculative=false"); + + " -Dmapreduce.map.speculative=false\n" + + " -Dmapreduce.reduce.speculative=false"); } /**
http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java index 5cd38b5..bdb19f4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java @@ -75,7 +75,6 @@ import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableNotDisabledException; import org.apache.hadoop.hbase.TableNotFoundException; import org.apache.hadoop.hbase.UnknownRegionException; -import org.apache.hadoop.hbase.backup.BackupManager; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.client.Result; @@ -385,7 +384,6 @@ public class HMaster extends HRegionServer implements MasterServices, Server { this.conf.setBoolean(HConstants.USE_META_REPLICAS, false); Replication.decorateMasterConfiguration(this.conf); - BackupManager.decorateMasterConfiguration(this.conf); // Hack! Maps DFSClient => Master for logs. HDFS made this // config param for task trackers, but we can piggyback off of it. http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManager.java index b6e11ea..95c3ffe 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManager.java @@ -37,7 +37,7 @@ public abstract class RegionServerProcedureManager extends ProcedureManager { * @param rss Region Server service interface * @throws KeeperException */ - public abstract void initialize(RegionServerServices rss) throws IOException; + public abstract void initialize(RegionServerServices rss) throws KeeperException; /** * Start accepting procedure requests. http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManagerHost.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManagerHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManagerHost.java index adb3604..0f4ea64 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManagerHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/RegionServerProcedureManagerHost.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.procedure.flush.RegionServerFlushTableProcedureManager; import org.apache.hadoop.hbase.regionserver.RegionServerServices; import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager; +import org.apache.zookeeper.KeeperException; /** * Provides the globally barriered procedure framework and environment @@ -38,7 +39,7 @@ public class RegionServerProcedureManagerHost extends private static final Log LOG = LogFactory .getLog(RegionServerProcedureManagerHost.class); - public void initialize(RegionServerServices rss) throws IOException { + public void initialize(RegionServerServices rss) throws KeeperException { for (RegionServerProcedureManager proc : procedures) { LOG.debug("Procedure " + proc.getProcedureSignature() + " is initializing"); proc.initialize(rss); http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java index 3865ba9..085d642 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureCoordinatorRpcs.java @@ -54,7 +54,7 @@ public class ZKProcedureCoordinatorRpcs implements ProcedureCoordinatorRpcs { * @throws KeeperException if an unexpected zk error occurs */ public ZKProcedureCoordinatorRpcs(ZooKeeperWatcher watcher, - String procedureClass, String coordName) throws IOException { + String procedureClass, String coordName) throws KeeperException { this.watcher = watcher; this.procedureType = procedureClass; this.coordName = coordName; http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java index fff75a7..2e03a60 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ZKProcedureMemberRpcs.java @@ -68,54 +68,49 @@ public class ZKProcedureMemberRpcs implements ProcedureMemberRpcs { * @throws KeeperException if we can't reach zookeeper */ public ZKProcedureMemberRpcs(final ZooKeeperWatcher watcher, final String procType) - throws IOException { - try { - this.zkController = new ZKProcedureUtil(watcher, procType) { - @Override - public void nodeCreated(String path) { - if (!isInProcedurePath(path)) { - return; - } + throws KeeperException { + this.zkController = new ZKProcedureUtil(watcher, procType) { + @Override + public void nodeCreated(String path) { + if (!isInProcedurePath(path)) { + return; + } - LOG.info("Received created event:" + path); - // if it is a simple start/end/abort then we just rewatch the node - if (isAcquiredNode(path)) { - waitForNewProcedures(); - return; - } else if (isAbortNode(path)) { - watchForAbortedProcedures(); - return; - } - String parent = ZKUtil.getParent(path); - // if its the end barrier, the procedure can be completed - if (isReachedNode(parent)) { - receivedReachedGlobalBarrier(path); - return; - } else if (isAbortNode(parent)) { - abort(path); - return; - } else if (isAcquiredNode(parent)) { - startNewSubprocedure(path); - } else { - LOG.debug("Ignoring created notification for node:" + path); - } + LOG.info("Received created event:" + path); + // if it is a simple start/end/abort then we just rewatch the node + if (isAcquiredNode(path)) { + waitForNewProcedures(); + return; + } else if (isAbortNode(path)) { + watchForAbortedProcedures(); + return; } + String parent = ZKUtil.getParent(path); + // if its the end barrier, the procedure can be completed + if (isReachedNode(parent)) { + receivedReachedGlobalBarrier(path); + return; + } else if (isAbortNode(parent)) { + abort(path); + return; + } else if (isAcquiredNode(parent)) { + startNewSubprocedure(path); + } else { + LOG.debug("Ignoring created notification for node:" + path); + } + } - @Override - public void nodeChildrenChanged(String path) { - if (path.equals(this.acquiredZnode)) { - LOG.info("Received procedure start children changed event: " + path); - waitForNewProcedures(); - } else if (path.equals(this.abortZnode)) { - LOG.info("Received procedure abort children changed event: " + path); - watchForAbortedProcedures(); - } + @Override + public void nodeChildrenChanged(String path) { + if (path.equals(this.acquiredZnode)) { + LOG.info("Received procedure start children changed event: " + path); + waitForNewProcedures(); + } else if (path.equals(this.abortZnode)) { + LOG.info("Received procedure abort children changed event: " + path); + watchForAbortedProcedures(); } - }; - } catch (KeeperException e) { - // TODO Auto-generated catch block - throw new IOException(e); - } + } + }; } public ZKProcedureUtil getZkController() { http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/RegionServerFlushTableProcedureManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/RegionServerFlushTableProcedureManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/RegionServerFlushTableProcedureManager.java index bd65cc7..1aa959c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/RegionServerFlushTableProcedureManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/flush/RegionServerFlushTableProcedureManager.java @@ -317,7 +317,7 @@ public class RegionServerFlushTableProcedureManager extends RegionServerProcedur * @throws KeeperException if the zookeeper cannot be reached */ @Override - public void initialize(RegionServerServices rss) throws IOException { + public void initialize(RegionServerServices rss) throws KeeperException { this.rss = rss; ZooKeeperWatcher zkw = rss.getZooKeeper(); this.memberRpcs = new ZKProcedureMemberRpcs(zkw, http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 1cd54fa..211fed5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -807,8 +807,8 @@ public class HRegionServer extends HasThread implements rspmHost = new RegionServerProcedureManagerHost(); rspmHost.loadProcedures(conf); rspmHost.initialize(this); - } catch (IOException e) { - this.abort("Failed to reach coordination cluster when creating procedure handler.", e); + } catch (KeeperException e) { + this.abort("Failed to reach zk cluster when creating procedure handler.", e); } // register watcher for recovering regions this.recoveringRegionWatcher = new RecoveringRegionWatcher(this.zooKeeper, this); http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java index e56dd28..537329a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/snapshot/RegionServerSnapshotManager.java @@ -390,7 +390,7 @@ public class RegionServerSnapshotManager extends RegionServerProcedureManager { * @throws KeeperException if the zookeeper cluster cannot be reached */ @Override - public void initialize(RegionServerServices rss) throws IOException { + public void initialize(RegionServerServices rss) throws KeeperException { this.rss = rss; ZooKeeperWatcher zkw = rss.getZooKeeper(); this.memberRpcs = new ZKProcedureMemberRpcs(zkw, http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java index acde21e..9ae72e6 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java @@ -97,8 +97,6 @@ import com.lmax.disruptor.TimeoutException; import com.lmax.disruptor.dsl.Disruptor; import com.lmax.disruptor.dsl.ProducerType; - - /** * Implementation of {@link WAL} to go against {@link FileSystem}; i.e. keep WALs in HDFS. * Only one WAL is ever being written at a time. When a WAL hits a configured maximum size, @@ -361,9 +359,7 @@ public class FSHLog implements WAL { public int compare(Path o1, Path o2) { long t1 = getFileNumFromFileName(o1); long t2 = getFileNumFromFileName(o2); - if (t1 == t2) { - return 0; - } + if (t1 == t2) return 0; return (t1 > t2) ? 1 : -1; } }; @@ -406,7 +402,7 @@ public class FSHLog implements WAL { * @param root path for stored and archived wals * @param logDir dir where wals are stored * @param conf configuration to use - * @throws IOException exception + * @throws IOException */ public FSHLog(final FileSystem fs, final Path root, final String logDir, final Configuration conf) throws IOException { @@ -414,7 +410,7 @@ public class FSHLog implements WAL { } /** - * Create an edit log at the given directory location. + * Create an edit log at the given <code>dir</code> location. * * You should never have to load an existing log. If there is a log at * startup, it should have already been processed and deleted by the time the @@ -429,13 +425,13 @@ public class FSHLog implements WAL { * be registered before we do anything else; e.g. the * Constructor {@link #rollWriter()}. * @param failIfWALExists If true IOException will be thrown if files related to this wal - * already exist. + * already exist. * @param prefix should always be hostname and port in distributed env and - * it will be URL encoded before being used. - * If prefix is null, "wal" will be used + * it will be URL encoded before being used. + * If prefix is null, "wal" will be used * @param suffix will be url encoded. null is treated as empty. non-empty must start with - * {@link DefaultWALProvider#WAL_FILE_NAME_DELIMITER} - * @throws IOException exception + * {@link DefaultWALProvider#WAL_FILE_NAME_DELIMITER} + * @throws IOException */ public FSHLog(final FileSystem fs, final Path rootDir, final String logDir, final String archiveDir, final Configuration conf, @@ -597,9 +593,7 @@ public class FSHLog implements WAL { @VisibleForTesting OutputStream getOutputStream() { FSDataOutputStream fsdos = this.hdfs_out; - if (fsdos == null) { - return null; - } + if (fsdos == null) return null; return fsdos.getWrappedStream(); } @@ -634,7 +628,7 @@ public class FSHLog implements WAL { /** * Tell listeners about pre log roll. - * @throws IOException exception + * @throws IOException */ private void tellListenersAboutPreLogRoll(final Path oldPath, final Path newPath) throws IOException { @@ -647,7 +641,7 @@ public class FSHLog implements WAL { /** * Tell listeners about post log roll. - * @throws IOException exception + * @throws IOException */ private void tellListenersAboutPostLogRoll(final Path oldPath, final Path newPath) throws IOException { @@ -660,7 +654,8 @@ public class FSHLog implements WAL { /** * Run a sync after opening to set up the pipeline. - * @param nextWriter next writer + * @param nextWriter + * @param startTimeNanos */ private void preemptiveSync(final ProtobufLogWriter nextWriter) { long startTimeNanos = System.nanoTime(); @@ -678,9 +673,7 @@ public class FSHLog implements WAL { rollWriterLock.lock(); try { // Return if nothing to flush. - if (!force && (this.writer != null && this.numEntries.get() <= 0)) { - return null; - } + if (!force && (this.writer != null && this.numEntries.get() <= 0)) return null; byte [][] regionsToFlush = null; if (this.closed) { LOG.debug("WAL closed. Skipping rolling of writer"); @@ -735,7 +728,7 @@ public class FSHLog implements WAL { /** * Archive old logs. A WAL is eligible for archiving if all its WALEdits have been flushed. - * @throws IOException exception + * @throws IOException */ private void cleanOldLogs() throws IOException { List<Path> logsToArchive = null; @@ -745,13 +738,9 @@ public class FSHLog implements WAL { Path log = e.getKey(); Map<byte[], Long> sequenceNums = e.getValue(); if (this.sequenceIdAccounting.areAllLower(sequenceNums)) { - if (logsToArchive == null) { - logsToArchive = new ArrayList<Path>(); - } + if (logsToArchive == null) logsToArchive = new ArrayList<Path>(); logsToArchive.add(log); - if (LOG.isTraceEnabled()) { - LOG.trace("WAL file ready for archiving " + log); - } + if (LOG.isTraceEnabled()) LOG.trace("WAL file ready for archiving " + log); } } if (logsToArchive != null) { @@ -781,9 +770,7 @@ public class FSHLog implements WAL { if (regions != null) { StringBuilder sb = new StringBuilder(); for (int i = 0; i < regions.length; i++) { - if (i > 0) { - sb.append(", "); - } + if (i > 0) sb.append(", "); sb.append(Bytes.toStringBinary(regions[i])); } LOG.info("Too many WALs; count=" + logCount + ", max=" + this.maxLogs + @@ -849,9 +836,7 @@ public class FSHLog implements WAL { } } catch (FailedSyncBeforeLogCloseException e) { // If unflushed/unsynced entries on close, it is reason to abort. - if (isUnflushedEntries()) { - throw e; - } + if (isUnflushedEntries()) throw e; LOG.warn("Failed sync-before-close but no outstanding appends; closing WAL: " + e.getMessage()); } @@ -912,9 +897,7 @@ public class FSHLog implements WAL { try { blockOnSync(syncFuture); } catch (IOException ioe) { - if (LOG.isTraceEnabled()) { - LOG.trace("Stale sync exception", ioe); - } + if (LOG.isTraceEnabled()) LOG.trace("Stale sync exception", ioe); } } } @@ -985,15 +968,7 @@ public class FSHLog implements WAL { public Path getCurrentFileName() { return computeFilename(this.filenum.get()); } - - /** - * To support old API compatibility - * @return current file number (timestamp) - */ - public long getFilenum() { - return filenum.get(); - } - + @Override public String toString() { return "FSHLog " + logFilePrefix + ":" + logFileSuffix + "(num " + filenum + ")"; http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCopy.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCopy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCopy.java deleted file mode 100644 index 0360000..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/SnapshotCopy.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.snapshot; - -import org.apache.hadoop.hbase.backup.BackupHandler; - -/* this class will be extended in future jira to support progress report */ -public class SnapshotCopy extends ExportSnapshot { - private BackupHandler backupHandler; - private String table; - - public SnapshotCopy(BackupHandler backupHandler, String table) { - super(); - this.backupHandler = backupHandler; - this.table = table; - } - - public BackupHandler getBackupHandler() { - return this.backupHandler; - } - - public String getTable() { - return this.table; - } - -} http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java index dd4d337..027e7a2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java @@ -209,18 +209,13 @@ public class DefaultWALProvider implements WALProvider { @VisibleForTesting public static long extractFileNumFromWAL(final WAL wal) { final Path walName = ((FSHLog)wal).getCurrentFileName(); - return extractFileNumFromWAL(walName); - } - - @VisibleForTesting - public static long extractFileNumFromWAL(final Path walName) { if (walName == null) { throw new IllegalArgumentException("The WAL path couldn't be null"); } final String[] walPathStrs = walName.toString().split("\\" + WAL_FILE_NAME_DELIMITER); return Long.parseLong(walPathStrs[walPathStrs.length - (isMetaFile(walName) ? 2:1)]); } - + /** * Pattern used to validate a WAL file name * see {@link #validateWALFilename(String)} for description. http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java deleted file mode 100644 index bc0c848..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBase.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.backup.BackupHandler.BACKUPSTATUS; -import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager; -import org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; -import org.junit.AfterClass; -import org.junit.BeforeClass; - -/** - * This class is only a base for other integration-level backup tests. - * Do not add tests here. - * TestBackupSmallTests is where tests that don't require bring machines up/down should go - * All other tests should have their own classes and extend this one - */ -public class TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestBackupBase.class); - - protected static Configuration conf1; - protected static Configuration conf2; - - protected static HBaseTestingUtility TEST_UTIL; - protected static HBaseTestingUtility TEST_UTIL2; - - protected static TableName table1; - protected static TableName table2; - protected static TableName table3; - protected static TableName table4; - - protected static String table1_restore = "table1_restore"; - protected static String table2_restore = "table2_restore"; - protected static String table3_restore = "table3_restore"; - protected static String table4_restore = "table4_restore"; - - protected static final int NB_ROWS_IN_BATCH = 100; - protected static final byte[] qualName = Bytes.toBytes("q1"); - protected static final byte[] famName = Bytes.toBytes("f"); - - protected static String BACKUP_ROOT_DIR = "/backupUT"; - protected static String BACKUP_REMOTE_ROOT_DIR = "/backupUT"; - - protected static final String BACKUP_ZNODE = "/backup/hbase"; - protected static final String BACKUP_SUCCEED_NODE = "complete"; - protected static final String BACKUP_FAILED_NODE = "failed"; - - - /** - * @throws java.lang.Exception - */ - @BeforeClass - public static void setUpBeforeClass() throws Exception { - - TEST_UTIL = new HBaseTestingUtility(); - TEST_UTIL.getConfiguration().set("hbase.procedure.regionserver.classes", - LogRollRegionServerProcedureManager.class.getName()); - TEST_UTIL.getConfiguration().set("hbase.procedure.master.classes", - LogRollMasterProcedureManager.class.getName()); - TEST_UTIL.getConfiguration().set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); - TEST_UTIL.startMiniZKCluster(); - MiniZooKeeperCluster miniZK = TEST_UTIL.getZkCluster(); - - conf1 = TEST_UTIL.getConfiguration(); - conf2 = HBaseConfiguration.create(conf1); - conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2"); - TEST_UTIL2 = new HBaseTestingUtility(conf2); - TEST_UTIL2.setZkCluster(miniZK); - TEST_UTIL.startMiniCluster(); - TEST_UTIL2.startMiniCluster(); - conf1 = TEST_UTIL.getConfiguration(); - - TEST_UTIL.startMiniMapReduceCluster(); - BACKUP_ROOT_DIR = TEST_UTIL.getConfiguration().get("fs.defaultFS") + "/backupUT"; - LOG.info("ROOTDIR " + BACKUP_ROOT_DIR); - BACKUP_REMOTE_ROOT_DIR = TEST_UTIL2.getConfiguration().get("fs.defaultFS") + "/backupUT"; - LOG.info("REMOTE ROOTDIR " + BACKUP_REMOTE_ROOT_DIR); - - BackupClient.setConf(conf1); - RestoreClient.setConf(conf1); - createTables(); - } - - /** - * @throws java.lang.Exception - */ - @AfterClass - public static void tearDownAfterClass() throws Exception { - SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin()); - SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL); - //zkw1.close(); - TEST_UTIL2.shutdownMiniCluster(); - TEST_UTIL.shutdownMiniCluster(); - TEST_UTIL.shutdownMiniMapReduceCluster(); - } - - protected static void loadTable(HTable table) throws Exception { - - Put p; // 100 + 1 row to t1_syncup - for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { - p = new Put(Bytes.toBytes("row" + i)); - p.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - table.put(p); - } - } - - protected static void createTables() throws Exception { - - long tid = System.currentTimeMillis(); - table1 = TableName.valueOf("test-" + tid); - HBaseAdmin ha = TEST_UTIL.getHBaseAdmin(); - HTableDescriptor desc = new HTableDescriptor(table1); - HColumnDescriptor fam = new HColumnDescriptor(famName); - desc.addFamily(fam); - ha.createTable(desc); - Connection conn = ConnectionFactory.createConnection(conf1); - HTable table = (HTable) conn.getTable(table1); - loadTable(table); - table.close(); - table2 = TableName.valueOf("test-" + tid + 1); - desc = new HTableDescriptor(table2); - desc.addFamily(fam); - ha.createTable(desc); - table = (HTable) conn.getTable(table2); - loadTable(table); - table.close(); - table3 = TableName.valueOf("test-" + tid + 2); - table = TEST_UTIL.createTable(table3, famName); - table.close(); - table4 = TableName.valueOf("test-" + tid + 3); - table = TEST_UTIL.createTable(table4, famName); - table.close(); - ha.close(); - conn.close(); - } - - protected boolean checkSucceeded(String backupId) throws IOException - { - BackupContext status = getBackupContext(backupId); - if(status == null) return false; - return status.getFlag() == BACKUPSTATUS.COMPLETE; - } - - protected boolean checkFailed(String backupId) throws IOException - { - BackupContext status = getBackupContext(backupId); - if(status == null) return false; - return status.getFlag() == BACKUPSTATUS.FAILED; - } - - private BackupContext getBackupContext(String backupId) throws IOException - { - Configuration conf = BackupClient.getConf(); - BackupSystemTable table = BackupSystemTable.getTable(conf); - BackupContext status = table.readBackupStatus(backupId); - return status; - } -} - http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java deleted file mode 100644 index 8be07bc..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupBoundaryTests.java +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category(LargeTests.class) -public class TestBackupBoundaryTests extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestBackupBoundaryTests.class); - - /** - * Verify that full backup is created on a single empty table correctly. - * @throws Exception - */ - @Test - public void testFullBackupSingleEmpty() throws Exception { - - LOG.info("create full backup image on single table"); - - String backupId = - BackupClient.create("full", BACKUP_ROOT_DIR, table3.getNameAsString(), null); - LOG.info("Finished Backup"); - assertTrue(checkSucceeded(backupId)); - } - - /** - * Verify that full backup is created on multiple empty tables correctly. - * @throws Exception - */ - @Test - public void testFullBackupMultipleEmpty() throws Exception { - LOG.info("create full backup image on mulitple empty tables"); - String tableset = - table3.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND - + table4.getNameAsString(); - String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null); - assertTrue(checkSucceeded(backupId)); - - } - - /** - * Verify that full backup fails on a single table that does not exist. - * @throws Exception - */ - @Test(expected = RuntimeException.class) - public void testFullBackupSingleDNE() throws Exception { - - LOG.info("test full backup fails on a single table that does not exist"); - BackupClient.create("full", BACKUP_ROOT_DIR, "tabledne", null); - } - - /** - * Verify that full backup fails on multiple tables that do not exist. - * @throws Exception - */ - @Test(expected = RuntimeException.class) - public void testFullBackupMultipleDNE() throws Exception { - - LOG.info("test full backup fails on multiple tables that do not exist"); - BackupClient.create("full", BACKUP_ROOT_DIR, "table1dne,table2dne", null); - } - - /** - * Verify that full backup fails on tableset containing real and fake tables. - * @throws Exception - */ - @Test(expected = RuntimeException.class) - public void testFullBackupMixExistAndDNE() throws Exception { - LOG.info("create full backup fails on tableset containing real and fake table"); - String tableset = - table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND - + "tabledne"; - BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null); - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupLogCleaner.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupLogCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupLogCleaner.java deleted file mode 100644 index 158479b..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupLogCleaner.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.LocatedFileStatus; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.RemoteIterator; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.backup.master.BackupLogCleaner; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.wal.DefaultWALProvider; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -import com.google.common.collect.Iterables; - -@Category(LargeTests.class) -public class TestBackupLogCleaner extends TestBackupBase { - private static final Log LOG = LogFactory.getLog(TestBackupLogCleaner.class); - - // implements all test cases in 1 test since incremental full backup/ - // incremental backup has dependencies - @Test - public void testBackupLogCleaner() throws Exception { - - // #1 - create full backup for all tables - LOG.info("create full backup image for all tables"); - String tablesetFull = - table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND - + table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND - + table3.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND - + table4.getNameAsString(); - - BackupSystemTable systemTable = BackupSystemTable.getTable(TEST_UTIL.getConfiguration()); - // Verify that we have no backup sessions yet - assertFalse(systemTable.hasBackupSessions()); - - List<FileStatus> walFiles = getListOfWALFiles(TEST_UTIL.getConfiguration()); - List<String> swalFiles = convert(walFiles); - BackupLogCleaner cleaner = new BackupLogCleaner(); - cleaner.setConf(TEST_UTIL.getConfiguration()); - - Iterable<FileStatus> deletable = cleaner.getDeletableFiles(walFiles); - // We can delete all files because we do not have yet recorded backup sessions - assertTrue(Iterables.size(deletable) == walFiles.size()); - - systemTable.addWALFiles(swalFiles, "backup"); - String backupIdFull = BackupClient.create("full", BACKUP_ROOT_DIR, tablesetFull, null); - assertTrue(checkSucceeded(backupIdFull)); - // Check one more time - deletable = cleaner.getDeletableFiles(walFiles); - // We can delete wal files because they were saved into hbase:backup table - int size = Iterables.size(deletable); - assertTrue(size == walFiles.size()); - - List<FileStatus> newWalFiles = getListOfWALFiles(TEST_UTIL.getConfiguration()); - LOG.debug("WAL list after full backup"); - convert(newWalFiles); - - // New list of wal files is greater than the previous one, - // because new wal per RS have been opened after full backup - assertTrue(walFiles.size() < newWalFiles.size()); - // TODO : verify that result files are not walFiles collection - Connection conn = ConnectionFactory.createConnection(conf1); - // #2 - insert some data to table - HTable t1 = (HTable) conn.getTable(table1); - Put p1; - for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { - p1 = new Put(Bytes.toBytes("row-t1" + i)); - p1.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - t1.put(p1); - } - - t1.close(); - - HTable t2 = (HTable) conn.getTable(table2); - Put p2; - for (int i = 0; i < 5; i++) { - p2 = new Put(Bytes.toBytes("row-t2" + i)); - p2.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - t2.put(p2); - } - - t2.close(); - - // #3 - incremental backup for multiple tables - String tablesetIncMultiple = - table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND - + table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND - + table3.getNameAsString(); - - String backupIdIncMultiple = - BackupClient.create("incremental", BACKUP_ROOT_DIR, tablesetIncMultiple, null); - assertTrue(checkSucceeded(backupIdIncMultiple)); - deletable = cleaner.getDeletableFiles(newWalFiles); - - assertTrue(Iterables.size(deletable) == newWalFiles.size()); - - conn.close(); - - } - - private List<String> convert(List<FileStatus> walFiles) { - List<String> result = new ArrayList<String>(); - for (FileStatus fs : walFiles) { - LOG.debug("+++WAL: " + fs.getPath().toString()); - result.add(fs.getPath().toString()); - } - return result; - } - - private List<FileStatus> getListOfWALFiles(Configuration c) throws IOException { - Path logRoot = new Path(FSUtils.getRootDir(c), HConstants.HREGION_LOGDIR_NAME); - FileSystem fs = FileSystem.get(c); - RemoteIterator<LocatedFileStatus> it = fs.listFiles(logRoot, true); - List<FileStatus> logFiles = new ArrayList<FileStatus>(); - while (it.hasNext()) { - LocatedFileStatus lfs = it.next(); - if (lfs.isFile() && !DefaultWALProvider.isMetaFile(lfs.getPath())) { - logFiles.add(lfs); - LOG.info(lfs); - } - } - return logFiles; - } - -} http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java deleted file mode 100644 index 7421707..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestBackupSystemTable.java +++ /dev/null @@ -1,341 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.MiniHBaseCluster; -import org.apache.hadoop.hbase.backup.BackupHandler.BACKUPSTATUS; -import org.apache.hadoop.hbase.backup.BackupUtil.BackupCompleteData; -import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** - * Test cases for hbase:backup API - * - */ -@Category(MediumTests.class) -public class TestBackupSystemTable { - - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); - protected static Configuration conf = UTIL.getConfiguration(); - protected static MiniHBaseCluster cluster; - - @BeforeClass - public static void setUp() throws Exception { - cluster = UTIL.startMiniCluster(); - - } - - @Test - public void testUpdateReadDeleteBackupStatus() throws IOException { - BackupSystemTable table = BackupSystemTable.getTable(conf); - BackupContext ctx = createBackupContext(); - table.updateBackupStatus(ctx); - BackupContext readCtx = table.readBackupStatus(ctx.getBackupId()); - assertTrue(compare(ctx, readCtx)); - - // try fake backup id - readCtx = table.readBackupStatus("fake"); - - assertNull(readCtx); - // delete backup context - table.deleteBackupStatus(ctx.getBackupId()); - readCtx = table.readBackupStatus(ctx.getBackupId()); - assertNull(readCtx); - cleanBackupTable(); - } - - @Test - public void testWriteReadBackupStartCode() throws IOException { - BackupSystemTable table = BackupSystemTable.getTable(conf); - String code = "100"; - table.writeBackupStartCode(code); - String readCode = table.readBackupStartCode(); - assertEquals(code, readCode); - cleanBackupTable(); - } - - private void cleanBackupTable() throws IOException { - Admin admin = UTIL.getHBaseAdmin(); - admin.disableTable(BackupSystemTable.getTableName()); - admin.truncateTable(BackupSystemTable.getTableName(), true); - if (admin.isTableDisabled(BackupSystemTable.getTableName())) { - admin.enableTable(BackupSystemTable.getTableName()); - } - } - - @Test - public void testBackupHistory() throws IOException { - BackupSystemTable table = BackupSystemTable.getTable(conf); - int n = 10; - List<BackupContext> list = createBackupContextList(n); - - // Load data - for (BackupContext bc : list) { - // Make sure we set right status - bc.setFlag(BACKUPSTATUS.COMPLETE); - table.updateBackupStatus(bc); - } - - // Reverse list for comparison - Collections.reverse(list); - ArrayList<BackupCompleteData> history = table.getBackupHistory(); - assertTrue(history.size() == n); - - for (int i = 0; i < n; i++) { - BackupContext ctx = list.get(i); - BackupCompleteData data = history.get(i); - assertTrue(compare(ctx, data)); - } - - cleanBackupTable(); - - } - - @Test - public void testRegionServerLastLogRollResults() throws IOException { - BackupSystemTable table = BackupSystemTable.getTable(conf); - - String[] servers = new String[] { "server1", "server2", "server3" }; - String[] timestamps = new String[] { "100", "102", "107" }; - - for (int i = 0; i < servers.length; i++) { - table.writeRegionServerLastLogRollResult(servers[i], timestamps[i]); - } - - HashMap<String, String> result = table.readRegionServerLastLogRollResult(); - assertTrue(servers.length == result.size()); - Set<String> keys = result.keySet(); - String[] keysAsArray = new String[keys.size()]; - keys.toArray(keysAsArray); - Arrays.sort(keysAsArray); - - for (int i = 0; i < keysAsArray.length; i++) { - assertEquals(keysAsArray[i], servers[i]); - String ts1 = timestamps[i]; - String ts2 = result.get(keysAsArray[i]); - assertEquals(ts1, ts2); - } - - cleanBackupTable(); - - } - - @Test - public void testIncrementalBackupTableSet() throws IOException { - BackupSystemTable table = BackupSystemTable.getTable(conf); - - TreeSet<String> tables1 = new TreeSet<String>(); - - tables1.add("t1"); - tables1.add("t2"); - tables1.add("t3"); - - TreeSet<String> tables2 = new TreeSet<String>(); - - tables2.add("t3"); - tables2.add("t4"); - tables2.add("t5"); - - table.addIncrementalBackupTableSet(tables1); - TreeSet<String> res1 = (TreeSet<String>) table.getIncrementalBackupTableSet(); - assertTrue(tables1.size() == res1.size()); - Iterator<String> desc1 = tables1.descendingIterator(); - Iterator<String> desc2 = res1.descendingIterator(); - while (desc1.hasNext()) { - assertEquals(desc1.next(), desc2.next()); - } - - table.addIncrementalBackupTableSet(tables2); - TreeSet<String> res2 = (TreeSet<String>) table.getIncrementalBackupTableSet(); - assertTrue((tables2.size() + tables1.size() - 1) == res2.size()); - - tables1.addAll(tables2); - - desc1 = tables1.descendingIterator(); - desc2 = res2.descendingIterator(); - - while (desc1.hasNext()) { - assertEquals(desc1.next(), desc2.next()); - } - cleanBackupTable(); - - } - - @Test - public void testRegionServerLogTimestampMap() throws IOException { - BackupSystemTable table = BackupSystemTable.getTable(conf); - - TreeSet<String> tables = new TreeSet<String>(); - - tables.add("t1"); - tables.add("t2"); - tables.add("t3"); - - HashMap<String, String> rsTimestampMap = new HashMap<String, String>(); - - rsTimestampMap.put("rs1", "100"); - rsTimestampMap.put("rs2", "101"); - rsTimestampMap.put("rs3", "103"); - - table.writeRegionServerLogTimestamp(tables, rsTimestampMap); - - HashMap<String, HashMap<String, String>> result = table.readLogTimestampMap(); - - assertTrue(tables.size() == result.size()); - - for (String t : tables) { - HashMap<String, String> rstm = result.get(t); - assertNotNull(rstm); - assertEquals(rstm.get("rs1"), "100"); - assertEquals(rstm.get("rs2"), "101"); - assertEquals(rstm.get("rs3"), "103"); - } - - Set<String> tables1 = new TreeSet<String>(); - - tables1.add("t3"); - tables1.add("t4"); - tables1.add("t5"); - - HashMap<String, String> rsTimestampMap1 = new HashMap<String, String>(); - - rsTimestampMap1.put("rs1", "200"); - rsTimestampMap1.put("rs2", "201"); - rsTimestampMap1.put("rs3", "203"); - - table.writeRegionServerLogTimestamp(tables1, rsTimestampMap1); - - result = table.readLogTimestampMap(); - - assertTrue(5 == result.size()); - - for (String t : tables) { - HashMap<String, String> rstm = result.get(t); - assertNotNull(rstm); - if (t.equals("t3") == false) { - assertEquals(rstm.get("rs1"), "100"); - assertEquals(rstm.get("rs2"), "101"); - assertEquals(rstm.get("rs3"), "103"); - } else { - assertEquals(rstm.get("rs1"), "200"); - assertEquals(rstm.get("rs2"), "201"); - assertEquals(rstm.get("rs3"), "203"); - } - } - - for (String t : tables1) { - HashMap<String, String> rstm = result.get(t); - assertNotNull(rstm); - assertEquals(rstm.get("rs1"), "200"); - assertEquals(rstm.get("rs2"), "201"); - assertEquals(rstm.get("rs3"), "203"); - } - - cleanBackupTable(); - - } - - @Test - public void testAddWALFiles() throws IOException { - BackupSystemTable table = BackupSystemTable.getTable(conf); - FileSystem fs = FileSystem.get(conf); - List<String> files = - Arrays.asList("hdfs://server/WALs/srv1,101,15555/srv1,101,15555.default.1", - "hdfs://server/WALs/srv2,102,16666/srv2,102,16666.default.2", - "hdfs://server/WALs/srv3,103,17777/srv3,103,17777.default.3"); - String newFile = "hdfs://server/WALs/srv1,101,15555/srv1,101,15555.default.5"; - - table.addWALFiles(files, "backup"); - - assertTrue(table.checkWALFile(files.get(0))); - assertTrue(table.checkWALFile(files.get(1))); - assertTrue(table.checkWALFile(files.get(2))); - assertFalse(table.checkWALFile(newFile)); - - cleanBackupTable(); - } - - private boolean compare(BackupContext ctx, BackupCompleteData data) { - - return ctx.getBackupId().equals(data.getBackupToken()) - && ctx.getTargetRootDir().equals(data.getBackupRootPath()) - && ctx.getType().equals(data.getType()) - && ctx.getStartTs() == Long.parseLong(data.getStartTime()) - && ctx.getEndTs() == Long.parseLong(data.getEndTime()); - - } - - private boolean compare(BackupContext one, BackupContext two) { - return one.getBackupId().equals(two.getBackupId()) && one.getType().equals(two.getType()) - && one.getTargetRootDir().equals(two.getTargetRootDir()) - && one.getStartTs() == two.getStartTs() && one.getEndTs() == two.getEndTs(); - } - - private BackupContext createBackupContext() { - - BackupContext ctxt = - new BackupContext("backup_" + System.nanoTime(), "full", new String[] { "t1", "t2", "t3" }, - "/hbase/backup", null); - ctxt.setStartTs(System.currentTimeMillis()); - ctxt.setEndTs(System.currentTimeMillis() + 1); - return ctxt; - } - - private List<BackupContext> createBackupContextList(int size) { - List<BackupContext> list = new ArrayList<BackupContext>(); - for (int i = 0; i < size; i++) { - list.add(createBackupContext()); - try { - Thread.sleep(10); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - return list; - } - - @AfterClass - public static void tearDown() throws IOException { - if (cluster != null) cluster.shutdown(); - } - -} http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java deleted file mode 100644 index 19caf37..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackup.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable - * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" - * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License - * for the specific language governing permissions and limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category(LargeTests.class) -public class TestFullBackup extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestFullBackup.class); - - /** - * Verify that full backup is created on a single table with data correctly. - * @throws Exception - */ - @Test - public void testFullBackupSingle() throws Exception { - - LOG.info("test full backup on a single table with data"); - String backupId = - BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(), null); - LOG.info("backup complete"); - assertTrue(checkSucceeded(backupId)); - } - - /** - * Verify that full backup is created on multiple tables correctly. - * @throws Exception - */ - @Test - public void testFullBackupMultiple() throws Exception { - LOG.info("create full backup image on multiple tables with data"); - String tableset = - table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND - + table2.getNameAsString(); - String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null); - assertTrue(checkSucceeded(backupId)); - - } - - /** - * Verify that full backup is created on all tables correctly. - * @throws Exception - */ - @Test - public void testFullBackupAll() throws Exception { - LOG.info("create full backup image on all tables"); - String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, null, null); - assertTrue(checkSucceeded(backupId)); - - } - - /** - * Verify that full backup is created on a table correctly using a snapshot. - * @throws Exception - */ - //@Test - //public void testFullBackupUsingSnapshot() throws Exception { - // HBaseAdmin hba = new HBaseAdmin(conf1); - //String snapshot = "snapshot"; - //hba.snapshot(snapshot, table1); - //LOG.info("create full backup image on a table using snapshot"); - //String backupId = - // BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(), - // snapshot); - // } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java deleted file mode 100644 index 1262c54..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java +++ /dev/null @@ -1,166 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable - * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" - * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License - * for the specific language governing permissions and limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import java.io.IOException; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category(LargeTests.class) -public class TestFullRestore extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestFullRestore.class); - - /** - * Verify that a single table is restored to a new table - * @throws Exception - */ - @Test - public void testFullRestoreSingle() throws Exception { - - LOG.info("test full restore on a single table empty table"); - String backupId = - BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(), null); - LOG.info("backup complete"); - assertTrue(checkSucceeded(backupId)); - - String[] tableset = new String[] { table1.getNameAsString() }; - String[] tablemap = new String[] { table1_restore }; - Path path = new Path(BACKUP_ROOT_DIR); - HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId); - RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, false, tableset, tablemap, - false); - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - assertTrue(hba.tableExists(TableName.valueOf(table1_restore))); - TEST_UTIL.deleteTable(TableName.valueOf(table1_restore)); - hba.close(); - } - - /** - * Verify that multiple tables are restored to new tables. - * @throws Exception - */ - @Test - public void testFullRestoreMultiple() throws Exception { - LOG.info("create full backup image on multiple tables"); - String tableset = - table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND - + table3.getNameAsString(); - String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null); - assertTrue(checkSucceeded(backupId)); - - String[] restore_tableset = new String[] { table2.getNameAsString(), table3.getNameAsString() }; - String[] tablemap = new String[] { table2_restore, table3_restore }; - Path path = new Path(BACKUP_ROOT_DIR); - HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId); - RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, false, - restore_tableset, tablemap, false); - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - assertTrue(hba.tableExists(TableName.valueOf(table2_restore))); - assertTrue(hba.tableExists(TableName.valueOf(table3_restore))); - TEST_UTIL.deleteTable(TableName.valueOf(table2_restore)); - TEST_UTIL.deleteTable(TableName.valueOf(table3_restore)); - hba.close(); - } - - /** - * Verify that a single table is restored using overwrite - * @throws Exception - */ - @Test - public void testFullRestoreSingleOverwrite() throws Exception { - - LOG.info("test full restore on a single table empty table"); - String backupId = - BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(), null); - LOG.info("backup complete"); - assertTrue(checkSucceeded(backupId)); - - String[] tableset = new String[] { table1.getNameAsString() }; - Path path = new Path(BACKUP_ROOT_DIR); - HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId); - RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, false, tableset, null, - true); - } - - /** - * Verify that multiple tables are restored to new tables using overwrite. - * @throws Exception - */ - @Test - public void testFullRestoreMultipleOverwrite() throws Exception { - LOG.info("create full backup image on multiple tables"); - String tableset = - table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND - + table3.getNameAsString(); - String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null); - assertTrue(checkSucceeded(backupId)); - - String[] restore_tableset = new String[] { table2.getNameAsString(), table3.getNameAsString() }; - Path path = new Path(BACKUP_ROOT_DIR); - HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId); - RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, - false, restore_tableset, null, true); - } - - /** - * Verify that restore fails on a single table that does not exist. - * @throws Exception - */ - @Test(expected = IOException.class) - public void testFullRestoreSingleDNE() throws Exception { - - LOG.info("test restore fails on a single table that does not exist"); - String backupId = - BackupClient.create("full", BACKUP_ROOT_DIR, table1.getNameAsString(), null); - LOG.info("backup complete"); - assertTrue(checkSucceeded(backupId)); - - String[] tableset = new String[] { "faketable" }; - String[] tablemap = new String[] { table1_restore }; - Path path = new Path(BACKUP_ROOT_DIR); - HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId); - RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, false, tableset, tablemap, - false); - } - - /** - * Verify that restore fails on multiple tables that do not exist. - * @throws Exception - */ - @Test(expected = IOException.class) - public void testFullRestoreMultipleDNE() throws Exception { - - LOG.info("test restore fails on multiple tables that do not exist"); - String tableset = - table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND - + table3.getNameAsString(); - String backupId = BackupClient.create("full", BACKUP_ROOT_DIR, tableset, null); - assertTrue(checkSucceeded(backupId)); - - String[] restore_tableset = new String[] { "faketable1", "faketable2" }; - String[] tablemap = new String[] { table2_restore, table3_restore }; - Path path = new Path(BACKUP_ROOT_DIR); - HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId); - RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupId, false, - false, restore_tableset, tablemap, false); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java deleted file mode 100644 index 5437b84..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java +++ /dev/null @@ -1,179 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.backup.HBackupFileSystem; -import org.apache.hadoop.fs.Path; -import org.hamcrest.CoreMatchers; -import org.junit.Assert; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category(LargeTests.class) -public class TestIncrementalBackup extends TestBackupBase { - private static final Log LOG = LogFactory.getLog(TestIncrementalBackup.class); - //implement all testcases in 1 test since incremental backup/restore has dependencies - @Test - public void TestIncBackupRestore() throws Exception { - HBackupFileSystem hbfs; - - // #1 - create full backup for all tables - LOG.info("create full backup image for all tables"); - String tablesetFull = - table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND - + table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND - + table3.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND - + table4.getNameAsString(); - - String backupIdFull = - BackupClient.create("full", BACKUP_ROOT_DIR, tablesetFull, null); - assertTrue(checkSucceeded(backupIdFull)); - - Connection conn = ConnectionFactory.createConnection(conf1); - // #2 - insert some data to table - HTable t1 = (HTable) conn.getTable(table1); - Put p1; - for (int i = 0; i < NB_ROWS_IN_BATCH; i++) { - p1 = new Put(Bytes.toBytes("row-t1" + i)); - p1.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - t1.put(p1); - } - - Assert.assertThat(TEST_UTIL.countRows(t1), CoreMatchers.equalTo(NB_ROWS_IN_BATCH * 2)); - t1.close(); - - HTable t2 = (HTable) conn.getTable(table2); - Put p2; - for (int i = 0; i < 5; i++) { - p2 = new Put(Bytes.toBytes("row-t2" + i)); - p2.addColumn(famName, qualName, Bytes.toBytes("val" + i)); - t2.put(p2); - } - - Assert.assertThat(TEST_UTIL.countRows(t2), CoreMatchers.equalTo(NB_ROWS_IN_BATCH + 5)); - t2.close(); - - // #3 - incremental backup for multiple tables - String tablesetIncMultiple = - table1.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND - + table2.getNameAsString() + BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND - + table3.getNameAsString(); - - String backupIdIncMultiple = BackupClient.create("incremental", BACKUP_ROOT_DIR, - tablesetIncMultiple, null); - assertTrue(checkSucceeded(backupIdIncMultiple)); - - - // #4 - restore full backup for all tables, without overwrite - String[] tablesRestoreFull = - new String[] { table1.getNameAsString(), table2.getNameAsString(), - table3.getNameAsString(), table4.getNameAsString() }; - - String[] tablesMapFull = - new String[] { table1_restore, table2_restore, table3_restore, table4_restore }; - - hbfs = new HBackupFileSystem(conf1, new Path(BACKUP_ROOT_DIR), backupIdFull); - RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupIdFull, false, false, - tablesRestoreFull, - tablesMapFull, false); - - // #5.1 - check tables for full restore - HBaseAdmin hAdmin = TEST_UTIL.getHBaseAdmin(); - assertTrue(hAdmin.tableExists(TableName.valueOf(table1_restore))); - assertTrue(hAdmin.tableExists(TableName.valueOf(table2_restore))); - assertTrue(hAdmin.tableExists(TableName.valueOf(table3_restore))); - assertTrue(hAdmin.tableExists(TableName.valueOf(table4_restore))); - - hAdmin.close(); - - // #5.2 - checking row count of tables for full restore - HTable hTable = (HTable) conn.getTable(TableName.valueOf(table1_restore)); - Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH)); - hTable.close(); - - hTable = (HTable) conn.getTable(TableName.valueOf(table2_restore)); - Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH)); - hTable.close(); - - hTable = (HTable) conn.getTable(TableName.valueOf(table3_restore)); - Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0)); - hTable.close(); - - hTable = (HTable) conn.getTable(TableName.valueOf(table4_restore)); - Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0)); - hTable.close(); - - // #6 - restore incremental backup for multiple tables, with overwrite - String[] tablesRestoreIncMultiple = - new String[] - { table1.getNameAsString(), table2.getNameAsString(), table3.getNameAsString() }; - String[] tablesMapIncMultiple = - new String[] { table1_restore, table2_restore, table3_restore }; - hbfs = new HBackupFileSystem(conf1, new Path(BACKUP_ROOT_DIR), backupIdIncMultiple); - RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupIdIncMultiple, false, false, - tablesRestoreIncMultiple, tablesMapIncMultiple, true); - - hTable = (HTable) conn.getTable(TableName.valueOf(table1_restore)); - Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH * 2)); - hTable.close(); - - hTable = (HTable) conn.getTable(TableName.valueOf(table2_restore)); - Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH + 5)); - hTable.close(); - - hTable = (HTable) conn.getTable(TableName.valueOf(table3_restore)); - Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0)); - hTable.close(); - - // #7 - incremental backup for single, empty table - - String tablesetIncEmpty = table4.getNameAsString(); - String backupIdIncEmpty = - BackupClient.create("incremental", BACKUP_ROOT_DIR, tablesetIncEmpty, null); - assertTrue(checkSucceeded(backupIdIncEmpty)); - - - // #8 - restore incremental backup for single empty table, with overwrite - String[] tablesRestoreIncEmpty = new String[] { table4.getNameAsString() }; - String[] tablesMapIncEmpty = new String[] { table4_restore }; - hbfs = new HBackupFileSystem(conf1, new Path(BACKUP_ROOT_DIR), backupIdIncEmpty); - RestoreClient.restore_stage1(hbfs, BACKUP_ROOT_DIR, backupIdIncEmpty, false, false, - tablesRestoreIncEmpty, - tablesMapIncEmpty, true); - - hTable = (HTable) conn.getTable(TableName.valueOf(table4_restore)); - Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(0)); - hTable.close(); - conn.close(); - - } - -} http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java deleted file mode 100644 index 03822a2..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable - * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" - * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License - * for the specific language governing permissions and limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category(LargeTests.class) -public class TestRemoteBackup extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestRemoteBackup.class); - - /** - * Verify that a remote full backup is created on a single table with data correctly. - * @throws Exception - */ - @Test - public void testFullBackupRemote() throws Exception { - - LOG.info("test remote full backup on a single table"); - - // String rootdir = TEST_UTIL2.getDefaultRootDirPath() + BACKUP_ROOT_DIR; - // LOG.info("ROOTDIR " + rootdir); - String backupId = - BackupClient.create("full", BACKUP_REMOTE_ROOT_DIR, table1.getNameAsString(), null); - LOG.info("backup complete"); - assertTrue(checkSucceeded(backupId)); - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/449fb812/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java deleted file mode 100644 index e1315c5..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable - * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" - * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License - * for the specific language governing permissions and limitations under the License. - */ - -package org.apache.hadoop.hbase.backup; - -import static org.junit.Assert.assertTrue; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.testclassification.LargeTests; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -@Category(LargeTests.class) -public class TestRemoteRestore extends TestBackupBase { - - private static final Log LOG = LogFactory.getLog(TestRemoteRestore.class); - - /** - * Verify that a remote restore on a single table is successful. - * @throws Exception - */ - @Test - public void testFullRestoreRemote() throws Exception { - - LOG.info("test remote full backup on a single table"); - String backupId = - BackupClient.create("full", BACKUP_REMOTE_ROOT_DIR, table1.getNameAsString(), null); - LOG.info("backup complete"); - assertTrue(checkSucceeded(backupId)); - String[] tableset = new String[] { table1.getNameAsString() }; - String[] tablemap = new String[] { table1_restore }; - Path path = new Path(BACKUP_REMOTE_ROOT_DIR); - HBackupFileSystem hbfs = new HBackupFileSystem(conf1, path, backupId); - RestoreClient.restore_stage1(hbfs, BACKUP_REMOTE_ROOT_DIR, backupId, false, false, tableset, - tablemap, false); - HBaseAdmin hba = TEST_UTIL.getHBaseAdmin(); - assertTrue(hba.tableExists(TableName.valueOf(table1_restore))); - TEST_UTIL.deleteTable(TableName.valueOf(table1_restore)); - hba.close(); - } - -} \ No newline at end of file