hbase git commit: HBASE-16454 Compactor's shipping logic decision should be based on the current store's block size (Ram)

2016-08-22 Thread ramkrishna 
Repository: hbase
Updated Branches:
  refs/heads/master 2aae923c3 -> ff6182b02


HBASE-16454 Compactor's shipping logic decision should be based on the
current store's block size (Ram)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ff6182b0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ff6182b0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ff6182b0

Branch: refs/heads/master
Commit: ff6182b020c0d75f8d077eaeef7ae84124e1d4ce
Parents: 2aae923
Author: Ramkrishna 
Authored: Tue Aug 23 11:06:02 2016 +0530
Committer: Ramkrishna 
Committed: Tue Aug 23 11:06:02 2016 +0530

--
 .../hadoop/hbase/mob/DefaultMobStoreCompactor.java  |  5 +++--
 .../hbase/regionserver/compactions/Compactor.java   | 12 +---
 2 files changed, 8 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/ff6182b0/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
index 711f31d..55aea00 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreCompactor.java
@@ -160,12 +160,13 @@ public class DefaultMobStoreCompactor extends 
DefaultCompactor {
* @param cleanSeqId When true, remove seqId(used to be mvcc) value which is 
<= smallestReadPoint
* @param throughputController The compaction throughput controller.
* @param major Is a major compaction.
+   * @param numofFilesToCompact the number of files to compact
* @return Whether compaction ended; false if it was interrupted for any 
reason.
*/
   @Override
   protected boolean performCompaction(FileDetails fd, InternalScanner scanner, 
CellSink writer,
-  long smallestReadPoint, boolean cleanSeqId,
-  ThroughputController throughputController,  boolean major) throws 
IOException {
+  long smallestReadPoint, boolean cleanSeqId, ThroughputController 
throughputController,
+  boolean major, int numofFilesToCompact) throws IOException {
 if (!(scanner instanceof MobCompactionStoreScanner)) {
   throw new IllegalArgumentException(
 "The scanner should be an instance of MobCompactionStoreScanner");

http://git-wip-us.apache.org/repos/asf/hbase/blob/ff6182b0/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
index 1796bca..c695788 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/Compactor.java
@@ -310,7 +310,7 @@ public abstract class Compactor {
   }
   writer = sinkFactory.createWriter(scanner, fd, 
store.throttleCompaction(request.getSize()));
   finished = performCompaction(fd, scanner, writer, smallestReadPoint, 
cleanSeqId,
-throughputController, request.isAllFiles());
+throughputController, request.isAllFiles(), request.getFiles().size());
   if (!finished) {
 throw new InterruptedIOException("Aborting compaction of store " + 
store + " in region "
 + store.getRegionInfo().getRegionNameAsString() + " because it was 
interrupted.");
@@ -385,11 +385,12 @@ public abstract class Compactor {
* @param cleanSeqId When true, remove seqId(used to be mvcc) value which is 
<=
*  smallestReadPoint
* @param major Is a major compaction.
+   * @param numofFilesToCompact the number of files to compact
* @return Whether compaction ended; false if it was interrupted for some 
reason.
*/
   protected boolean performCompaction(FileDetails fd, InternalScanner scanner, 
CellSink writer,
-  long smallestReadPoint, boolean cleanSeqId,
-  ThroughputController throughputController, boolean major) throws 
IOException {
+  long smallestReadPoint, boolean cleanSeqId, ThroughputController 
throughputController,
+  boolean major, int numofFilesToCompact) throws IOException {
 long bytesWrittenProgressForCloseCheck = 0;
 long bytesWrittenProgressForLog = 0;
 long bytesWrittenProgressForShippedCall = 0;
@@ -409,10 +410,7 @@ public abstract class Compactor {
 
 throughputController.start(compactionName);
 KeyValueScanner kvs = (scanner instanceof KeyValu

hbase git commit: HBASE-16444 CellUtil#estimatedSerializedSizeOfKey() should consider KEY_INFRASTRUCTURE_SIZ (Ram)

2016-08-22 Thread ramkrishna 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 a8909f5cb -> e7cd7fffb


HBASE-16444 CellUtil#estimatedSerializedSizeOfKey() should consider
KEY_INFRASTRUCTURE_SIZ (Ram)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e7cd7fff
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e7cd7fff
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e7cd7fff

Branch: refs/heads/branch-1
Commit: e7cd7fffbf4b297e54438c1378f0b9aa2c98523b
Parents: a8909f5
Author: Ramkrishna 
Authored: Tue Aug 23 10:53:33 2016 +0530
Committer: Ramkrishna 
Committed: Tue Aug 23 10:58:50 2016 +0530

--
 .../main/java/org/apache/hadoop/hbase/CellUtil.java   | 14 +++---
 1 file changed, 11 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e7cd7fff/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
index ff6ad90..be50069 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
@@ -526,6 +526,7 @@ public final class CellUtil {
   }
 
   /**
+   * Estimate based on keyvalue's serialization format. 
* @param cell
* @return Estimate of the cell size in bytes.
* @deprecated please use estimatedSerializedSizeOf(Cell)
@@ -549,7 +550,7 @@ public final class CellUtil {
 return getSumOfCellElementLengths(cell) +
   // Use the KeyValue's infrastructure size presuming that another 
implementation would have
   // same basic cost.
-  KeyValue.KEY_INFRASTRUCTURE_SIZE +
+  KeyValue.ROW_LENGTH_SIZE + KeyValue.FAMILY_LENGTH_SIZE +
   // Serialization is probably preceded by a length (it is in the 
KeyValueCodec at least).
   Bytes.SIZEOF_INT;
   }
@@ -573,10 +574,17 @@ public final class CellUtil {
 KeyValue.TIMESTAMP_TYPE_SIZE;
   }
 
+  /**
+   * Calculates the serialized key size. We always serialize in the KeyValue's 
serialization
+   * format.
+   * @param cell the cell for which the key size has to be calculated.
+   * @return the key size
+   */
   public static int estimatedSerializedSizeOfKey(final Cell cell) {
 if (cell instanceof KeyValue) return ((KeyValue)cell).getKeyLength();
-// This will be a low estimate.  Will do for now.
-return getSumOfCellKeyElementLengths(cell);
+return cell.getRowLength() + cell.getFamilyLength() +
+cell.getQualifierLength() +
+KeyValue.KEY_INFRASTRUCTURE_SIZE;
   }
 
   /**

hbase git commit: HBASE-16444 CellUtil#estimatedSerializedSizeOfKey() should consider KEY_INFRASTRUCTURE_SIZ (Ram)

2016-08-22 Thread ramkrishna 
Repository: hbase
Updated Branches:
  refs/heads/master abc64fa69 -> 2aae923c3


HBASE-16444 CellUtil#estimatedSerializedSizeOfKey() should consider
KEY_INFRASTRUCTURE_SIZ (Ram)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2aae923c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2aae923c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2aae923c

Branch: refs/heads/master
Commit: 2aae923c32528667644c57085a0180bede1ad262
Parents: abc64fa
Author: Ramkrishna 
Authored: Tue Aug 23 10:53:33 2016 +0530
Committer: Ramkrishna 
Committed: Tue Aug 23 10:53:33 2016 +0530

--
 .../main/java/org/apache/hadoop/hbase/CellUtil.java   | 14 +++---
 1 file changed, 11 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2aae923c/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
--
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
index c6a0a93..ac55112 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
@@ -902,6 +902,7 @@ public final class CellUtil {
   }
 
   /**
+   * Estimate based on keyvalue's serialization format. 
* @param cell
* @return Estimate of the cell size in bytes.
*/
@@ -915,7 +916,7 @@ public final class CellUtil {
 return getSumOfCellElementLengths(cell) +
   // Use the KeyValue's infrastructure size presuming that another 
implementation would have
   // same basic cost.
-  KeyValue.KEY_INFRASTRUCTURE_SIZE +
+  KeyValue.ROW_LENGTH_SIZE + KeyValue.FAMILY_LENGTH_SIZE +
   // Serialization is probably preceded by a length (it is in the 
KeyValueCodec at least).
   Bytes.SIZEOF_INT;
   }
@@ -939,10 +940,17 @@ public final class CellUtil {
 KeyValue.TIMESTAMP_TYPE_SIZE;
   }
 
+  /**
+   * Calculates the serialized key size. We always serialize in the KeyValue's 
serialization
+   * format.
+   * @param cell the cell for which the key size has to be calculated.
+   * @return the key size
+   */
   public static int estimatedSerializedSizeOfKey(final Cell cell) {
 if (cell instanceof KeyValue) return ((KeyValue)cell).getKeyLength();
-// This will be a low estimate.  Will do for now.
-return getSumOfCellKeyElementLengths(cell);
+return cell.getRowLength() + cell.getFamilyLength() +
+cell.getQualifierLength() +
+KeyValue.KEY_INFRASTRUCTURE_SIZE;
   }
 
   /**

hbase git commit: HBASE-16450 Shell tool to dump replication queues

2016-08-22 Thread esteban 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 b30ad10a9 -> c3ccbbc8d


HBASE-16450 Shell tool to dump replication queues

New tool to dump existing replication peers, configurations and
queues when using HBase Replication. The tool provides two flags:

 --distributed  This flag will poll each RS for information about
the replication queues being processed on this RS.
By default this is not enabled and the information
about the replication queues and configuration will
be obtained from ZooKeeper.
 --hdfs When --distributed is used, this flag will attempt
to calculate the total size of the WAL files used
by the replication queues. Since its possible that
multiple peers can be configured this value can be
overestimated.

Signed-off-by: Matteo Bertozzi 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c3ccbbc8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c3ccbbc8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c3ccbbc8

Branch: refs/heads/branch-1.3
Commit: c3ccbbc8dc2a8fbbdf54dc82458aa7bb477a8a6a
Parents: b30ad10
Author: Esteban Gutierrez 
Authored: Mon Aug 22 19:53:29 2016 -0700
Committer: Esteban Gutierrez 
Committed: Mon Aug 22 19:53:44 2016 -0700

--
 .../replication/ReplicationQueuesZKImpl.java|   2 +-
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java   |  11 +
 .../regionserver/DumpReplicationQueues.java | 355 +++
 3 files changed, 367 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/c3ccbbc8/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
index 0833bca..beeaea0 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
@@ -161,7 +161,7 @@ public class ReplicationQueuesZKImpl extends 
ReplicationStateZKBase implements R
   return ZKUtil.parseWALPositionFrom(bytes);
 } catch (DeserializationException de) {
   LOG.warn("Failed to parse WALPosition for queueId=" + queueId + " and 
wal=" + filename
-  + "znode content, continuing.");
+  + " znode content, continuing.");
 }
 // if we can not parse the position, start at the beginning of the wal file
 // again

http://git-wip-us.apache.org/repos/asf/hbase/blob/c3ccbbc8/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index ff837cb..66ddb62 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -1825,6 +1825,17 @@ public class ZKUtil {
 }
   }
 
+  /**
+   * Returns a string with replication znodes and position of the replication 
log
+   * @param zkw
+   * @return aq string of replication znodes and log positions
+   */
+  public static String getReplicationZnodesDump(ZooKeeperWatcher zkw) throws 
KeeperException {
+StringBuilder sb = new StringBuilder();
+getReplicationZnodesDump(zkw, sb);
+return sb.toString();
+  }
+
   private static void appendRSZnodes(ZooKeeperWatcher zkw, String znode, 
StringBuilder sb)
   throws KeeperException {
 List stack = new LinkedList();

http://git-wip-us.apache.org/repos/asf/hbase/blob/c3ccbbc8/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
new file mode 100644
index 000..bf38d6f
--- /dev/null
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
@@ -0,0 +1,355 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ *

hbase git commit: HBASE-16450 Shell tool to dump replication queues

2016-08-22 Thread esteban 
Repository: hbase
Updated Branches:
  refs/heads/master ae42a934d -> abc64fa69


HBASE-16450 Shell tool to dump replication queues

New tool to dump existing replication peers, configurations and
queues when using HBase Replication. The tool provides two flags:

 --distributed  This flag will poll each RS for information about
the replication queues being processed on this RS.
By default this is not enabled and the information
about the replication queues and configuration will
be obtained from ZooKeeper.
 --hdfs When --distributed is used, this flag will attempt
to calculate the total size of the WAL files used
by the replication queues. Since its possible that
multiple peers can be configured this value can be
overestimated.

Signed-off-by: Matteo Bertozzi 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/abc64fa6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/abc64fa6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/abc64fa6

Branch: refs/heads/master
Commit: abc64fa69fcdff41611e1b4bd64e67f54617a28a
Parents: ae42a93
Author: Esteban Gutierrez 
Authored: Thu Aug 18 15:28:18 2016 -0700
Committer: Esteban Gutierrez 
Committed: Mon Aug 22 19:50:10 2016 -0700

--
 .../replication/ReplicationQueuesZKImpl.java|   2 +-
 .../apache/hadoop/hbase/zookeeper/ZKUtil.java   |  11 +
 .../regionserver/DumpReplicationQueues.java | 356 +++
 3 files changed, 368 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/abc64fa6/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
index c1e85cb..1c579ab 100644
--- 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
+++ 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/replication/ReplicationQueuesZKImpl.java
@@ -165,7 +165,7 @@ public class ReplicationQueuesZKImpl extends 
ReplicationStateZKBase implements R
   return ZKUtil.parseWALPositionFrom(bytes);
 } catch (DeserializationException de) {
   LOG.warn("Failed to parse WALPosition for queueId=" + queueId + " and 
wal=" + filename
-  + "znode content, continuing.");
+  + " znode content, continuing.");
 }
 // if we can not parse the position, start at the beginning of the wal file
 // again

http://git-wip-us.apache.org/repos/asf/hbase/blob/abc64fa6/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
index f5b720e..f6914ae 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/zookeeper/ZKUtil.java
@@ -1823,6 +1823,17 @@ public class ZKUtil {
 }
   }
 
+  /**
+   * Returns a string with replication znodes and position of the replication 
log
+   * @param zkw
+   * @return aq string of replication znodes and log positions
+   */
+  public static String getReplicationZnodesDump(ZooKeeperWatcher zkw) throws 
KeeperException {
+StringBuilder sb = new StringBuilder();
+getReplicationZnodesDump(zkw, sb);
+return sb.toString();
+  }
+
   private static void appendRSZnodes(ZooKeeperWatcher zkw, String znode, 
StringBuilder sb)
   throws KeeperException {
 List stack = new LinkedList();

http://git-wip-us.apache.org/repos/asf/hbase/blob/abc64fa6/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
new file mode 100644
index 000..5305149
--- /dev/null
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/replication/regionserver/DumpReplicationQueues.java
@@ -0,0 +1,356 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you

hbase git commit: HBASE-16255 Backup/Restore IT (Vladimir Rodionov)

2016-08-22 Thread tedyu 
Repository: hbase
Updated Branches:
  refs/heads/HBASE-7912 cd2712e43 -> 7c6b10ff4


HBASE-16255 Backup/Restore IT (Vladimir Rodionov)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7c6b10ff
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7c6b10ff
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7c6b10ff

Branch: refs/heads/HBASE-7912
Commit: 7c6b10ff44404db384987698155d7b96741dd1d8
Parents: cd2712e
Author: tedyu 
Authored: Mon Aug 22 18:40:10 2016 -0700
Committer: tedyu 
Committed: Mon Aug 22 18:40:10 2016 -0700

--
 .../hbase/IntegrationTestBackupRestore.java | 293 +++
 .../hadoop/hbase/HBaseTestingUtility.java   |  31 +-
 2 files changed, 321 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/7c6b10ff/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java
--
diff --git 
a/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java
 
b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java
new file mode 100644
index 000..52f1c32
--- /dev/null
+++ 
b/hbase-it/src/test/java/org/apache/hadoop/hbase/IntegrationTestBackupRestore.java
@@ -0,0 +1,293 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.backup.BackupInfo;
+import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
+import org.apache.hadoop.hbase.backup.BackupRequest;
+import org.apache.hadoop.hbase.backup.BackupType;
+import org.apache.hadoop.hbase.backup.RestoreRequest;
+import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.BackupAdmin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.testclassification.IntegrationTests;
+import org.apache.hadoop.hbase.util.RegionSplitter;
+import org.apache.hadoop.hbase.util.RegionSplitter.SplitAlgorithm;
+import org.apache.hadoop.util.ToolRunner;
+import org.hamcrest.CoreMatchers;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.common.base.Objects;
+import com.google.common.collect.Lists;
+
+/**
+ * An integration test to detect regressions in HBASE-7912. Create
+ * a table with many regions, load data, perform series backup/load operations,
+ * then restore and verify data
+ * @see https://issues.apache.org/jira/browse/HBASE-7912";>HBASE-7912
+ */
+@Category(IntegrationTests.class)
+public class IntegrationTestBackupRestore extends IntegrationTestBase {
+  private static final String CLASS_NAME = 
IntegrationTestBackupRestore.class.getSimpleName();
+  protected static final Log LOG = 
LogFactory.getLog(IntegrationTestBackupRestore.class);
+  protected static final TableName TABLE_NAME1 = TableName.valueOf(CLASS_NAME 
+ ".table1");
+  protected static final TableName TABLE_NAME2 = TableName.valueOf(CLASS_NAME 
+ ".table2");
+  protected static final String COLUMN_NAME = "f";
+  protected static final String REGION_COUNT_KEY = 
String.format("hbase.%s.regions.perRS", CLASS_NAME);
+  protected static final String REGIONSERVER_COUNT_KEY = 
String.format("hbase.%s.regionServers",
+CLASS_NAME);
+  protected static final int DEFAULT_REGION_COUNT = 10;
+  protected static final int DEFAULT_REGIONSERVER_COUNT = 5;
+  protected static int regionsCoun

hbase git commit: HBASE-16467 Move AbstractHBaseTool to hbase-common.

2016-08-22 Thread appy 
Repository: hbase
Updated Branches:
  refs/heads/0.98 3186d3d70 -> ac3ec6333


HBASE-16467 Move AbstractHBaseTool to hbase-common.

Change-Id: I49da7ab465c029028af37fb701058a7c68ebeb2f


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ac3ec633
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ac3ec633
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ac3ec633

Branch: refs/heads/0.98
Commit: ac3ec633388eb6160efa2ba593207ce5fef7ecfc
Parents: 3186d3d
Author: Apekshit Sharma 
Authored: Mon Aug 22 17:22:04 2016 -0700
Committer: Apekshit Sharma 
Committed: Mon Aug 22 17:22:04 2016 -0700

--
 .../hadoop/hbase/util/AbstractHBaseTool.java| 203 +++
 .../hadoop/hbase/util/AbstractHBaseTool.java| 203 ---
 2 files changed, 203 insertions(+), 203 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/ac3ec633/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
new file mode 100644
index 000..7e7523a
--- /dev/null
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Common base class used for HBase command-line tools. Simplifies workflow and
+ * command-line argument parsing.
+ */
+@InterfaceAudience.Private
+public abstract class AbstractHBaseTool implements Tool {
+
+  protected static final int EXIT_SUCCESS = 0;
+  protected static final int EXIT_FAILURE = 1;
+
+  private static final String SHORT_HELP_OPTION = "h";
+  private static final String LONG_HELP_OPTION = "help";
+
+  private static final Log LOG = LogFactory.getLog(AbstractHBaseTool.class);
+
+  private final Options options = new Options();
+
+  protected Configuration conf = null;
+
+  private static final Set requiredOptions = new TreeSet();
+
+  protected String[] cmdLineArgs = null;
+
+  /**
+   * Override this to add command-line options using {@link #addOptWithArg}
+   * and similar methods.
+   */
+  protected abstract void addOptions();
+
+  /**
+   * This method is called to process the options after they have been parsed.
+   */
+  protected abstract void processOptions(CommandLine cmd);
+
+  /** The "main function" of the tool */
+  protected abstract int doWork() throws Exception;
+
+  @Override
+  public Configuration getConf() {
+return conf;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+this.conf = conf;
+  }
+
+  @Override
+  public final int run(String[] args) throws IOException {
+if (conf == null) {
+  LOG.error("Tool configuration is not initialized");
+  throw new NullPointerException("conf");
+}
+
+CommandLine cmd;
+try {
+  // parse the command line arguments
+  cmd = parseArgs(args);
+  cmdLineArgs = args;
+} catch (ParseException e) {
+  LOG.error("Error when parsing command-line arguemnts", e);
+  printUsage();
+  return EXIT_FAILURE;
+}
+
+if (cmd.hasOption(SHORT_HELP_OPTION) || cmd.hasOption(LONG_HELP_OPTION) ||
+!sanityCheckOp

hbase git commit: HBASE-16467 Move AbstractHBaseTool to hbase-common.

2016-08-22 Thread appy 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.0 a049e518f -> cf2e2fde0


HBASE-16467 Move AbstractHBaseTool to hbase-common.

Change-Id: Iea3a3cd91fe4ecc9ff253e2795ab885945fa6a25


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cf2e2fde
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cf2e2fde
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cf2e2fde

Branch: refs/heads/branch-1.0
Commit: cf2e2fde0d66ebc8abc773f0a70825ac4a38523c
Parents: a049e51
Author: Apekshit Sharma 
Authored: Mon Aug 22 17:19:40 2016 -0700
Committer: Apekshit Sharma 
Committed: Mon Aug 22 17:19:40 2016 -0700

--
 .../hadoop/hbase/util/AbstractHBaseTool.java| 203 +++
 .../hadoop/hbase/util/AbstractHBaseTool.java| 203 ---
 2 files changed, 203 insertions(+), 203 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/cf2e2fde/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
new file mode 100644
index 000..a876aef
--- /dev/null
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Common base class used for HBase command-line tools. Simplifies workflow and
+ * command-line argument parsing.
+ */
+@InterfaceAudience.Private
+public abstract class AbstractHBaseTool implements Tool {
+
+  protected static final int EXIT_SUCCESS = 0;
+  protected static final int EXIT_FAILURE = 1;
+
+  private static final String SHORT_HELP_OPTION = "h";
+  private static final String LONG_HELP_OPTION = "help";
+
+  private static final Log LOG = LogFactory.getLog(AbstractHBaseTool.class);
+
+  private final Options options = new Options();
+
+  protected Configuration conf = null;
+
+  private static final Set requiredOptions = new TreeSet();
+
+  protected String[] cmdLineArgs = null;
+
+  /**
+   * Override this to add command-line options using {@link #addOptWithArg}
+   * and similar methods.
+   */
+  protected abstract void addOptions();
+
+  /**
+   * This method is called to process the options after they have been parsed.
+   */
+  protected abstract void processOptions(CommandLine cmd);
+
+  /** The "main function" of the tool */
+  protected abstract int doWork() throws Exception;
+
+  @Override
+  public Configuration getConf() {
+return conf;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+this.conf = conf;
+  }
+
+  @Override
+  public final int run(String[] args) throws IOException {
+if (conf == null) {
+  LOG.error("Tool configuration is not initialized");
+  throw new NullPointerException("conf");
+}
+
+CommandLine cmd;
+try {
+  // parse the command line arguments
+  cmd = parseArgs(args);
+  cmdLineArgs = args;
+} catch (ParseException e) {
+  LOG.error("Error when parsing command-line arguments", e);
+  printUsage();
+  return EXIT_FAILURE;
+}
+
+if (cmd.hasOption(SHORT_HELP_OPTION) || cmd.hasOption(LONG_HELP_OPTION) ||
+!s

hbase git commit: HBASE-16467 Move AbstractHBaseTool to hbase-common.

2016-08-22 Thread appy 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.1 1c92f4c0b -> 5317c7b31


HBASE-16467 Move AbstractHBaseTool to hbase-common.

Change-Id: I90cbf4a8bf7b6b715930dcd97708a121fd89fa2f


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5317c7b3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5317c7b3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5317c7b3

Branch: refs/heads/branch-1.1
Commit: 5317c7b31421872972786775ae08f8ebad52fec7
Parents: 1c92f4c
Author: Apekshit Sharma 
Authored: Mon Aug 22 17:18:28 2016 -0700
Committer: Apekshit Sharma 
Committed: Mon Aug 22 17:18:28 2016 -0700

--
 .../hadoop/hbase/util/AbstractHBaseTool.java| 203 +++
 .../hadoop/hbase/util/AbstractHBaseTool.java| 203 ---
 2 files changed, 203 insertions(+), 203 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/5317c7b3/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
new file mode 100644
index 000..a876aef
--- /dev/null
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Common base class used for HBase command-line tools. Simplifies workflow and
+ * command-line argument parsing.
+ */
+@InterfaceAudience.Private
+public abstract class AbstractHBaseTool implements Tool {
+
+  protected static final int EXIT_SUCCESS = 0;
+  protected static final int EXIT_FAILURE = 1;
+
+  private static final String SHORT_HELP_OPTION = "h";
+  private static final String LONG_HELP_OPTION = "help";
+
+  private static final Log LOG = LogFactory.getLog(AbstractHBaseTool.class);
+
+  private final Options options = new Options();
+
+  protected Configuration conf = null;
+
+  private static final Set requiredOptions = new TreeSet();
+
+  protected String[] cmdLineArgs = null;
+
+  /**
+   * Override this to add command-line options using {@link #addOptWithArg}
+   * and similar methods.
+   */
+  protected abstract void addOptions();
+
+  /**
+   * This method is called to process the options after they have been parsed.
+   */
+  protected abstract void processOptions(CommandLine cmd);
+
+  /** The "main function" of the tool */
+  protected abstract int doWork() throws Exception;
+
+  @Override
+  public Configuration getConf() {
+return conf;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+this.conf = conf;
+  }
+
+  @Override
+  public final int run(String[] args) throws IOException {
+if (conf == null) {
+  LOG.error("Tool configuration is not initialized");
+  throw new NullPointerException("conf");
+}
+
+CommandLine cmd;
+try {
+  // parse the command line arguments
+  cmd = parseArgs(args);
+  cmdLineArgs = args;
+} catch (ParseException e) {
+  LOG.error("Error when parsing command-line arguments", e);
+  printUsage();
+  return EXIT_FAILURE;
+}
+
+if (cmd.hasOption(SHORT_HELP_OPTION) || cmd.hasOption(LONG_HELP_OPTION) ||
+!s

hbase git commit: HBASE-16467 Move AbstractHBaseTool to hbase-common.

2016-08-22 Thread appy 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 444910ec7 -> 4ef97c32b


HBASE-16467 Move AbstractHBaseTool to hbase-common.

Change-Id: Ia4f504456ee99a3e38aa6ac642efb23a1fd4bd31


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/4ef97c32
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/4ef97c32
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/4ef97c32

Branch: refs/heads/branch-1.2
Commit: 4ef97c32bfbb42311cd8d38236c5e47ad3f56078
Parents: 444910e
Author: Apekshit Sharma 
Authored: Mon Aug 22 16:37:11 2016 -0700
Committer: Apekshit Sharma 
Committed: Mon Aug 22 16:37:11 2016 -0700

--
 .../hadoop/hbase/util/AbstractHBaseTool.java| 203 +++
 .../hadoop/hbase/util/AbstractHBaseTool.java| 203 ---
 2 files changed, 203 insertions(+), 203 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/4ef97c32/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
new file mode 100644
index 000..a876aef
--- /dev/null
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Common base class used for HBase command-line tools. Simplifies workflow and
+ * command-line argument parsing.
+ */
+@InterfaceAudience.Private
+public abstract class AbstractHBaseTool implements Tool {
+
+  protected static final int EXIT_SUCCESS = 0;
+  protected static final int EXIT_FAILURE = 1;
+
+  private static final String SHORT_HELP_OPTION = "h";
+  private static final String LONG_HELP_OPTION = "help";
+
+  private static final Log LOG = LogFactory.getLog(AbstractHBaseTool.class);
+
+  private final Options options = new Options();
+
+  protected Configuration conf = null;
+
+  private static final Set requiredOptions = new TreeSet();
+
+  protected String[] cmdLineArgs = null;
+
+  /**
+   * Override this to add command-line options using {@link #addOptWithArg}
+   * and similar methods.
+   */
+  protected abstract void addOptions();
+
+  /**
+   * This method is called to process the options after they have been parsed.
+   */
+  protected abstract void processOptions(CommandLine cmd);
+
+  /** The "main function" of the tool */
+  protected abstract int doWork() throws Exception;
+
+  @Override
+  public Configuration getConf() {
+return conf;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+this.conf = conf;
+  }
+
+  @Override
+  public final int run(String[] args) throws IOException {
+if (conf == null) {
+  LOG.error("Tool configuration is not initialized");
+  throw new NullPointerException("conf");
+}
+
+CommandLine cmd;
+try {
+  // parse the command line arguments
+  cmd = parseArgs(args);
+  cmdLineArgs = args;
+} catch (ParseException e) {
+  LOG.error("Error when parsing command-line arguments", e);
+  printUsage();
+  return EXIT_FAILURE;
+}
+
+if (cmd.hasOption(SHORT_HELP_OPTION) || cmd.hasOption(LONG_HELP_OPTION) ||
+!s

hbase git commit: HBASE-16467 Move AbstractHBaseTool to hbase-common.

2016-08-22 Thread appy 
Repository: hbase
Updated Branches:
  refs/heads/branch-1.3 aba2b0337 -> b30ad10a9


HBASE-16467 Move AbstractHBaseTool to hbase-common.

Change-Id: I3aa79a08ef44b7120f1c0d2a7ab94508e6ee7302


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b30ad10a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b30ad10a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b30ad10a

Branch: refs/heads/branch-1.3
Commit: b30ad10a99389bb1e014c11583a2485cc8cf87fe
Parents: aba2b03
Author: Apekshit Sharma 
Authored: Mon Aug 22 16:34:13 2016 -0700
Committer: Apekshit Sharma 
Committed: Mon Aug 22 16:34:13 2016 -0700

--
 .../hadoop/hbase/util/AbstractHBaseTool.java| 203 +++
 .../hadoop/hbase/util/AbstractHBaseTool.java| 203 ---
 2 files changed, 203 insertions(+), 203 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/b30ad10a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
new file mode 100644
index 000..a876aef
--- /dev/null
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Common base class used for HBase command-line tools. Simplifies workflow and
+ * command-line argument parsing.
+ */
+@InterfaceAudience.Private
+public abstract class AbstractHBaseTool implements Tool {
+
+  protected static final int EXIT_SUCCESS = 0;
+  protected static final int EXIT_FAILURE = 1;
+
+  private static final String SHORT_HELP_OPTION = "h";
+  private static final String LONG_HELP_OPTION = "help";
+
+  private static final Log LOG = LogFactory.getLog(AbstractHBaseTool.class);
+
+  private final Options options = new Options();
+
+  protected Configuration conf = null;
+
+  private static final Set requiredOptions = new TreeSet();
+
+  protected String[] cmdLineArgs = null;
+
+  /**
+   * Override this to add command-line options using {@link #addOptWithArg}
+   * and similar methods.
+   */
+  protected abstract void addOptions();
+
+  /**
+   * This method is called to process the options after they have been parsed.
+   */
+  protected abstract void processOptions(CommandLine cmd);
+
+  /** The "main function" of the tool */
+  protected abstract int doWork() throws Exception;
+
+  @Override
+  public Configuration getConf() {
+return conf;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+this.conf = conf;
+  }
+
+  @Override
+  public final int run(String[] args) throws IOException {
+if (conf == null) {
+  LOG.error("Tool configuration is not initialized");
+  throw new NullPointerException("conf");
+}
+
+CommandLine cmd;
+try {
+  // parse the command line arguments
+  cmd = parseArgs(args);
+  cmdLineArgs = args;
+} catch (ParseException e) {
+  LOG.error("Error when parsing command-line arguments", e);
+  printUsage();
+  return EXIT_FAILURE;
+}
+
+if (cmd.hasOption(SHORT_HELP_OPTION) || cmd.hasOption(LONG_HELP_OPTION) ||
+!s

hbase git commit: HBASE-16467 Move AbstractHBaseTool to hbase-common.

2016-08-22 Thread appy 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 2a35019a3 -> a8909f5cb


HBASE-16467 Move AbstractHBaseTool to hbase-common.

Change-Id: Ifb81c15b89b308f53e6ee8a76e18189b5b526cd3


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a8909f5c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a8909f5c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a8909f5c

Branch: refs/heads/branch-1
Commit: a8909f5cbad6f68426d433861319063272762336
Parents: 2a35019
Author: Apekshit Sharma 
Authored: Mon Aug 22 16:32:46 2016 -0700
Committer: Apekshit Sharma 
Committed: Mon Aug 22 16:32:46 2016 -0700

--
 .../hadoop/hbase/util/AbstractHBaseTool.java| 203 +++
 .../hadoop/hbase/util/AbstractHBaseTool.java| 203 ---
 2 files changed, 203 insertions(+), 203 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/a8909f5c/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
new file mode 100644
index 000..a876aef
--- /dev/null
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Common base class used for HBase command-line tools. Simplifies workflow and
+ * command-line argument parsing.
+ */
+@InterfaceAudience.Private
+public abstract class AbstractHBaseTool implements Tool {
+
+  protected static final int EXIT_SUCCESS = 0;
+  protected static final int EXIT_FAILURE = 1;
+
+  private static final String SHORT_HELP_OPTION = "h";
+  private static final String LONG_HELP_OPTION = "help";
+
+  private static final Log LOG = LogFactory.getLog(AbstractHBaseTool.class);
+
+  private final Options options = new Options();
+
+  protected Configuration conf = null;
+
+  private static final Set requiredOptions = new TreeSet();
+
+  protected String[] cmdLineArgs = null;
+
+  /**
+   * Override this to add command-line options using {@link #addOptWithArg}
+   * and similar methods.
+   */
+  protected abstract void addOptions();
+
+  /**
+   * This method is called to process the options after they have been parsed.
+   */
+  protected abstract void processOptions(CommandLine cmd);
+
+  /** The "main function" of the tool */
+  protected abstract int doWork() throws Exception;
+
+  @Override
+  public Configuration getConf() {
+return conf;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+this.conf = conf;
+  }
+
+  @Override
+  public final int run(String[] args) throws IOException {
+if (conf == null) {
+  LOG.error("Tool configuration is not initialized");
+  throw new NullPointerException("conf");
+}
+
+CommandLine cmd;
+try {
+  // parse the command line arguments
+  cmd = parseArgs(args);
+  cmdLineArgs = args;
+} catch (ParseException e) {
+  LOG.error("Error when parsing command-line arguments", e);
+  printUsage();
+  return EXIT_FAILURE;
+}
+
+if (cmd.hasOption(SHORT_HELP_OPTION) || cmd.hasOption(LONG_HELP_OPTION) ||
+!sanit

hbase git commit: HBASE-16467 Move AbstractHBaseTool to hbase-common.

2016-08-22 Thread appy 
Repository: hbase
Updated Branches:
  refs/heads/master f174fec39 -> ae42a934d


HBASE-16467 Move AbstractHBaseTool to hbase-common.

Change-Id: I64775f875f6900520e66c22c007704a067e8c2a4


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ae42a934
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ae42a934
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ae42a934

Branch: refs/heads/master
Commit: ae42a934d81e6b79cc07e6ff3f74383ed3e306e2
Parents: f174fec
Author: Apekshit Sharma 
Authored: Mon Aug 22 12:38:14 2016 -0700
Committer: Apekshit Sharma 
Committed: Mon Aug 22 16:06:17 2016 -0700

--
 .../hadoop/hbase/util/AbstractHBaseTool.java| 203 +++
 .../hadoop/hbase/util/AbstractHBaseTool.java| 203 ---
 2 files changed, 203 insertions(+), 203 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/ae42a934/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
new file mode 100644
index 000..a876aef
--- /dev/null
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/AbstractHBaseTool.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import java.io.IOException;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Common base class used for HBase command-line tools. Simplifies workflow and
+ * command-line argument parsing.
+ */
+@InterfaceAudience.Private
+public abstract class AbstractHBaseTool implements Tool {
+
+  protected static final int EXIT_SUCCESS = 0;
+  protected static final int EXIT_FAILURE = 1;
+
+  private static final String SHORT_HELP_OPTION = "h";
+  private static final String LONG_HELP_OPTION = "help";
+
+  private static final Log LOG = LogFactory.getLog(AbstractHBaseTool.class);
+
+  private final Options options = new Options();
+
+  protected Configuration conf = null;
+
+  private static final Set requiredOptions = new TreeSet();
+
+  protected String[] cmdLineArgs = null;
+
+  /**
+   * Override this to add command-line options using {@link #addOptWithArg}
+   * and similar methods.
+   */
+  protected abstract void addOptions();
+
+  /**
+   * This method is called to process the options after they have been parsed.
+   */
+  protected abstract void processOptions(CommandLine cmd);
+
+  /** The "main function" of the tool */
+  protected abstract int doWork() throws Exception;
+
+  @Override
+  public Configuration getConf() {
+return conf;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+this.conf = conf;
+  }
+
+  @Override
+  public final int run(String[] args) throws IOException {
+if (conf == null) {
+  LOG.error("Tool configuration is not initialized");
+  throw new NullPointerException("conf");
+}
+
+CommandLine cmd;
+try {
+  // parse the command line arguments
+  cmd = parseArgs(args);
+  cmdLineArgs = args;
+} catch (ParseException e) {
+  LOG.error("Error when parsing command-line arguments", e);
+  printUsage();
+  return EXIT_FAILURE;
+}
+
+if (cmd.hasOption(SHORT_HELP_OPTION) || cmd.hasOption(LONG_HELP_OPTION) ||
+!sanityChe

hbase git commit: HBASE-14450 Addendum aligns with API from HBASE-16455

2016-08-22 Thread tedyu 
Repository: hbase
Updated Branches:
  refs/heads/HBASE-7912 0309725e8 -> cd2712e43


HBASE-14450 Addendum aligns with API from HBASE-16455


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/cd2712e4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/cd2712e4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/cd2712e4

Branch: refs/heads/HBASE-7912
Commit: cd2712e43a62a6736bfd62da7b9aa4b060d81b6b
Parents: 0309725
Author: tedyu 
Authored: Mon Aug 22 10:03:37 2016 -0700
Committer: tedyu 
Committed: Mon Aug 22 10:03:37 2016 -0700

--
 .../regionserver/LogRollBackupSubprocedure.java | 21 +---
 .../hbase/regionserver/HRegionServer.java   |  5 +++--
 .../regionserver/RegionServerServices.java  |  6 ++
 .../hadoop/hbase/wal/DefaultWALProvider.java| 10 +-
 .../hadoop/hbase/wal/DisabledWALProvider.java   |  7 +--
 .../hbase/wal/RegionGroupingProvider.java   | 15 ++
 .../org/apache/hadoop/hbase/wal/WALFactory.java |  4 ++--
 .../apache/hadoop/hbase/wal/WALProvider.java|  5 ++---
 .../hadoop/hbase/MockRegionServerServices.java  |  6 ++
 .../hadoop/hbase/master/MockRegionServer.java   |  6 ++
 .../apache/hadoop/hbase/wal/IOTestProvider.java |  7 +--
 11 files changed, 60 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/cd2712e4/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java
index 2a94ab2..ed5c44e 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/regionserver/LogRollBackupSubprocedure.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hbase.backup.regionserver;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.concurrent.Callable;
 
 import org.apache.commons.logging.Log;
@@ -34,6 +35,7 @@ import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
 import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.wal.WAL;
 
 /**
  * This backup subprocedure implementation forces a log roll on the RS.
@@ -73,20 +75,25 @@ public class LogRollBackupSubprocedure extends Subprocedure 
{
   if (LOG.isDebugEnabled()) {
 LOG.debug("++ DRPC started: " + rss.getServerName());
   }
-  hlog = (FSHLog) rss.getWAL(null);
-  long filenum = hlog.getFilenum();
-  long highest = ((HRegionServer)rss).getHighestFilenum();
+  List wals = rss.getWALs();
+  long highest = 0;
+  for (WAL wal : wals) {
+if (wal == null) continue;
+if (((FSHLog)wal).getFilenum() > highest) {
+  highest = ((FSHLog)wal).getFilenum();
+}
+  }
 
-  LOG.info("Trying to roll log in backup subprocedure, current log number: 
" + filenum
-  + " highest: " + highest + " on " + rss.getServerName());
+  LOG.info("Trying to roll log in backup subprocedure, "
+  + " highest log num: " + highest + " on " + rss.getServerName());
   ((HRegionServer)rss).walRoller.requestRollAll();
   long start = EnvironmentEdgeManager.currentTime();
   while (!((HRegionServer)rss).walRoller.walRollFinished()) {
 Thread.sleep(20);
   }
   LOG.debug("log roll took " + 
(EnvironmentEdgeManager.currentTime()-start));
-  LOG.info("After roll log in backup subprocedure, current log number: " + 
hlog.getFilenum()
-  + " highest: " + ((HRegionServer)rss).getHighestFilenum() + " on " + 
rss.getServerName());
+  LOG.info("After roll log in backup subprocedure, "
+  + " on " + rss.getServerName());
 
   Connection connection = rss.getConnection();
   try(final BackupSystemTable table = new BackupSystemTable(connection)) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/cd2712e4/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index f6a1395..36d2c53 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/re

hbase git commit: HBASE-16455 Provide API for obtaining all the WAL files

2016-08-22 Thread tedyu 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 d260108e3 -> 2a35019a3


HBASE-16455 Provide API for obtaining all the WAL files


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2a35019a
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2a35019a
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2a35019a

Branch: refs/heads/branch-1
Commit: 2a35019a31e5616b364a0363b3ee1be1d3aaefcf
Parents: d260108
Author: tedyu 
Authored: Mon Aug 22 06:56:14 2016 -0700
Committer: tedyu 
Committed: Mon Aug 22 06:56:14 2016 -0700

--
 .../apache/hadoop/hbase/regionserver/HRegionServer.java |  4 
 .../hadoop/hbase/regionserver/RegionServerServices.java |  1 +
 .../apache/hadoop/hbase/regionserver/wal/FSHLog.java|  7 +++
 .../org/apache/hadoop/hbase/wal/DefaultWALProvider.java | 12 
 .../apache/hadoop/hbase/wal/DisabledWALProvider.java|  8 
 .../apache/hadoop/hbase/wal/RegionGroupingProvider.java | 10 ++
 .../java/org/apache/hadoop/hbase/wal/WALFactory.java|  4 
 .../java/org/apache/hadoop/hbase/wal/WALProvider.java   |  4 
 .../apache/hadoop/hbase/master/MockRegionServer.java|  1 -
 .../org/apache/hadoop/hbase/wal/IOTestProvider.java |  8 
 10 files changed, 58 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/2a35019a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 5dda768..03dcd4e 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1882,6 +1882,10 @@ public class HRegionServer extends HasThread implements
 
   private static final byte[] UNSPECIFIED_REGION = new byte[]{};
 
+  public List getWALs() throws IOException {
+return walFactory.getWALs();
+  }
+
   @Override
   public WAL getWAL(HRegionInfo regionInfo) throws IOException {
 WAL wal;

http://git-wip-us.apache.org/repos/asf/hbase/blob/2a35019a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
index 993b072..99a7b12 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;

http://git-wip-us.apache.org/repos/asf/hbase/blob/2a35019a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
index 097101b..ceced2f 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/FSHLog.java
@@ -986,6 +986,13 @@ public class FSHLog implements WAL {
 return computeFilename(this.filenum.get());
   }
 
+  /**
+   * @return current file number (timestamp)
+   */
+  public long getFilenum() {
+return filenum.get();
+  }
+  
   @Override
   public String toString() {
 return "FSHLog " + logFilePrefix + ":" + logFileSuffix + "(num " + filenum 
+ ")";

http://git-wip-us.apache.org/repos/asf/hbase/blob/2a35019a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
index b618a0f..4b355d3 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/DefaultWALProvider.java
@@ -22,6 +22,8 @@ import java.io.Closeable;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.util.ArrayList;
+impor

hbase git commit: HBASE-16455 Provide API for obtaining all the WAL files

2016-08-22 Thread tedyu 
Repository: hbase
Updated Branches:
  refs/heads/master d077219d3 -> f174fec39


HBASE-16455 Provide API for obtaining all the WAL files


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f174fec3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f174fec3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f174fec3

Branch: refs/heads/master
Commit: f174fec391f604f56c12cbc3ad6062d7fe640866
Parents: d077219
Author: tedyu 
Authored: Mon Aug 22 01:34:58 2016 -0700
Committer: tedyu 
Committed: Mon Aug 22 01:34:58 2016 -0700

--
 .../apache/hadoop/hbase/regionserver/HRegionServer.java |  5 +
 .../hadoop/hbase/regionserver/RegionServerServices.java |  6 ++
 .../hadoop/hbase/regionserver/wal/AbstractFSWAL.java|  4 
 .../apache/hadoop/hbase/wal/AbstractFSWALProvider.java  | 12 
 .../apache/hadoop/hbase/wal/DisabledWALProvider.java|  8 
 .../apache/hadoop/hbase/wal/RegionGroupingProvider.java | 10 ++
 .../java/org/apache/hadoop/hbase/wal/WALFactory.java|  4 
 .../java/org/apache/hadoop/hbase/wal/WALProvider.java   |  4 
 .../apache/hadoop/hbase/MockRegionServerServices.java   |  6 ++
 .../apache/hadoop/hbase/master/MockRegionServer.java|  7 ++-
 .../org/apache/hadoop/hbase/wal/IOTestProvider.java |  8 
 11 files changed, 73 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/f174fec3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 89bfbf3..257e724 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -1906,6 +1906,11 @@ public class HRegionServer extends HasThread implements
   private static final byte[] UNSPECIFIED_REGION = new byte[]{};
 
   @Override
+  public List getWALs() throws IOException {
+return walFactory.getWALs();
+  }
+
+  @Override
   public WAL getWAL(HRegionInfo regionInfo) throws IOException {
 WAL wal;
 LogRoller roller = walRoller;

http://git-wip-us.apache.org/repos/asf/hbase/blob/f174fec3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
index 356a88b..bfd0431 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.hbase.regionserver;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
@@ -55,6 +56,11 @@ public interface RegionServerServices extends OnlineRegions, 
FavoredNodesForRegi
* default (common) WAL */
   WAL getWAL(HRegionInfo regionInfo) throws IOException;
 
+  /** @return the List of WALs that are used by this server
+   *  Doesn't include the meta WAL
+   */
+  List getWALs() throws IOException;
+
   /**
* @return Implementation of {@link CompactionRequestor} or null.
*/

http://git-wip-us.apache.org/repos/asf/hbase/blob/f174fec3/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
index 3aafc23..e33ae33 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/wal/AbstractFSWAL.java
@@ -239,6 +239,10 @@ public abstract class AbstractFSWAL implements WAL {
*/
   private final ConcurrentMap syncFuturesByHandler;
 
+  public long getFilenum() {
+return this.filenum.get();
+  }
+
   /**
* A log file has a creation timestamp (in ms) in its file name ({@link 
#filenum}. This helper
* method returns the creation timestamp from a given log file. It extracts 
the timestamp assuming

http://git-wip-us.apache.org/repos/asf/hbase/blob/f174fec3/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/AbstractFSWALProvide