hbase git commit: HBASE-16055 TestHFileOutputFormat2 is broken (Ram)

2016-06-26 Thread ramkrishna 
Repository: hbase
Updated Branches:
  refs/heads/branch-1 f4c1fd1a4 -> 385a6e878


HBASE-16055 TestHFileOutputFormat2 is broken (Ram)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/385a6e87
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/385a6e87
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/385a6e87

Branch: refs/heads/branch-1
Commit: 385a6e878fca1ccc889d8bf9d6527b8fe152c935
Parents: f4c1fd1
Author: Ramkrishna 
Authored: Mon Jun 27 11:02:03 2016 +0530
Committer: Ramkrishna 
Committed: Mon Jun 27 11:02:03 2016 +0530

--
 .../org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/385a6e87/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
--
diff --git 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
index c50d34d..9136256 100644
--- 
a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
+++ 
b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
@@ -530,7 +530,7 @@ public class TestHFileOutputFormat2  {
 doIncrementalLoadTest(true, true, false, 
"testMRIncrementalLoadWithLocality2");
   }
 
-  @Test
+  @Ignore("Goes zombie too frequently; needs work. See HBASE-14563") @Test
   public void testMRIncrementalLoadWithPutSortReducer() throws Exception {
 LOG.info("\nStarting test testMRIncrementalLoadWithPutSortReducer\n");
 doIncrementalLoadTest(false, false, true, 
"testMRIncrementalLoadWithPutSortReducer");

[8/9] hbase git commit: HBASE-16090 ResultScanner is not closed in SyncTable#finishRemainingHashRanges()

2016-06-26 Thread apurtell 
HBASE-16090 ResultScanner is not closed in SyncTable#finishRemainingHashRanges()


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/099b3da9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/099b3da9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/099b3da9

Branch: refs/heads/0.98
Commit: 099b3da9e0b8dd023d0ec5933a901ff9bad6d267
Parents: 9cdf8c3
Author: tedyu 
Authored: Thu Jun 23 07:29:34 2016 -0700
Committer: Andrew Purtell 
Committed: Fri Jun 24 11:24:57 2016 -0700

--
 .../org/apache/hadoop/hbase/mapreduce/SyncTable.java   | 13 ++---
 1 file changed, 10 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/099b3da9/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
index 0c8f185..08ad1b0 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/SyncTable.java
@@ -653,9 +653,16 @@ public class SyncTable extends Configured implements Tool {
 scan.setStopRow(nextSourceKey.copyBytes());
   }
   
-  ResultScanner targetScanner = targetTable.getScanner(scan);
-  for (Result row : targetScanner) {
-targetHasher.hashResult(row);  
+  ResultScanner targetScanner = null;
+  try {
+targetScanner = targetTable.getScanner(scan);
+for (Result row : targetScanner) {
+  targetHasher.hashResult(row);
+}
+  } finally {
+if (targetScanner != null) {
+  targetScanner.close();
+}
   }
 } // else current batch ends exactly at split end row

[1/9] hbase git commit: HBASE-15946 Eliminate possible security concerns in RS web UI's store file metrics (Sean Mackrory)

2016-06-26 Thread apurtell 
Repository: hbase
Updated Branches:
  refs/heads/0.98 4a37af5db -> 41f6f7e30


HBASE-15946 Eliminate possible security concerns in RS web UI's store file 
metrics (Sean Mackrory)

Conflicts:

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
hbase-server/src/main/resources/hbase-webapps/rest/rest.jsp


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/947e74ef
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/947e74ef
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/947e74ef

Branch: refs/heads/0.98
Commit: 947e74efa7eb92eec9b8b02cf2c73d5391be0ab1
Parents: 4a37af5
Author: Sean Mackrory 
Authored: Tue May 31 10:28:27 2016 -0600
Committer: Andrew Purtell 
Committed: Fri Jun 24 10:37:35 2016 -0700

--
 .../hbase/io/hfile/HFilePrettyPrinter.java  | 115 ---
 .../main/resources/hbase-webapps/rest/rest.jsp  |  64 +--
 2 files changed, 101 insertions(+), 78 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/947e74ef/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
index d5abdc4..270b552 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
@@ -1,4 +1,3 @@
-
 /*
  *
  * Licensed to the Apache Software Foundation (ASF) under one
@@ -88,6 +87,9 @@ public class HFilePrettyPrinter extends Configured implements 
Tool {
   private boolean checkFamily;
   private boolean isSeekToRow = false;
 
+  private PrintStream out = System.out;
+  private PrintStream err = System.err;
+
   /**
* The row which the user wants to specify and print all the KeyValues for.
*/
@@ -131,6 +133,11 @@ public class HFilePrettyPrinter extends Configured 
implements Tool {
 options.addOptionGroup(files);
   }
 
+  public void setPrintStreams(PrintStream out, PrintStream err) {
+this.out = out;
+this.err = err;
+  }
+
   public boolean parseOptions(String args[]) throws ParseException,
   IOException {
 if (args.length == 0) {
@@ -161,7 +168,7 @@ public class HFilePrettyPrinter extends Configured 
implements Tool {
 row = key.getBytes();
 isSeekToRow = true;
   } else {
-System.err.println("Invalid row is specified.");
+err.println("Invalid row is specified.");
 System.exit(-1);
   }
 }
@@ -175,17 +182,17 @@ public class HFilePrettyPrinter extends Configured 
implements Tool {
   String enc = HRegionInfo.encodeRegionName(rn);
   Path regionDir = new Path(tableDir, enc);
   if (verbose)
-System.out.println("region dir -> " + regionDir);
+out.println("region dir -> " + regionDir);
   List regionFiles = HFile.getStoreFiles(FileSystem.get(getConf()),
   regionDir);
   if (verbose)
-System.out.println("Number of region files found -> "
+out.println("Number of region files found -> "
 + regionFiles.size());
   if (verbose) {
 int i = 1;
 for (Path p : regionFiles) {
   if (verbose)
-System.out.println("Found file[" + i++ + "] -> " + p);
+out.println("Found file[" + i++ + "] -> " + p);
 }
   }
   files.addAll(regionFiles);
@@ -217,27 +224,46 @@ public class HFilePrettyPrinter extends Configured 
implements Tool {
 // iterate over all files found
 for (Path fileName : files) {
   try {
-processFile(fileName);
+int exitCode = processFile(fileName);
+if (exitCode != 0) {
+  return exitCode;
+}
   } catch (IOException ex) {
 LOG.error("Error reading " + fileName, ex);
-System.exit(-2);
+return -2;
   }
 }
 
 if (verbose || printKey) {
-  System.out.println("Scanned kv count -> " + count);
+  out.println("Scanned kv count -> " + count);
 }
 
 return 0;
   }
 
-  private void processFile(Path file) throws IOException {
+  public int processFile(Path file) throws IOException {
 if (verbose)
-  System.out.println("Scanning -> " + file);
+  out.println("Scanning -> " + file);
+
+Path rootPath = FSUtils.getRootDir(getConf());
+String rootString = rootPath + rootPath.SEPARATOR;
+if (!file.toString().startsWith(rootString)) {
+  // First we see if fully-qualified URI matches the root dir. It might
+  // also be an absolute path in the same

[6/9] hbase git commit: HBASE-15870 Specify columns in REST multi gets (Matt Warhaftig)

2016-06-26 Thread apurtell 
HBASE-15870 Specify columns in REST multi gets (Matt Warhaftig)

Amending-Author: Andrew Purtell 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0fd4c1dc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0fd4c1dc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0fd4c1dc

Branch: refs/heads/0.98
Commit: 0fd4c1dcdc00d70820d9e816cad0a43c6b039c11
Parents: ce13b59
Author: Jerry He 
Authored: Mon Jun 20 13:51:41 2016 -0700
Committer: Andrew Purtell 
Committed: Fri Jun 24 11:17:48 2016 -0700

--
 .../hadoop/hbase/rest/MultiRowResource.java | 15 -
 .../apache/hadoop/hbase/rest/TableResource.java |  8 ++---
 .../hadoop/hbase/rest/TestMultiRowResource.java | 34 
 3 files changed, 52 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/0fd4c1dc/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
--
diff --git 
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java 
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
index 8ff3ef6..9cafe27 100644
--- 
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
+++ 
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/MultiRowResource.java
@@ -42,6 +42,7 @@ public class MultiRowResource extends ResourceBase implements 
Constants {
 
   TableResource tableResource;
   Integer versions = null;
+  String[] columns = null;
 
   /**
* Constructor
@@ -50,10 +51,15 @@ public class MultiRowResource extends ResourceBase 
implements Constants {
* @param versions
* @throws java.io.IOException
*/
-  public MultiRowResource(TableResource tableResource, String versions) throws 
IOException {
+  public MultiRowResource(TableResource tableResource, String versions, String 
columnsStr)
+  throws IOException {
 super();
 this.tableResource = tableResource;
 
+if (columnsStr != null && !columnsStr.equals("")) {
+  this.columns = columnsStr.split(",");
+}
+
 if (versions != null) {
   this.versions = Integer.valueOf(versions);
 
@@ -74,6 +80,13 @@ public class MultiRowResource extends ResourceBase 
implements Constants {
 if (this.versions != null) {
   rowSpec.setMaxVersions(this.versions);
 }
+
+if (this.columns != null) {
+  for (int i = 0; i < this.columns.length; i++) {
+rowSpec.addColumn(this.columns[i].getBytes());
+  }
+}
+
 ResultGenerator generator =
   ResultGenerator.fromRowSpec(this.tableResource.getName(), rowSpec, 
null,
 !params.containsKey(NOCACHE_PARAM_NAME));

http://git-wip-us.apache.org/repos/asf/hbase/blob/0fd4c1dc/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
--
diff --git 
a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java 
b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
index bfbb549..b55d0e0 100644
--- a/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
+++ b/hbase-rest/src/main/java/org/apache/hadoop/hbase/rest/TableResource.java
@@ -90,10 +90,10 @@ public class TableResource extends ResourceBase {
 return new SchemaResource(this);
   }
 
-  @Path("multiget")
-  public MultiRowResource getMultipleRowResource(
-  final @QueryParam("v") String versions) throws IOException {
-return new MultiRowResource(this, versions);
+  @Path("{multiget: multiget.*}")
+  public MultiRowResource getMultipleRowResource(final @QueryParam("v") String 
versions,
+  @PathParam("multiget") String path) throws IOException {
+return new MultiRowResource(this, versions, path.replace("multiget", 
"").replace("/", ""));
   }
 
   @Path("{rowspec: [^*]+}")

http://git-wip-us.apache.org/repos/asf/hbase/blob/0fd4c1dc/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java
--
diff --git 
a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java
 
b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java
index dd54366..aacd3b0 100644
--- 
a/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java
+++ 
b/hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestMultiRowResource.java
@@ -155,6 +155,40 @@ public class TestMultiRowResource {
   }
 
   @Test
+  public void testMultiCellGetWithColsJSON() throws IOException, JAXBException 
{
+String row_5_url = "/" + TABLE + "/" + ROW_1 + "/" + COLUMN_1;

[4/9] hbase git commit: HBASE-16045 endtime argument for VerifyReplication was incorrectly specified in usage

2016-06-26 Thread apurtell 
HBASE-16045 endtime argument for VerifyReplication was incorrectly specified in 
usage


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e18daa0c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e18daa0c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e18daa0c

Branch: refs/heads/0.98
Commit: e18daa0c9c1653c1d6c409bf990893798ce53cbf
Parents: 6a8206a
Author: tedyu 
Authored: Thu Jun 16 15:52:16 2016 -0700
Committer: Andrew Purtell 
Committed: Fri Jun 24 11:04:24 2016 -0700

--
 .../hadoop/hbase/mapreduce/replication/VerifyReplication.java  | 6 +-
 src/main/asciidoc/_chapters/ops_mgt.adoc   | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/e18daa0c/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
index 44bfdfe..a5d1ddd 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
@@ -383,6 +383,10 @@ public class VerifyReplication extends Configured 
implements Tool {
   continue;
 }
 
+if (cmd.startsWith("--")) {
+  printUsage("Invalid argument '" + cmd + "'");
+}
+
 if (i == args.length-2) {
   peerId = cmd;
 }
@@ -418,7 +422,7 @@ public class VerifyReplication extends Configured 
implements Tool {
   System.err.println("ERROR: " + errorMsg);
 }
 System.err.println("Usage: verifyrep [--starttime=X]" +
-" [--stoptime=Y] [--families=A] [--row-prefixes=B]  
");
+" [--endtime=Y] [--families=A] [--row-prefixes=B]  
");
 System.err.println();
 System.err.println("Options:");
 System.err.println(" starttimebeginning of the time range");

http://git-wip-us.apache.org/repos/asf/hbase/blob/e18daa0c/src/main/asciidoc/_chapters/ops_mgt.adoc
--
diff --git a/src/main/asciidoc/_chapters/ops_mgt.adoc 
b/src/main/asciidoc/_chapters/ops_mgt.adoc
index d7ac987..fa31c8c 100644
--- a/src/main/asciidoc/_chapters/ops_mgt.adoc
+++ b/src/main/asciidoc/_chapters/ops_mgt.adoc
@@ -1358,7 +1358,7 @@ The `VerifyReplication` MapReduce job, which is included 
in HBase, performs a sy
 +
 [source,bash]
 
-$ HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` 
"${HADOOP_HOME}/bin/hadoop" jar "${HBASE_HOME}/hbase-server-VERSION.jar" 
verifyrep --starttime= --stoptime= --families= 
 
+$ HADOOP_CLASSPATH=`${HBASE_HOME}/bin/hbase classpath` 
"${HADOOP_HOME}/bin/hadoop" jar "${HBASE_HOME}/hbase-server-VERSION.jar" 
verifyrep --starttime= --endtime= --families=  

 
 +
 The `VerifyReplication` command prints out `GOODROWS` and `BADROWS` counters 
to indicate rows that did and did not replicate correctly.

[3/9] hbase git commit: HBASE-16033 Add more details in logging of responseTooSlow/TooLarge

2016-06-26 Thread apurtell 
HBASE-16033 Add more details in logging of responseTooSlow/TooLarge

Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/6a8206aa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/6a8206aa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/6a8206aa

Branch: refs/heads/0.98
Commit: 6a8206aaf704c7f8e96776d52d5cdf8e7328b0fe
Parents: 13dee1c
Author: Yu Li 
Authored: Thu Jun 16 16:40:38 2016 +0800
Committer: Andrew Purtell 
Committed: Fri Jun 24 10:49:23 2016 -0700

--
 .../org/apache/hadoop/hbase/ipc/RpcServer.java  | 47 
 1 file changed, 8 insertions(+), 39 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/6a8206aa/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index 6ab976d..b3e9103 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -68,10 +68,7 @@ import org.apache.hadoop.hbase.CellScanner;
 import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.HBaseInterfaceAudience;
 import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.Server;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Operation;
 import org.apache.hadoop.hbase.codec.Codec;
 import org.apache.hadoop.hbase.exceptions.RegionMovedException;
 import org.apache.hadoop.hbase.io.ByteBufferInputStream;
@@ -86,7 +83,6 @@ import 
org.apache.hadoop.hbase.protobuf.generated.RPCProtos.ExceptionResponse;
 import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.RequestHeader;
 import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.ResponseHeader;
 import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.UserInformation;
-import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.security.AccessDeniedException;
 import org.apache.hadoop.hbase.security.AuthMethod;
 import org.apache.hadoop.hbase.security.HBasePolicyProvider;
@@ -2220,13 +2216,9 @@ public class RpcServer implements RpcServerInterface {
   if (tooSlow || tooLarge) {
 // when tagging, we let TooLarge trump TooSmall to keep output simple
 // note that large responses will often also be slow.
-StringBuilder buffer = new StringBuilder(256);
-buffer.append(md.getName());
-buffer.append("(");
-buffer.append(param.getClass().getName());
-buffer.append(")");
-logResponse(new Object[]{param},
-md.getName(), buffer.toString(), (tooLarge ? "TooLarge" : 
"TooSlow"),
+logResponse(param,
+md.getName(), md.getName() + "(" + param.getClass().getName() + 
")",
+(tooLarge ? "TooLarge" : "TooSlow"),
 status.getClient(), startTime, processingTime, qTime,
 responseSize);
   }
@@ -2251,7 +2243,7 @@ public class RpcServer implements RpcServerInterface {
   /**
* Logs an RPC response to the LOG file, producing valid JSON objects for
* client Operations.
-   * @param params The parameters received in the call.
+   * @param param The parameters received in the call.
* @param methodName The name of the method invoked
* @param call The string representation of the call
* @param tag  The tag that will be used to indicate this event in the log.
@@ -2262,7 +2254,7 @@ public class RpcServer implements RpcServerInterface {
*prior to being initiated, in ms.
* @param responseSizeThe size in bytes of the response buffer.
*/
-  void logResponse(Object[] params, String methodName, String call, String tag,
+  void logResponse(Message param, String methodName, String call, String tag,
   String clientAddress, long startTime, int processingTime, int qTime,
   long responseSize)
   throws IOException {
@@ -2275,32 +2267,9 @@ public class RpcServer implements RpcServerInterface {
 responseInfo.put("client", clientAddress);
 responseInfo.put("class", serverInstance == null? "": 
serverInstance.getClass().getSimpleName());
 responseInfo.put("method", methodName);
-if (params.length == 2 && serverInstance instanceof HRegionServer &&
-params[0] instanceof byte[] &&
-params[1] instanceof Operation) {
-  // if the slow process is a query, we want to log its table as well
-  // as its own

[5/9] hbase git commit: HBASE-16061 Allow logging to a buffered console

2016-06-26 Thread apurtell 
HBASE-16061 Allow logging to a buffered console


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ce13b595
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ce13b595
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ce13b595

Branch: refs/heads/0.98
Commit: ce13b5953b27fae8a16cf6293e95d2e88c96b8bc
Parents: e18daa0
Author: Elliott Clark 
Authored: Fri Jun 17 16:32:57 2016 -0700
Committer: Andrew Purtell 
Committed: Fri Jun 24 11:13:49 2016 -0700

--
 conf/log4j.properties   |  5 +-
 .../hadoop/hbase/AsyncConsoleAppender.java  | 48 
 2 files changed, 52 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/ce13b595/conf/log4j.properties
--
diff --git a/conf/log4j.properties b/conf/log4j.properties
index 1d7656b..6248a1b 100644
--- a/conf/log4j.properties
+++ b/conf/log4j.properties
@@ -80,13 +80,16 @@ 
log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
 
 #
 # console
-# Add "console" to rootlogger above if you want to use this 
+# Add "console" to rootlogger above if you want to use this
 #
 log4j.appender.console=org.apache.log4j.ConsoleAppender
 log4j.appender.console.target=System.err
 log4j.appender.console.layout=org.apache.log4j.PatternLayout
 log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: 
%m%n
 
+log4j.appender.asyncconsole=org.apache.hadoop.hbase.AsyncConsoleAppender
+log4j.appender.asyncconsole.target=System.err
+
 # Custom Logging levels
 
 log4j.logger.org.apache.zookeeper=INFO

http://git-wip-us.apache.org/repos/asf/hbase/blob/ce13b595/hbase-common/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java
--
diff --git 
a/hbase-common/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java
new file mode 100644
index 000..338265d
--- /dev/null
+++ 
b/hbase-common/src/main/java/org/apache/hadoop/hbase/AsyncConsoleAppender.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase;
+
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.log4j.AsyncAppender;
+import org.apache.log4j.ConsoleAppender;
+import org.apache.log4j.PatternLayout;
+
+/**
+ * Logger class that buffers before trying to log to the specified console.
+ */
+@InterfaceAudience.Private
+public class AsyncConsoleAppender extends AsyncAppender {
+  private final ConsoleAppender consoleAppender;
+
+  public AsyncConsoleAppender() {
+super();
+consoleAppender = new ConsoleAppender(new PatternLayout(
+"%d{ISO8601} %-5p [%t] %c{2}: %m%n"));
+this.addAppender(consoleAppender);
+  }
+
+  public void setTarget(String value) {
+consoleAppender.setTarget(value);
+  }
+
+  public void activateOptions() {
+consoleAppender.activateOptions();
+super.activateOptions();
+  }
+
+}

[7/9] hbase git commit: HBASE-16070 Mapreduce Serialization classes do not have Interface audience (Ram)

2016-06-26 Thread apurtell 
HBASE-16070 Mapreduce Serialization classes do not have Interface audience
(Ram)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9cdf8c36
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9cdf8c36
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9cdf8c36

Branch: refs/heads/0.98
Commit: 9cdf8c36b71ab1a30eca6da0eecd4ee2baad3a7a
Parents: 0fd4c1d
Author: Ramkrishna 
Authored: Thu Jun 23 10:55:54 2016 +0530
Committer: Andrew Purtell 
Committed: Fri Jun 24 11:24:33 2016 -0700

--
 .../apache/hadoop/hbase/mapreduce/KeyValueSerialization.java| 5 -
 .../apache/hadoop/hbase/mapreduce/MutationSerialization.java| 4 
 .../org/apache/hadoop/hbase/mapreduce/ResultSerialization.java  | 4 
 3 files changed, 12 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/9cdf8c36/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyValueSerialization.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyValueSerialization.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyValueSerialization.java
index 192afd8..2b92439 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyValueSerialization.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/KeyValueSerialization.java
@@ -23,11 +23,14 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.io.serializer.Deserializer;
 import org.apache.hadoop.io.serializer.Serialization;
 import org.apache.hadoop.io.serializer.Serializer;
-
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
 public class KeyValueSerialization implements Serialization {
   @Override
   public boolean accept(Class c) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9cdf8c36/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MutationSerialization.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MutationSerialization.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MutationSerialization.java
index 4d200e8..ce2cf1a 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MutationSerialization.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/MutationSerialization.java
@@ -21,6 +21,8 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Mutation;
 import org.apache.hadoop.hbase.client.Put;
@@ -31,6 +33,8 @@ import org.apache.hadoop.io.serializer.Deserializer;
 import org.apache.hadoop.io.serializer.Serialization;
 import org.apache.hadoop.io.serializer.Serializer;
 
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
 public class MutationSerialization implements Serialization {
   @Override
   public boolean accept(Class c) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/9cdf8c36/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ResultSerialization.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ResultSerialization.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ResultSerialization.java
index 19b12c5..a267ca9 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ResultSerialization.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/ResultSerialization.java
@@ -31,6 +31,8 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
@@ -39,6 +41,8 @@ import org.apache.hadoop.io.serializer.Deserializer;
 import org.apache.hadoop.io.serializer.Serialization;
 import org.apache.hadoop.io.serializer.Serializer;
 
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
 public

[9/9] hbase git commit: HBASE-16085 Add a metric for failed compactions

2016-06-26 Thread apurtell 
HBASE-16085 Add a metric for failed compactions

Conflicts:

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java

hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java

hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java

Amending-Author: Andrew Purtell 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/41f6f7e3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/41f6f7e3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/41f6f7e3

Branch: refs/heads/0.98
Commit: 41f6f7e30915977f178071fae2ac3b3585347222
Parents: 099b3da
Author: Gary Helmling 
Authored: Wed Jun 22 17:08:28 2016 -0700
Committer: Andrew Purtell 
Committed: Fri Jun 24 11:36:22 2016 -0700

--
 .../hbase/regionserver/MetricsRegionSource.java |  2 +
 .../regionserver/MetricsRegionWrapper.java  |  7 
 .../regionserver/MetricsRegionSourceImpl.java   |  3 ++
 .../TestMetricsRegionSourceImpl.java|  6 +++
 .../regionserver/MetricsRegionSourceImpl.java   |  3 ++
 .../TestMetricsRegionSourceImpl.java|  5 +++
 .../hbase/regionserver/CompactSplitThread.java  |  2 +
 .../hadoop/hbase/regionserver/HRegion.java  | 10 -
 .../regionserver/MetricsRegionWrapperImpl.java  |  5 +++
 .../regionserver/MetricsRegionWrapperStub.java  |  5 +++
 .../hbase/regionserver/TestCompaction.java  | 43 
 11 files changed, 89 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/41f6f7e3/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
--
diff --git 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
index 8abbb1f..cfe95ed 100644
--- 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
+++ 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSource.java
@@ -28,9 +28,11 @@ public interface MetricsRegionSource extends 
Comparable {
   String OPS_SAMPLE_NAME = "ops";
   String SIZE_VALUE_NAME = "size";
   String COMPACTIONS_COMPLETED_COUNT = "compactionsCompletedCount";
+  String COMPACTIONS_FAILED_COUNT = "compactionsFailedCount";
   String NUM_BYTES_COMPACTED_COUNT = "numBytesCompactedCount";
   String NUM_FILES_COMPACTED_COUNT = "numFilesCompactedCount";
   String COMPACTIONS_COMPLETED_DESC = "Number of compactions that have 
completed.";
+  String COMPACTIONS_FAILED_DESC = "Number of compactions that have failed.";
   String  NUM_BYTES_COMPACTED_DESC =
   "Sum of filesize on all files entering a finished, successful or 
aborted, compaction";
   String NUM_FILES_COMPACTED_DESC =

http://git-wip-us.apache.org/repos/asf/hbase/blob/41f6f7e3/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
--
diff --git 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
index 2a3bfb7..5349b5b 100644
--- 
a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
+++ 
b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
@@ -105,6 +105,13 @@ public interface MetricsRegionWrapper {
   long getNumCompactionsCompleted();
 
   /**
+   * Returns the total number of compactions that have been reported as failed 
on this region.
+   * Note that a given compaction can be reported as both completed and failed 
if an exception
+   * is thrown in the processing after {@code HRegion.compact()}.
+   */
+  long getNumCompactionsFailed();
+
+  /**
* Get the time spent by coprocessors in this region.
*/
   Map getCoprocessorExecutionStatistics();

http://git-wip-us.apache.org/repos/asf/hbase/blob/41f6f7e3/hbase-hadoop1-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
--
diff --git

[2/9] hbase git commit: HBASE-15746 Remove extra RegionCoprocessor preClose() in RSRpcServices#closeRegion (Stephen Yuan Jiang)

2016-06-26 Thread apurtell 
HBASE-15746 Remove extra RegionCoprocessor preClose() in 
RSRpcServices#closeRegion (Stephen Yuan Jiang)

Amending-Author: Andrew Purtell 


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/13dee1cf
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/13dee1cf
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/13dee1cf

Branch: refs/heads/0.98
Commit: 13dee1cf467e66147c24a7cff158eb6dedf9f3a9
Parents: 947e74e
Author: Stephen Yuan Jiang 
Authored: Tue Jun 14 13:30:41 2016 -0700
Committer: Andrew Purtell 
Committed: Fri Jun 24 10:45:01 2016 -0700

--
 .../org/apache/hadoop/hbase/regionserver/HRegionServer.java| 6 --
 1 file changed, 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/13dee1cf/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
--
diff --git 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index 9df2a2c..d238fef 100644
--- 
a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ 
b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -4129,12 +4129,6 @@ public class HRegionServer implements 
ClientProtos.ClientService.BlockingInterfa
   }
   final String encodedRegionName = 
ProtobufUtil.getRegionEncodedName(request.getRegion());
 
-  // Can be null if we're calling close on a region that's not online
-  final HRegion region = this.getFromOnlineRegions(encodedRegionName);
-  if ((region  != null) && (region .getCoprocessorHost() != null)) {
-region.getCoprocessorHost().preClose(false);
-  }
-
   requestCount.increment();
   LOG.info("Close " + encodedRegionName + ", via zk=" + (zk ? "yes" : 
"no") +
 ", znode version=" + versionOfClosingNode + ", on " + sn);