[07/30] hbase git commit: HBASE-15525 OutOfMemory could occur when using BoundedByteBufferPool during RPC bursts.

[syuanjiang]

HBASE-15525 OutOfMemory could occur when using BoundedByteBufferPool during RPC 
bursts.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/17bcf14f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/17bcf14f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/17bcf14f

Branch: refs/heads/hbase-12439
Commit: 17bcf14fea2637fe0e5ca23bb0008c1cca208c98
Parents: e486d27
Author: anoopsjohn 
Authored: Tue Jun 14 19:03:54 2016 +0530
Committer: anoopsjohn 
Committed: Tue Jun 14 19:03:54 2016 +0530

--
 .../org/apache/hadoop/hbase/ipc/IPCUtil.java| 107 ++--
 .../hbase/io/ByteBufferListOutputStream.java| 173 +++
 .../hadoop/hbase/io/ByteBufferOutputStream.java |  46 ++---
 .../apache/hadoop/hbase/io/ByteBufferPool.java  | 154 +
 .../io/TestByteBufferListOutputStream.java  |  77 +
 .../hadoop/hbase/io/TestByteBufferPool.java |  60 +++
 .../apache/hadoop/hbase/ipc/BufferChain.java|   7 +
 .../org/apache/hadoop/hbase/ipc/RpcServer.java  |  99 +++
 8 files changed, 609 insertions(+), 114 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/17bcf14f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
index 74466b5..74f934c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
@@ -34,10 +34,10 @@ import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.HBaseIOException;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.codec.Codec;
-import org.apache.hadoop.hbase.io.BoundedByteBufferPool;
 import org.apache.hadoop.hbase.io.ByteBufferInputStream;
 import org.apache.hadoop.hbase.io.ByteBufferOutputStream;
-import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.ByteBufferPool;
+import org.apache.hadoop.hbase.io.ByteBufferListOutputStream;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.io.compress.CodecPool;
@@ -90,30 +90,7 @@ public class IPCUtil {
*/
   @SuppressWarnings("resource")
   public ByteBuffer buildCellBlock(final Codec codec, final CompressionCodec 
compressor,
-final CellScanner cellScanner)
-  throws IOException {
-return buildCellBlock(codec, compressor, cellScanner, null);
-  }
-
-  /**
-   * Puts CellScanner Cells into a cell block using passed in 
codec and/or
-   * compressor.
-   * @param codec to use for encoding
-   * @param compressor to use for encoding
-   * @param cellScanner to encode
-   * @param pool Pool of ByteBuffers to make use of. Can be null and then 
we'll allocate
-   *   our own ByteBuffer.
-   * @return Null or byte buffer filled with a cellblock filled with passed-in 
Cells encoded using
-   *   passed in codec and/or compressor; the 
returned buffer has been
-   *   flipped and is ready for reading.  Use limit to find total size. If 
pool was not
-   *   null, then this returned ByteBuffer came from there and should be 
returned to the pool when
-   *   done.
-   * @throws IOException if encoding the cells fail
-   */
-  @SuppressWarnings("resource")
-  public ByteBuffer buildCellBlock(final Codec codec, final CompressionCodec 
compressor,
-final CellScanner cellScanner, final BoundedByteBufferPool pool)
-  throws IOException {
+  final CellScanner cellScanner) throws IOException {
 if (cellScanner == null) {
   return null;
 }
@@ -121,25 +98,25 @@ public class IPCUtil {
   throw new CellScannerButNoCodecException();
 }
 int bufferSize = this.cellBlockBuildingInitialBufferSize;
-ByteBufferOutputStream baos;
-if (pool != null) {
-  ByteBuffer bb = pool.getBuffer();
-  bufferSize = bb.capacity();
-  baos = new ByteBufferOutputStream(bb);
-} else {
-  // Then we need to make our own to return.
-  if (cellScanner instanceof HeapSize) {
-long longSize = ((HeapSize)cellScanner).heapSize();
-// Just make sure we don't have a size bigger than an int.
-if (longSize > Integer.MAX_VALUE) {
-  throw new IOException("Size " + longSize + " > " + 
Integer.MAX_VALUE);
-}
-bufferSize = ClassSize.align((int)longSize);
+ByteBufferOutputStream baos = new ByteBufferOutputStream(bufferSize);
+encodeCellsTo(baos, cellScanner, codec, compressor);
+if (LOG.isTraceEnabled()) {
+  if (bufferSize < baos.size()) {
+   

hbase git commit: HBASE-15525 OutOfMemory could occur when using BoundedByteBufferPool during RPC bursts.

[anoopsamjohn]

Repository: hbase
Updated Branches:
  refs/heads/master e486d274c -> 17bcf14fe


HBASE-15525 OutOfMemory could occur when using BoundedByteBufferPool during RPC 
bursts.


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/17bcf14f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/17bcf14f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/17bcf14f

Branch: refs/heads/master
Commit: 17bcf14fea2637fe0e5ca23bb0008c1cca208c98
Parents: e486d27
Author: anoopsjohn 
Authored: Tue Jun 14 19:03:54 2016 +0530
Committer: anoopsjohn 
Committed: Tue Jun 14 19:03:54 2016 +0530

--
 .../org/apache/hadoop/hbase/ipc/IPCUtil.java| 107 ++--
 .../hbase/io/ByteBufferListOutputStream.java| 173 +++
 .../hadoop/hbase/io/ByteBufferOutputStream.java |  46 ++---
 .../apache/hadoop/hbase/io/ByteBufferPool.java  | 154 +
 .../io/TestByteBufferListOutputStream.java  |  77 +
 .../hadoop/hbase/io/TestByteBufferPool.java |  60 +++
 .../apache/hadoop/hbase/ipc/BufferChain.java|   7 +
 .../org/apache/hadoop/hbase/ipc/RpcServer.java  |  99 +++
 8 files changed, 609 insertions(+), 114 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hbase/blob/17bcf14f/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
--
diff --git 
a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java 
b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
index 74466b5..74f934c 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
@@ -34,10 +34,10 @@ import org.apache.hadoop.hbase.DoNotRetryIOException;
 import org.apache.hadoop.hbase.HBaseIOException;
 import org.apache.hadoop.hbase.classification.InterfaceAudience;
 import org.apache.hadoop.hbase.codec.Codec;
-import org.apache.hadoop.hbase.io.BoundedByteBufferPool;
 import org.apache.hadoop.hbase.io.ByteBufferInputStream;
 import org.apache.hadoop.hbase.io.ByteBufferOutputStream;
-import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.ByteBufferPool;
+import org.apache.hadoop.hbase.io.ByteBufferListOutputStream;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.ClassSize;
 import org.apache.hadoop.io.compress.CodecPool;
@@ -90,30 +90,7 @@ public class IPCUtil {
*/
   @SuppressWarnings("resource")
   public ByteBuffer buildCellBlock(final Codec codec, final CompressionCodec 
compressor,
-final CellScanner cellScanner)
-  throws IOException {
-return buildCellBlock(codec, compressor, cellScanner, null);
-  }
-
-  /**
-   * Puts CellScanner Cells into a cell block using passed in 
codec and/or
-   * compressor.
-   * @param codec to use for encoding
-   * @param compressor to use for encoding
-   * @param cellScanner to encode
-   * @param pool Pool of ByteBuffers to make use of. Can be null and then 
we'll allocate
-   *   our own ByteBuffer.
-   * @return Null or byte buffer filled with a cellblock filled with passed-in 
Cells encoded using
-   *   passed in codec and/or compressor; the 
returned buffer has been
-   *   flipped and is ready for reading.  Use limit to find total size. If 
pool was not
-   *   null, then this returned ByteBuffer came from there and should be 
returned to the pool when
-   *   done.
-   * @throws IOException if encoding the cells fail
-   */
-  @SuppressWarnings("resource")
-  public ByteBuffer buildCellBlock(final Codec codec, final CompressionCodec 
compressor,
-final CellScanner cellScanner, final BoundedByteBufferPool pool)
-  throws IOException {
+  final CellScanner cellScanner) throws IOException {
 if (cellScanner == null) {
   return null;
 }
@@ -121,25 +98,25 @@ public class IPCUtil {
   throw new CellScannerButNoCodecException();
 }
 int bufferSize = this.cellBlockBuildingInitialBufferSize;
-ByteBufferOutputStream baos;
-if (pool != null) {
-  ByteBuffer bb = pool.getBuffer();
-  bufferSize = bb.capacity();
-  baos = new ByteBufferOutputStream(bb);
-} else {
-  // Then we need to make our own to return.
-  if (cellScanner instanceof HeapSize) {
-long longSize = ((HeapSize)cellScanner).heapSize();
-// Just make sure we don't have a size bigger than an int.
-if (longSize > Integer.MAX_VALUE) {
-  throw new IOException("Size " + longSize + " > " + 
Integer.MAX_VALUE);
-}
-bufferSize = ClassSize.align((int)longSize);
+ByteBufferOutputStream baos = new ByteBufferOutputStream(bufferSize);
+encodeCellsTo(baos, cellScanner, codec, compressor);
+