Author: todd Date: Thu Jul 24 08:20:25 2014 New Revision: 1613034 URL: http://svn.apache.org/r1613034 Log: MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common. Contributed by Todd Lipcon.
Removed: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/DirectBufferPool.java hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/test/java/org/apache/hadoop/mapred/nativetask/buffer/TestDirectBufferPool.java Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/InputBuffer.java Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt?rev=1613034&r1=1613033&r2=1613034&view=diff ============================================================================== --- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt (original) +++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/CHANGES.MAPREDUCE-2841.txt Thu Jul 24 08:20:25 2014 @@ -4,3 +4,4 @@ Changes for Hadoop Native Map Output Col MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by Binglin Chang MAPREDUCE-5994. Simplify ByteUtils and fix failing test. (todd) MAPREDUCE-5996. native-task: Rename system tests into standard directory layout (todd) +MAPREDUCE-5997. native-task: Use DirectBufferPool from Hadoop Common (todd) Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java?rev=1613034&r1=1613033&r2=1613034&view=diff ============================================================================== --- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java (original) +++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/NativeBatchProcessor.java Thu Jul 24 08:20:25 2014 @@ -24,12 +24,13 @@ import java.nio.ByteBuffer; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.mapred.nativetask.buffer.BufferType; -import org.apache.hadoop.mapred.nativetask.buffer.DirectBufferPool; import org.apache.hadoop.mapred.nativetask.buffer.InputBuffer; import org.apache.hadoop.mapred.nativetask.buffer.OutputBuffer; import org.apache.hadoop.mapred.nativetask.util.ReadWriteBuffer; import org.apache.hadoop.mapred.nativetask.util.ConfigUtil; +import org.apache.hadoop.util.DirectBufferPool; /** * used to create channel, transfer data and command between Java and native @@ -126,9 +127,8 @@ public class NativeBatchProcessor implem NativeRuntime.releaseNativeObject(nativeHandlerAddr); nativeHandlerAddr = 0; } - if (null != in && null != in.getByteBuffer() && in.getByteBuffer().isDirect()) { - DirectBufferPool.getInstance().returnBuffer(in.getByteBuffer()); - } + IOUtils.cleanup(LOG, in); + in = null; } @Override Modified: hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/InputBuffer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/InputBuffer.java?rev=1613034&r1=1613033&r2=1613034&view=diff ============================================================================== --- hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/InputBuffer.java (original) +++ hadoop/common/branches/MR-2841/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/java/org/apache/hadoop/mapred/nativetask/buffer/InputBuffer.java Thu Jul 24 08:20:25 2014 @@ -18,11 +18,16 @@ package org.apache.hadoop.mapred.nativetask.buffer; +import org.apache.hadoop.util.DirectBufferPool; + +import java.io.Closeable; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; -public class InputBuffer { +public class InputBuffer implements Closeable { + + static DirectBufferPool bufferPool = new DirectBufferPool(); private ByteBuffer byteBuffer; private final BufferType type; @@ -36,7 +41,7 @@ public class InputBuffer { switch (type) { case DIRECT_BUFFER: - this.byteBuffer = DirectBufferPool.getInstance().borrowBuffer(capacity); + this.byteBuffer = bufferPool.getBuffer(capacity); this.byteBuffer.order(ByteOrder.BIG_ENDIAN); break; case HEAP_BUFFER: @@ -118,4 +123,12 @@ public class InputBuffer { } return byteBuffer.array(); } + + @Override + public void close() { + if (byteBuffer != null && byteBuffer.isDirect()) { + bufferPool.returnBuffer(byteBuffer); + byteBuffer = null; + } + } }