HDFS-8640. Make reserved RBW space visible through JMX. (Contributed by kanaka kumar avvaru)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71fa8eba Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71fa8eba Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71fa8eba Branch: refs/heads/YARN-2928 Commit: 71fa8ebac83f362dc21e1172e26537ce82c4feac Parents: 08e95a3 Author: Arpit Agarwal <[email protected]> Authored: Thu Jun 25 10:13:22 2015 -0700 Committer: Zhijie Shen <[email protected]> Committed: Mon Jun 29 10:28:25 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../datanode/fsdataset/impl/FsDatasetImpl.java | 5 +++- .../fsdataset/impl/TestRbwSpaceReservation.java | 28 ++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/71fa8eba/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e5c30bd..01bb92d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -669,6 +669,9 @@ Release 2.8.0 - UNRELEASED HDFS-8462. Implement GETXATTRS and LISTXATTRS operations for WebImageViewer. (Jagadesh Kiran N via aajisaka) + HDFS-8640. Make reserved RBW space visible through JMX. (kanaka kumar + avvaru via Arpit Agarwal) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/71fa8eba/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index a1ff918..a2bb2c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -2559,13 +2559,15 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> { final String directory; final long usedSpace; // size of space used by HDFS final long freeSpace; // size of free space excluding reserved space - final long reservedSpace; // size of space reserved for non-HDFS and RBW + final long reservedSpace; // size of space reserved for non-HDFS + final long reservedSpaceForRBW; // size of space reserved RBW VolumeInfo(FsVolumeImpl v, long usedSpace, long freeSpace) { this.directory = v.toString(); this.usedSpace = usedSpace; this.freeSpace = freeSpace; this.reservedSpace = v.getReserved(); + this.reservedSpaceForRBW = v.getReservedForRbw(); } } @@ -2599,6 +2601,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> { innerInfo.put("usedSpace", v.usedSpace); innerInfo.put("freeSpace", v.freeSpace); innerInfo.put("reservedSpace", v.reservedSpace); + innerInfo.put("reservedSpaceForRBW", v.reservedSpaceForRBW); info.put(v.directory, innerInfo); } return info; http://git-wip-us.apache.org/repos/asf/hadoop/blob/71fa8eba/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestRbwSpaceReservation.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestRbwSpaceReservation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestRbwSpaceReservation.java index 44bf81b..a647d96 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestRbwSpaceReservation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestRbwSpaceReservation.java @@ -49,11 +49,15 @@ import org.mockito.Mockito; import java.io.IOException; import java.io.OutputStream; +import java.lang.management.ManagementFactory; import java.lang.reflect.Field; import java.util.Map; import java.util.Random; import java.util.concurrent.TimeoutException; +import javax.management.MBeanServer; +import javax.management.ObjectName; + /** * Ensure that the DN reserves disk space equivalent to a full block for * replica being written (RBW). @@ -324,6 +328,30 @@ public class TestRbwSpaceReservation { fsVolumeImpl.getReservedForRbw() == 0); } + @Test(timeout = 30000) + public void testRBWInJMXBean() throws Exception { + + final short replication = 1; + startCluster(BLOCK_SIZE, replication, -1); + + final String methodName = GenericTestUtils.getMethodName(); + final Path file = new Path("/" + methodName + ".01.dat"); + + try (FSDataOutputStream os = fs.create(file, replication)) { + // Write 1 byte to the file + os.write(new byte[1]); + os.hsync(); + + final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + final ObjectName mxbeanName = new ObjectName( + "Hadoop:service=DataNode,name=DataNodeInfo"); + final String volumeInfo = (String) mbs.getAttribute(mxbeanName, + "VolumeInfo"); + + assertTrue(volumeInfo.contains("reservedSpaceForRBW")); + } + } + /** * Stress test to ensure we are not leaking reserved space. * @throws IOException
