Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 39B9D18D5E for ; Thu, 25 Jun 2015 17:14:00 +0000 (UTC) Received: (qmail 47752 invoked by uid 500); 25 Jun 2015 17:14:00 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 47686 invoked by uid 500); 25 Jun 2015 17:14:00 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 47673 invoked by uid 99); 25 Jun 2015 17:13:59 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 25 Jun 2015 17:13:59 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id CC95FE3684; Thu, 25 Jun 2015 17:13:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: arp@apache.org To: common-commits@hadoop.apache.org Date: Thu, 25 Jun 2015 17:13:59 -0000 Message-Id: <3e408ec665c04b49b529b8a7ce9f8c71@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/2] hadoop git commit: HDFS-8640. Make reserved RBW space visible through JMX. (Contributed by kanaka kumar avvaru) Repository: hadoop Updated Branches: refs/heads/branch-2 44dbddb02 -> 05e9ffdd6 refs/heads/trunk bc433908d -> 67a62da5c HDFS-8640. Make reserved RBW space visible through JMX. (Contributed by kanaka kumar avvaru) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/67a62da5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/67a62da5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/67a62da5 Branch: refs/heads/trunk Commit: 67a62da5c5f592b07d083440ced3666c7709b20d Parents: bc43390 Author: Arpit Agarwal Authored: Thu Jun 25 10:13:22 2015 -0700 Committer: Arpit Agarwal Committed: Thu Jun 25 10:13:22 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../datanode/fsdataset/impl/FsDatasetImpl.java | 5 +++- .../fsdataset/impl/TestRbwSpaceReservation.java | 28 ++++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/67a62da5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e5c30bd..01bb92d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -669,6 +669,9 @@ Release 2.8.0 - UNRELEASED HDFS-8462. Implement GETXATTRS and LISTXATTRS operations for WebImageViewer. (Jagadesh Kiran N via aajisaka) + HDFS-8640. Make reserved RBW space visible through JMX. (kanaka kumar + avvaru via Arpit Agarwal) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/67a62da5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index a1ff918..a2bb2c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -2559,13 +2559,15 @@ class FsDatasetImpl implements FsDatasetSpi { final String directory; final long usedSpace; // size of space used by HDFS final long freeSpace; // size of free space excluding reserved space - final long reservedSpace; // size of space reserved for non-HDFS and RBW + final long reservedSpace; // size of space reserved for non-HDFS + final long reservedSpaceForRBW; // size of space reserved RBW VolumeInfo(FsVolumeImpl v, long usedSpace, long freeSpace) { this.directory = v.toString(); this.usedSpace = usedSpace; this.freeSpace = freeSpace; this.reservedSpace = v.getReserved(); + this.reservedSpaceForRBW = v.getReservedForRbw(); } } @@ -2599,6 +2601,7 @@ class FsDatasetImpl implements FsDatasetSpi { innerInfo.put("usedSpace", v.usedSpace); innerInfo.put("freeSpace", v.freeSpace); innerInfo.put("reservedSpace", v.reservedSpace); + innerInfo.put("reservedSpaceForRBW", v.reservedSpaceForRBW); info.put(v.directory, innerInfo); } return info; http://git-wip-us.apache.org/repos/asf/hadoop/blob/67a62da5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestRbwSpaceReservation.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestRbwSpaceReservation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestRbwSpaceReservation.java index 44bf81b..a647d96 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestRbwSpaceReservation.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestRbwSpaceReservation.java @@ -49,11 +49,15 @@ import org.mockito.Mockito; import java.io.IOException; import java.io.OutputStream; +import java.lang.management.ManagementFactory; import java.lang.reflect.Field; import java.util.Map; import java.util.Random; import java.util.concurrent.TimeoutException; +import javax.management.MBeanServer; +import javax.management.ObjectName; + /** * Ensure that the DN reserves disk space equivalent to a full block for * replica being written (RBW). @@ -324,6 +328,30 @@ public class TestRbwSpaceReservation { fsVolumeImpl.getReservedForRbw() == 0); } + @Test(timeout = 30000) + public void testRBWInJMXBean() throws Exception { + + final short replication = 1; + startCluster(BLOCK_SIZE, replication, -1); + + final String methodName = GenericTestUtils.getMethodName(); + final Path file = new Path("/" + methodName + ".01.dat"); + + try (FSDataOutputStream os = fs.create(file, replication)) { + // Write 1 byte to the file + os.write(new byte[1]); + os.hsync(); + + final MBeanServer mbs = ManagementFactory.getPlatformMBeanServer(); + final ObjectName mxbeanName = new ObjectName( + "Hadoop:service=DataNode,name=DataNodeInfo"); + final String volumeInfo = (String) mbs.getAttribute(mxbeanName, + "VolumeInfo"); + + assertTrue(volumeInfo.contains("reservedSpaceForRBW")); + } + } + /** * Stress test to ensure we are not leaking reserved space. * @throws IOException