Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 8AA99200C55 for ; Thu, 9 Mar 2017 00:46:05 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 893D9160B89; Wed, 8 Mar 2017 23:46:05 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 6079C160B86 for ; Thu, 9 Mar 2017 00:46:04 +0100 (CET) Received: (qmail 77795 invoked by uid 500); 8 Mar 2017 23:46:00 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 77171 invoked by uid 99); 8 Mar 2017 23:46:00 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 08 Mar 2017 23:46:00 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id DA42EF4B5F; Wed, 8 Mar 2017 23:45:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aengineer@apache.org To: common-commits@hadoop.apache.org Date: Wed, 08 Mar 2017 23:46:13 -0000 Message-Id: <31c57d305318435981777570b2f73527@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [15/33] hadoop git commit: HDFS-10838. Last full block report received time for each DN should be easily discoverable. Contributed by Surendra Singh Lilhore. archived-at: Wed, 08 Mar 2017 23:46:05 -0000 HDFS-10838. Last full block report received time for each DN should be easily discoverable. Contributed by Surendra Singh Lilhore. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79b8d7ed Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79b8d7ed Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79b8d7ed Branch: refs/heads/HDFS-7240 Commit: 79b8d7ed384c8700bb000ef5e9ad00691404aea6 Parents: 52a53fe Author: Arpit Agarwal Authored: Mon Mar 6 16:39:53 2017 -0800 Committer: Anu Engineer Committed: Wed Mar 8 15:33:59 2017 -0800 ---------------------------------------------------------------------- .../hadoop/hdfs/protocol/DatanodeInfo.java | 52 +++++++++++++++++++- .../hadoop/hdfs/protocolPB/PBHelperClient.java | 8 ++- .../apache/hadoop/hdfs/web/JsonUtilClient.java | 2 + .../src/main/proto/hdfs.proto | 2 + .../server/blockmanagement/BlockManager.java | 3 ++ .../hdfs/server/namenode/FSNamesystem.java | 8 ++- .../org/apache/hadoop/hdfs/web/JsonUtil.java | 3 ++ .../src/main/webapps/hdfs/dfshealth.html | 2 + .../src/main/webapps/hdfs/dfshealth.js | 1 + .../server/namenode/TestNameNodeMXBean.java | 1 + 10 files changed, 78 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d7ed/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java index acbcffa..e1698c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java @@ -85,6 +85,8 @@ public class DatanodeInfo extends DatanodeID implements Node { protected AdminStates adminState; private long maintenanceExpireTimeInMS; + private long lastBlockReportTime; + private long lastBlockReportMonotonic; protected DatanodeInfo(DatanodeInfo from) { super(from); @@ -101,6 +103,8 @@ public class DatanodeInfo extends DatanodeID implements Node { this.location = from.getNetworkLocation(); this.adminState = from.getAdminState(); this.upgradeDomain = from.getUpgradeDomain(); + this.lastBlockReportTime = from.getLastBlockReportTime(); + this.lastBlockReportMonotonic = from.getLastBlockReportMonotonic(); } protected DatanodeInfo(DatanodeID nodeID) { @@ -116,6 +120,8 @@ public class DatanodeInfo extends DatanodeID implements Node { this.lastUpdateMonotonic = 0L; this.xceiverCount = 0; this.adminState = null; + this.lastBlockReportTime = 0L; + this.lastBlockReportMonotonic = 0L; } protected DatanodeInfo(DatanodeID nodeID, String location) { @@ -131,7 +137,8 @@ public class DatanodeInfo extends DatanodeID implements Node { final long blockPoolUsed, final long cacheCapacity, final long cacheUsed, final long lastUpdate, final long lastUpdateMonotonic, final int xceiverCount, final String networkLocation, - final AdminStates adminState, final String upgradeDomain) { + final AdminStates adminState, final String upgradeDomain, + final long lastBlockReportTime, final long lastBlockReportMonotonic) { super(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort, ipcPort); this.capacity = capacity; @@ -147,6 +154,8 @@ public class DatanodeInfo extends DatanodeID implements Node { this.location = networkLocation; this.adminState = adminState; this.upgradeDomain = upgradeDomain; + this.lastBlockReportTime = lastBlockReportTime; + this.lastBlockReportMonotonic = lastBlockReportMonotonic; } /** Network location name. */ @@ -391,6 +400,11 @@ public class DatanodeInfo extends DatanodeID implements Node { .append(percent2String(cacheRemainingPercent)).append("\n"); buffer.append("Xceivers: ").append(getXceiverCount()).append("\n"); buffer.append("Last contact: ").append(new Date(lastUpdate)).append("\n"); + buffer + .append("Last Block Report: ") + .append( + lastBlockReportTime != 0 ? new Date(lastBlockReportTime) : "Never") + .append("\n"); return buffer.toString(); } @@ -503,6 +517,26 @@ public class DatanodeInfo extends DatanodeID implements Node { return this.maintenanceExpireTimeInMS; } + /** Sets the last block report time. */ + public void setLastBlockReportTime(long lastBlockReportTime) { + this.lastBlockReportTime = lastBlockReportTime; + } + + /** Sets the last block report monotonic time. */ + public void setLastBlockReportMonotonic(long lastBlockReportMonotonic) { + this.lastBlockReportMonotonic = lastBlockReportMonotonic; + } + + /** Last block report time. */ + public long getLastBlockReportTime() { + return lastBlockReportTime; + } + + /** Last block report monotonic time. */ + public long getLastBlockReportMonotonic() { + return lastBlockReportMonotonic; + } + /** * Take the node out of maintenance mode. */ @@ -643,6 +677,8 @@ public class DatanodeInfo extends DatanodeID implements Node { private int infoSecurePort; private int ipcPort; private long nonDfsUsed = 0L; + private long lastBlockReportTime = 0L; + private long lastBlockReportMonotonic = 0L; public DatanodeInfoBuilder setFrom(DatanodeInfo from) { this.capacity = from.getCapacity(); @@ -658,6 +694,8 @@ public class DatanodeInfo extends DatanodeID implements Node { this.location = from.getNetworkLocation(); this.adminState = from.getAdminState(); this.upgradeDomain = from.getUpgradeDomain(); + this.lastBlockReportTime = from.getLastBlockReportTime(); + this.lastBlockReportMonotonic = from.getLastBlockReportMonotonic(); setNodeID(from); return this; } @@ -775,12 +813,22 @@ public class DatanodeInfo extends DatanodeID implements Node { return this; } + public DatanodeInfoBuilder setLastBlockReportTime(long time) { + this.lastBlockReportTime = time; + return this; + } + + public DatanodeInfoBuilder setLastBlockReportMonotonic(long time) { + this.lastBlockReportMonotonic = time; + return this; + } + public DatanodeInfo build() { return new DatanodeInfo(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort, ipcPort, capacity, dfsUsed, nonDfsUsed, remaining, blockPoolUsed, cacheCapacity, cacheUsed, lastUpdate, lastUpdateMonotonic, xceiverCount, location, adminState, - upgradeDomain); + upgradeDomain, lastBlockReportTime, lastBlockReportMonotonic); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d7ed/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index ad80bc2..b4fa926 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -303,6 +303,8 @@ public class PBHelperClient { .setLastUpdateMonotonic(info.getLastUpdateMonotonic()) .setXceiverCount(info.getXceiverCount()) .setAdminState(convert(info.getAdminState())) + .setLastBlockReportTime(info.getLastBlockReportTime()) + .setLastBlockReportMonotonic(info.getLastBlockReportMonotonic()) .build(); return builder.build(); } @@ -650,7 +652,11 @@ public class PBHelperClient { .setLastUpdateMonotonic(di.getLastUpdateMonotonic()) .setXceiverCount(di.getXceiverCount()) .setAdminState(convert(di.getAdminState())).setUpgradeDomain( - di.hasUpgradeDomain() ? di.getUpgradeDomain() : null); + di.hasUpgradeDomain() ? di.getUpgradeDomain() : null) + .setLastBlockReportTime(di.hasLastBlockReportTime() ? + di.getLastBlockReportTime() : 0) + .setLastBlockReportMonotonic(di.hasLastBlockReportMonotonic() ? + di.getLastBlockReportMonotonic() : 0); if (di.hasNonDfsUsed()) { dinfo.setNonDfsUsed(di.getNonDfsUsed()); } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d7ed/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java index 4204c54..cfcb4c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java @@ -295,6 +295,8 @@ class JsonUtilClient { DatanodeInfo.AdminStates .valueOf(getString(m, "adminState", "NORMAL"))) .setUpgradeDomain(getString(m, "upgradeDomain", "")) + .setLastBlockReportTime(getLong(m, "lastBlockReportTime", 0L)) + .setLastBlockReportMonotonic(getLong(m, "lastBlockReportMonotonic", 0L)) .build(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d7ed/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto index 3216e4e..b12ee2f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto @@ -103,6 +103,8 @@ message DatanodeInfoProto { optional uint64 cacheUsed = 12 [default = 0]; optional uint64 lastUpdateMonotonic = 13 [default = 0]; optional string upgradeDomain = 14; + optional uint64 lastBlockReportTime = 15 [default = 0]; + optional uint64 lastBlockReportMonotonic = 16 [default = 0]; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d7ed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 5ca0fa7..9ec28f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement; import static org.apache.hadoop.hdfs.protocol.BlockType.CONTIGUOUS; import static org.apache.hadoop.hdfs.protocol.BlockType.STRIPED; import static org.apache.hadoop.util.ExitUtil.terminate; +import static org.apache.hadoop.util.Time.now; import java.io.IOException; import java.io.PrintWriter; @@ -2377,6 +2378,8 @@ public class BlockManager implements BlockStatsMXBean { long leaseId = this.getBlockReportLeaseManager().removeLease(node); BlockManagerFaultInjector.getInstance(). removeBlockReportLease(node, leaseId); + node.setLastBlockReportTime(now()); + node.setLastBlockReportMonotonic(Time.monotonicNow()); } LOG.debug("Processing RPC with index {} out of total {} RPCs in " + "processReport 0x{}", context.getCurRpc(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d7ed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 81c5759..03277af 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -5472,7 +5472,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, .put("blockScheduled", node.getBlocksScheduled()) .put("blockPoolUsed", node.getBlockPoolUsed()) .put("blockPoolUsedPercent", node.getBlockPoolUsedPercent()) - .put("volfails", node.getVolumeFailures()); + .put("volfails", node.getVolumeFailures()) + // Block report time in minutes + .put("lastBlockReport", getLastBlockReport(node)); VolumeFailureSummary volumeFailureSummary = node.getVolumeFailureSummary(); if (volumeFailureSummary != null) { innerinfo @@ -5571,6 +5573,10 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, return (monotonicNow() - alivenode.getLastUpdateMonotonic())/1000; } + private Object getLastBlockReport(DatanodeDescriptor node) { + return (monotonicNow() - node.getLastBlockReportMonotonic()) / 60000; + } + private long getDfsUsed(DatanodeDescriptor alivenode) { return alivenode.getDfsUsed(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d7ed/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java index 0d89113..827e1a3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java @@ -181,6 +181,9 @@ public class JsonUtil { if (datanodeinfo.getUpgradeDomain() != null) { m.put("upgradeDomain", datanodeinfo.getUpgradeDomain()); } + m.put("lastBlockReportTime", datanodeinfo.getLastBlockReportTime()); + m.put("lastBlockReportMonotonic", + datanodeinfo.getLastBlockReportMonotonic()); return m; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d7ed/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index b33b9a2..c01f36c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -309,6 +309,7 @@ Node Http Address Last contact + Last Block Report Capacity Blocks Block pool used @@ -320,6 +321,7 @@ {name} ({xferaddr}) {dnWebAddress} {lastContact}s + {lastBlockReport}m
{capacity|fmt_bytes}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d7ed/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js index 88096d3..59958a1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js @@ -333,6 +333,7 @@ { 'orderDataType': 'ng-value', 'searchable': true }, { 'orderDataType': 'ng-value', 'type': 'numeric' }, { 'orderDataType': 'ng-value', 'type': 'numeric' }, + { 'orderDataType': 'ng-value', 'type': 'numeric' }, { 'orderData': 3, 'type': 'numeric' }, { 'orderDataType': 'ng-value', 'type': 'numeric'}, { 'orderData': 5 } http://git-wip-us.apache.org/repos/asf/hadoop/blob/79b8d7ed/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java index 037fd40..d4d4763 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java @@ -174,6 +174,7 @@ public class TestNameNodeMXBean { assertTrue(((Long)liveNode.get("capacity")) > 0); assertTrue(liveNode.containsKey("numBlocks")); assertTrue(((Long)liveNode.get("numBlocks")) == 0); + assertTrue(liveNode.containsKey("lastBlockReport")); // a. By default the upgrade domain isn't defined on any DN. // b. If the upgrade domain is set on a DN, JMX should have the same // value. --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org