Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 7C6F5200BCF for ; Mon, 5 Dec 2016 21:08:16 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 7AFAE160B18; Mon, 5 Dec 2016 20:08:16 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 5846C160B09 for ; Mon, 5 Dec 2016 21:08:15 +0100 (CET) Received: (qmail 60170 invoked by uid 500); 5 Dec 2016 20:08:13 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 59706 invoked by uid 99); 5 Dec 2016 20:08:12 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 05 Dec 2016 20:08:12 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id AFE10E0159; Mon, 5 Dec 2016 20:08:12 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: asuresh@apache.org To: common-commits@hadoop.apache.org Date: Mon, 05 Dec 2016 20:08:17 -0000 Message-Id: <5e5f1a4d7feb4c97a84c6150a9082ce0@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [6/6] hadoop git commit: HDFS-11094. Send back HAState along with NamespaceInfo during a versionRequest as an optional parameter. Contributed by Eric Badger archived-at: Mon, 05 Dec 2016 20:08:16 -0000 HDFS-11094. Send back HAState along with NamespaceInfo during a versionRequest as an optional parameter. Contributed by Eric Badger Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8c468085 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8c468085 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8c468085 Branch: refs/heads/YARN-5085 Commit: 8c4680852b20ad0e65e77dd123c9ba5bb6f2fa39 Parents: 43ebff2 Author: Mingliang Liu Authored: Mon Dec 5 11:34:13 2016 -0800 Committer: Mingliang Liu Committed: Mon Dec 5 11:48:58 2016 -0800 ---------------------------------------------------------------------- .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 76 +++++++++++++------- .../hdfs/server/datanode/BPOfferService.java | 10 ++- .../hdfs/server/datanode/BPServiceActor.java | 4 +- .../hdfs/server/namenode/FSNamesystem.java | 8 ++- .../hdfs/server/protocol/NamespaceInfo.java | 26 +++++++ .../hadoop-hdfs/src/main/proto/HdfsServer.proto | 2 + .../server/datanode/TestBPOfferService.java | 31 ++++++++ .../hdfs/server/namenode/TestFSNamesystem.java | 21 ++++++ 8 files changed, 148 insertions(+), 30 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c468085/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index 78371f5..1e6d882 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -26,7 +26,7 @@ import com.google.protobuf.ByteString; import org.apache.hadoop.fs.StorageType; import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; -import org.apache.hadoop.ha.proto.HAServiceProtocolProtos; +import org.apache.hadoop.ha.proto.HAServiceProtocolProtos.HAServiceStateProto; import org.apache.hadoop.hdfs.DFSUtilClient; import org.apache.hadoop.hdfs.protocol.Block; import org.apache.hadoop.hdfs.protocol.DatanodeID; @@ -338,7 +338,8 @@ public class PBHelper { StorageInfoProto storage = info.getStorageInfo(); return new NamespaceInfo(storage.getNamespceID(), storage.getClusterID(), info.getBlockPoolID(), storage.getCTime(), info.getBuildVersion(), - info.getSoftwareVersion(), info.getCapabilities()); + info.getSoftwareVersion(), info.getCapabilities(), + convert(info.getState())); } public static NamenodeCommand convert(NamenodeCommandProto cmd) { @@ -744,43 +745,68 @@ public class PBHelper { } public static NamespaceInfoProto convert(NamespaceInfo info) { - return NamespaceInfoProto.newBuilder() - .setBlockPoolID(info.getBlockPoolID()) + NamespaceInfoProto.Builder builder = NamespaceInfoProto.newBuilder(); + builder.setBlockPoolID(info.getBlockPoolID()) .setBuildVersion(info.getBuildVersion()) .setUnused(0) .setStorageInfo(PBHelper.convert((StorageInfo)info)) .setSoftwareVersion(info.getSoftwareVersion()) - .setCapabilities(info.getCapabilities()) - .build(); + .setCapabilities(info.getCapabilities()); + HAServiceState state = info.getState(); + if(state != null) { + builder.setState(convert(info.getState())); + } + return builder.build(); } - public static NNHAStatusHeartbeat convert(NNHAStatusHeartbeatProto s) { - if (s == null) return null; - switch (s.getState()) { + public static HAServiceState convert(HAServiceStateProto s) { + if (s == null) { + return null; + } + switch (s) { + case INITIALIZING: + return HAServiceState.INITIALIZING; case ACTIVE: - return new NNHAStatusHeartbeat(HAServiceState.ACTIVE, s.getTxid()); + return HAServiceState.ACTIVE; case STANDBY: - return new NNHAStatusHeartbeat(HAServiceState.STANDBY, s.getTxid()); + return HAServiceState.STANDBY; default: - throw new IllegalArgumentException("Unexpected NNHAStatusHeartbeat.State:" + s.getState()); + throw new IllegalArgumentException("Unexpected HAServiceStateProto:" + + s); } } + public static HAServiceStateProto convert(HAServiceState s) { + if (s == null) { + return null; + } + switch (s) { + case INITIALIZING: + return HAServiceStateProto.INITIALIZING; + case ACTIVE: + return HAServiceStateProto.ACTIVE; + case STANDBY: + return HAServiceStateProto.STANDBY; + default: + throw new IllegalArgumentException("Unexpected HAServiceState:" + + s); + } + } + + public static NNHAStatusHeartbeat convert(NNHAStatusHeartbeatProto s) { + if (s == null) { + return null; + } + return new NNHAStatusHeartbeat(convert(s.getState()), s.getTxid()); + } + public static NNHAStatusHeartbeatProto convert(NNHAStatusHeartbeat hb) { - if (hb == null) return null; - NNHAStatusHeartbeatProto.Builder builder = - NNHAStatusHeartbeatProto.newBuilder(); - switch (hb.getState()) { - case ACTIVE: - builder.setState(HAServiceProtocolProtos.HAServiceStateProto.ACTIVE); - break; - case STANDBY: - builder.setState(HAServiceProtocolProtos.HAServiceStateProto.STANDBY); - break; - default: - throw new IllegalArgumentException("Unexpected NNHAStatusHeartbeat.State:" + - hb.getState()); + if (hb == null) { + return null; } + NNHAStatusHeartbeatProto.Builder builder = + NNHAStatusHeartbeatProto.newBuilder(); + builder.setState(convert(hb.getState())); builder.setTxid(hb.getTxId()); return builder.build(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c468085/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java index 00102eb..00e6b3e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java @@ -307,8 +307,16 @@ class BPOfferService { * verifies that this namespace matches (eg to prevent a misconfiguration * where a StandbyNode from a different cluster is specified) */ - void verifyAndSetNamespaceInfo(NamespaceInfo nsInfo) throws IOException { + void verifyAndSetNamespaceInfo(BPServiceActor actor, NamespaceInfo nsInfo) + throws IOException { writeLock(); + + if(nsInfo.getState() == HAServiceState.ACTIVE + && bpServiceToActive == null) { + LOG.info("Acknowledging ACTIVE Namenode during handshake" + actor); + bpServiceToActive = actor; + } + try { if (this.bpNSInfo == null) { this.bpNSInfo = nsInfo; http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c468085/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index f3247fc..dffe14f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -269,11 +269,11 @@ class BPServiceActor implements Runnable { // First phase of the handshake with NN - get the namespace // info. NamespaceInfo nsInfo = retrieveNamespaceInfo(); - + // Verify that this matches the other NN in this HA pair. // This also initializes our block pool in the DN if we are // the first NN connection for this BP. - bpos.verifyAndSetNamespaceInfo(nsInfo); + bpos.verifyAndSetNamespaceInfo(this, nsInfo); // Second phase of the handshake with the NN. register(nsInfo); http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c468085/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 8a750a0..90fb924 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -1594,7 +1594,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, NamespaceInfo unprotectedGetNamespaceInfo() { return new NamespaceInfo(getFSImage().getStorage().getNamespaceID(), getClusterId(), getBlockPoolId(), - getFSImage().getStorage().getCTime()); + getFSImage().getStorage().getCTime(), getState()); } /** @@ -4531,12 +4531,16 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, return 0; } } - + @Metric public int getBlockCapacity() { return blockManager.getCapacity(); } + public HAServiceState getState() { + return haContext == null ? null : haContext.getState().getServiceState(); + } + @Override // FSNamesystemMBean public String getFSState() { return isInSafeMode() ? "safeMode" : "Operational"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c468085/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java index 90d0aac..66ce9ee 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.StorageInfo; @@ -44,6 +45,7 @@ public class NamespaceInfo extends StorageInfo { String blockPoolID = ""; // id of the block pool String softwareVersion; long capabilities; + HAServiceState state; // only authoritative on the server-side to determine advertisement to // clients. enum will update the supported values @@ -88,6 +90,14 @@ public class NamespaceInfo extends StorageInfo { CAPABILITIES_SUPPORTED); } + public NamespaceInfo(int nsID, String clusterID, String bpID, + long cT, String buildVersion, String softwareVersion, + long capabilities, HAServiceState st) { + this(nsID, clusterID, bpID, cT, buildVersion, softwareVersion, + capabilities); + this.state = st; + } + // for use by server and/or client public NamespaceInfo(int nsID, String clusterID, String bpID, long cT, String buildVersion, String softwareVersion, @@ -105,6 +115,13 @@ public class NamespaceInfo extends StorageInfo { this(nsID, clusterID, bpID, cT, Storage.getBuildVersion(), VersionInfo.getVersion()); } + + public NamespaceInfo(int nsID, String clusterID, String bpID, + long cT, HAServiceState st) { + this(nsID, clusterID, bpID, cT, Storage.getBuildVersion(), + VersionInfo.getVersion()); + this.state = st; + } public long getCapabilities() { return capabilities; @@ -115,6 +132,11 @@ public class NamespaceInfo extends StorageInfo { this.capabilities = capabilities; } + @VisibleForTesting + public void setState(HAServiceState state) { + this.state = state; + } + public boolean isCapabilitySupported(Capability capability) { Preconditions.checkArgument(capability != Capability.UNKNOWN, "cannot test for unknown capability"); @@ -134,6 +156,10 @@ public class NamespaceInfo extends StorageInfo { return softwareVersion; } + public HAServiceState getState() { + return state; + } + @Override public String toString(){ return super.toString() + ";bpid=" + blockPoolID; http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c468085/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto index 910e03b..d7deebf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto @@ -32,6 +32,7 @@ option java_generate_equals_and_hash = true; package hadoop.hdfs; import "hdfs.proto"; +import "HAServiceProtocol.proto"; /** * Block access token information @@ -101,6 +102,7 @@ message NamespaceInfoProto { required StorageInfoProto storageInfo = 4;// Node information required string softwareVersion = 5; // Software version number (e.g. 2.0.0) optional uint64 capabilities = 6 [default = 0]; // feature flags + optional hadoop.common.HAServiceStateProto state = 7; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c468085/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java index 2d50c75..f8f0a3c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.datanode; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; @@ -799,4 +800,34 @@ public class TestBPOfferService { } return -1; } + + /* + * + */ + @Test + public void testNNHAStateUpdateFromVersionRequest() throws Exception { + final BPOfferService bpos = setupBPOSForNNs(mockNN1, mockNN2); + BPServiceActor actor = bpos.getBPServiceActors().get(0); + bpos.start(); + waitForInitialization(bpos); + // Should start with neither NN as active. + assertNull(bpos.getActiveNN()); + + // getNamespaceInfo() will not include HAServiceState + NamespaceInfo nsInfo = mockNN1.versionRequest(); + bpos.verifyAndSetNamespaceInfo(actor, nsInfo); + + assertNull(bpos.getActiveNN()); + + // Change mock so getNamespaceInfo() will include HAServiceState + Mockito.doReturn(new NamespaceInfo(1, FAKE_CLUSTERID, FAKE_BPID, 0, + HAServiceState.ACTIVE)).when(mockNN1).versionRequest(); + + // Update the bpos NamespaceInfo + nsInfo = mockNN1.versionRequest(); + bpos.verifyAndSetNamespaceInfo(actor, nsInfo); + + assertNotNull(bpos.getActiveNN()); + + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c468085/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java index f02c679..6a0dd6f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java @@ -33,6 +33,7 @@ import java.util.Collection; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.ha.HAServiceProtocol; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; @@ -43,6 +44,7 @@ import org.apache.hadoop.hdfs.server.namenode.ha.HAContext; import org.apache.hadoop.hdfs.server.namenode.ha.HAState; import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot; import org.apache.hadoop.hdfs.server.namenode.top.TopAuditLogger; +import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.junit.After; import org.junit.Test; import org.mockito.Mockito; @@ -155,6 +157,25 @@ public class TestFSNamesystem { } @Test + public void testHAStateInNamespaceInfo() throws IOException { + Configuration conf = new Configuration(); + + FSEditLog fsEditLog = Mockito.mock(FSEditLog.class); + FSImage fsImage = Mockito.mock(FSImage.class); + Mockito.when(fsImage.getEditLog()).thenReturn(fsEditLog); + NNStorage nnStorage = Mockito.mock(NNStorage.class); + Mockito.when(fsImage.getStorage()).thenReturn(nnStorage); + + FSNamesystem fsNamesystem = new FSNamesystem(conf, fsImage); + FSNamesystem fsn = Mockito.spy(fsNamesystem); + Mockito.when(fsn.getState()).thenReturn( + HAServiceProtocol.HAServiceState.ACTIVE); + + NamespaceInfo nsInfo = fsn.unprotectedGetNamespaceInfo(); + assertNotNull(nsInfo.getState()); + } + + @Test public void testReset() throws Exception { Configuration conf = new Configuration(); FSEditLog fsEditLog = Mockito.mock(FSEditLog.class); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org