Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 22B4D200C5C for ; Thu, 20 Apr 2017 20:32:31 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 21672160B90; Thu, 20 Apr 2017 18:32:31 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 412A4160B9F for ; Thu, 20 Apr 2017 20:32:30 +0200 (CEST) Received: (qmail 60037 invoked by uid 500); 20 Apr 2017 18:32:29 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 59954 invoked by uid 99); 20 Apr 2017 18:32:29 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 20 Apr 2017 18:32:29 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 048DFF4A1D; Thu, 20 Apr 2017 18:32:29 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: xyao@apache.org To: common-commits@hadoop.apache.org Message-Id: <57e5efb8cbcb4ce8a54bd50e277d4c98@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-11668. Ozone: misc improvements for SCM CLI. Contributed by Weiwei Yang. Date: Thu, 20 Apr 2017 18:32:29 +0000 (UTC) archived-at: Thu, 20 Apr 2017 18:32:31 -0000 Repository: hadoop Updated Branches: refs/heads/HDFS-7240 9154d77ca -> a2c6f594b HDFS-11668. Ozone: misc improvements for SCM CLI. Contributed by Weiwei Yang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a2c6f594 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a2c6f594 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a2c6f594 Branch: refs/heads/HDFS-7240 Commit: a2c6f594baab7357cf83749986c31d07acbb1980 Parents: 9154d77 Author: Xiaoyu Yao Authored: Thu Apr 20 11:32:13 2017 -0700 Committer: Xiaoyu Yao Committed: Thu Apr 20 11:32:13 2017 -0700 ---------------------------------------------------------------------- .../hadoop-hdfs/src/main/bin/hdfs | 12 ++++-- .../common/statemachine/StateContext.java | 8 ++++ .../states/datanode/InitDatanodeState.java | 6 +++ .../cli/container/CreateContainerHandler.java | 2 +- .../common/TestDatanodeStateMachine.java | 45 +++++++++++++++----- 5 files changed, 57 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2c6f594/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index 90b2ae3..9d21f60 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -200,10 +200,14 @@ function hdfscmd_case HADOOP_CLASSNAME=org.apache.hadoop.portmap.Portmap ;; scm) - HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" - HADOOP_CLASSNAME='org.apache.hadoop.ozone.scm.StorageContainerManager' - hadoop_debug "Appending HADOOP_SCM_OPTS onto HADOOP_OPTS" - HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SCM_OPTS}" + if [ "$#" -gt 0 ]; then + HADOOP_CLASSNAME=org.apache.hadoop.ozone.scm.cli.SCMCLI + else + HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" + HADOOP_CLASSNAME='org.apache.hadoop.ozone.scm.StorageContainerManager' + hadoop_debug "Appending HADOOP_SCM_OPTS onto HADOOP_OPTS" + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SCM_OPTS}" + fi ;; secondarynamenode) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2c6f594/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java index e020791..eb2ca3a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java @@ -23,6 +23,8 @@ import org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeS import org.apache.hadoop.ozone.protocol.commands.SCMCommand; import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMNodeReport; import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.LinkedList; import java.util.Queue; @@ -42,6 +44,8 @@ import static org.apache.hadoop.ozone.protocol.proto * Current Context of State Machine. */ public class StateContext { + static final Logger LOG = + LoggerFactory.getLogger(StateContext.class); private final Queue commandQueue; private final Lock lock; private final DatanodeStateMachine parent; @@ -187,6 +191,10 @@ public class StateContext { task.execute(service); DatanodeStateMachine.DatanodeStates newState = task.await(time, unit); if (this.state != newState) { + if (LOG.isDebugEnabled()) { + LOG.debug("Task {} executed, state transited from {} to {}", + task.getClass().getSimpleName(), this.state, newState); + } if (isExiting(newState)) { task.onExit(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2c6f594/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java index 0427392..3720ec8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java @@ -105,6 +105,12 @@ public class InitDatanodeState implements DatanodeState, */ private void persistContainerDatanodeID() throws IOException { String dataNodeIDPath = conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID); + if (Strings.isNullOrEmpty(dataNodeIDPath)) { + LOG.error("A valid file path is needed for config setting {}", + ScmConfigKeys.OZONE_SCM_DATANODE_ID); + this.context.setState(DatanodeStateMachine.DatanodeStates.SHUTDOWN); + return; + } File idPath = new File(dataNodeIDPath); int containerPort = this.context.getContainerPort(); DatanodeID datanodeID = this.context.getParent().getDatanodeID(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2c6f594/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CreateContainerHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CreateContainerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CreateContainerHandler.java index 9b30f0d..dd0b1e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CreateContainerHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CreateContainerHandler.java @@ -56,7 +56,7 @@ public class CreateContainerHandler extends OzoneCommandHandler { } } String pipelineID = cmd.getOptionValue(PIPELINE_ID); - LOG.info("Create container :" + pipelineID + " " + getScmClient()); + LOG.info("Create container : {}", pipelineID); getScmClient().createContainer(pipelineID); LOG.debug("Container creation returned"); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2c6f594/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java index c9bd32a..9c2a0bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java @@ -16,6 +16,7 @@ */ package org.apache.hadoop.ozone.container.common; +import com.google.common.collect.Maps; import com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.DFSTestUtil; @@ -46,6 +47,7 @@ import java.net.URL; import java.nio.file.Paths; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; @@ -294,21 +296,40 @@ public class TestDatanodeStateMachine { } /** - * Test state transition with a list of invalid SCM names, + * Test state transition with a list of invalid scm configurations, * and verify the state transits to SHUTDOWN each time. */ @Test - public void testDatanodeStateMachineWithInvalidSCMNames() + public void testDatanodeStateMachineWithInvalidConfiguration() throws Exception { - for (String name : new String[] { - "", // Empty - "x..y", // Invalid schema - "scm:xyz", // Invalid port - "scm:123456" // Port out of range - }) { - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, name); + + LinkedList> confList = + new LinkedList>(); + confList.add(Maps.immutableEntry(ScmConfigKeys.OZONE_SCM_NAMES, "")); + + // Invalid ozone.scm.names + /** Empty **/ + confList.add(Maps.immutableEntry( + ScmConfigKeys.OZONE_SCM_NAMES, "")); + /** Invalid schema **/ + confList.add(Maps.immutableEntry( + ScmConfigKeys.OZONE_SCM_NAMES, "x..y")); + /** Invalid port **/ + confList.add(Maps.immutableEntry( + ScmConfigKeys.OZONE_SCM_NAMES, "scm:xyz")); + /** Port out of range **/ + confList.add(Maps.immutableEntry( + ScmConfigKeys.OZONE_SCM_NAMES, "scm:123456")); + // Invalid ozone.scm.datanode.id + /** Empty **/ + confList.add(Maps.immutableEntry( + ScmConfigKeys.OZONE_SCM_DATANODE_ID, "")); + + confList.forEach((entry) -> { + Configuration perTestConf = new Configuration(conf); + perTestConf.setStrings(entry.getKey(), entry.getValue()); try (DatanodeStateMachine stateMachine = - new DatanodeStateMachine(conf)) { + new DatanodeStateMachine(perTestConf)) { DatanodeStateMachine.DatanodeStates currentState = stateMachine.getContext().getState(); Assert.assertEquals(DatanodeStateMachine.DatanodeStates.INIT, @@ -320,7 +341,9 @@ public class TestDatanodeStateMachine { task.await(2, TimeUnit.SECONDS); Assert.assertEquals(DatanodeStateMachine.DatanodeStates.SHUTDOWN, newState); + } catch (Exception e) { + Assert.fail("Unexpected exception found"); } - } + }); } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org