Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D6641200C59 for ; Mon, 17 Apr 2017 19:28:11 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id D4EEF160BB1; Mon, 17 Apr 2017 17:28:11 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id A8B87160BB0 for ; Mon, 17 Apr 2017 19:28:10 +0200 (CEST) Received: (qmail 70725 invoked by uid 500); 17 Apr 2017 17:28:02 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 70396 invoked by uid 99); 17 Apr 2017 17:28:02 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 17 Apr 2017 17:28:02 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 052D6DFC70; Mon, 17 Apr 2017 17:28:02 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aengineer@apache.org To: common-commits@hadoop.apache.org Date: Mon, 17 Apr 2017 17:28:28 -0000 Message-Id: <7a4c084ced67442cb21b23dec42dacf4@git.apache.org> In-Reply-To: <6f0e0764edb649c798acbd7db5c2b794@git.apache.org> References: <6f0e0764edb649c798acbd7db5c2b794@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [29/50] hadoop git commit: HDFS-11062. Ozone:SCM: Remove null command. Contributed by Yuanbo Liu. archived-at: Mon, 17 Apr 2017 17:28:12 -0000 HDFS-11062. Ozone:SCM: Remove null command. Contributed by Yuanbo Liu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a2ff806d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a2ff806d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a2ff806d Branch: refs/heads/HDFS-7240 Commit: a2ff806d2e157b2c3d1f07732407705bfca20281 Parents: 349a19b Author: Xiaoyu Yao Authored: Tue Apr 11 13:33:40 2017 -0700 Committer: Xiaoyu Yao Committed: Tue Apr 11 13:33:40 2017 -0700 ---------------------------------------------------------------------- .../states/endpoint/HeartbeatEndpointTask.java | 5 -- .../ozone/protocol/commands/NullCommand.java | 81 -------------------- .../ozone/scm/StorageContainerManager.java | 44 ++++++----- .../hadoop/ozone/scm/node/CommandQueue.java | 9 +-- .../StorageContainerDatanodeProtocol.proto | 14 +--- .../ozone/container/common/ScmTestMock.java | 33 +++----- .../ozone/container/common/TestEndPoint.java | 7 +- .../ozone/scm/node/TestContainerPlacement.java | 4 +- 8 files changed, 44 insertions(+), 153 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2ff806d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java index 5aba687..c0d301e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java @@ -126,11 +126,6 @@ public class HeartbeatEndpointTask for (SCMCommandResponseProto commandResponseProto : response .getCommandsList()) { if (commandResponseProto.getCmdType() == - StorageContainerDatanodeProtocolProtos.Type.nullCmd) { - //this.context.addCommand(NullCommand.newBuilder().build()); - LOG.debug("Discarding a null command from SCM."); - } - if (commandResponseProto.getCmdType() == StorageContainerDatanodeProtocolProtos.Type.sendContainerReport) { this.context.addCommand(SendContainerCommand.getFromProtobuf( commandResponseProto.getSendReport())); http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2ff806d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/NullCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/NullCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/NullCommand.java deleted file mode 100644 index 7ae1117..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/NullCommand.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.protocol.commands; - -import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.Type; -import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.NullCmdResponseProto; - - - -/** - * For each command that SCM can return we have a class in this commands - * directory. This is the Null command, that tells datanode that no action is - * needed from it. - */ -public class NullCommand extends SCMCommand { - /** - * Returns the type of this command. - * - * @return Type - */ - @Override - public Type getType() { - return Type.nullCmd; - } - - /** - * Gets the protobuf message of this object. - * - * @return A protobuf message. - */ - @Override - public byte[] getProtoBufMessage() { - return NullCmdResponseProto.newBuilder().build().toByteArray(); - } - - /** - * Returns a NullCommand class from NullCommandResponse Proto. - * @param unused - unused - * @return NullCommand - */ - public static NullCommand getFromProtobuf(final NullCmdResponseProto - unused) { - return new NullCommand(); - } - - /** - * returns a new builder. - * @return Builder - */ - public static Builder newBuilder() { - return new Builder(); - } - - /** - * A Builder class this is the standard pattern we are using for all commands. - */ - public static class Builder { - /** - * Return a null command. - * @return - NullCommand. - */ - public NullCommand build() { - return new NullCommand(); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2ff806d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java index 91a7376..bdc0418 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java @@ -43,8 +43,6 @@ import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocol.proto .StorageContainerDatanodeProtocolProtos.ReportState; import org.apache.hadoop.ozone.protocol.proto - .StorageContainerDatanodeProtocolProtos.NullCmdResponseProto; -import org.apache.hadoop.ozone.protocol.proto .StorageContainerDatanodeProtocolProtos.SCMCommandResponseProto; import org.apache.hadoop.ozone.protocol.proto .StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto; @@ -59,6 +57,8 @@ import org.apache.hadoop.ozone.protocol.proto import org.apache.hadoop.ozone.protocol.proto .StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto; import org.apache.hadoop.ozone.protocol.proto + .StorageContainerDatanodeProtocolProtos.SendContainerReportProto; +import org.apache.hadoop.ozone.protocol.proto .StorageContainerDatanodeProtocolProtos.Type; import org.apache.hadoop.ozone.protocol.proto .StorageContainerLocationProtocolProtos; @@ -296,26 +296,27 @@ public class StorageContainerManager public static SCMCommandResponseProto getCommandResponse(SCMCommand cmd) throws InvalidProtocolBufferException { Type type = cmd.getType(); + SCMCommandResponseProto.Builder builder = + SCMCommandResponseProto.newBuilder(); switch (type) { - case nullCmd: - return getNullCmdResponse(); + case registeredCommand: + return builder.setCmdType(Type.registeredCommand) + .setRegisteredProto( + SCMRegisteredCmdResponseProto.getDefaultInstance()) + .build(); + case versionCommand: + return builder.setCmdType(Type.versionCommand) + .setVersionProto(SCMVersionResponseProto.getDefaultInstance()) + .build(); + case sendContainerReport: + return builder.setCmdType(Type.sendContainerReport) + .setSendReport(SendContainerReportProto.getDefaultInstance()) + .build(); default: throw new IllegalArgumentException("Not implemented"); } } - /** - * Returns a null command response. - * @return - * @throws InvalidProtocolBufferException - */ - private static SCMCommandResponseProto getNullCmdResponse() { - return SCMCommandResponseProto.newBuilder() - .setCmdType(Type.nullCmd) - .setNullCommand(NullCmdResponseProto.getDefaultInstance()) - .build(); - } - @VisibleForTesting public static SCMRegisteredCmdResponseProto getRegisteredResponse( SCMCommand cmd, SCMNodeAddressList addressList) { @@ -465,11 +466,11 @@ public class StorageContainerManager SCMNodeReport nodeReport, ReportState reportState) throws IOException { List commands = getScmNodeManager().sendHeartbeat(datanodeID, nodeReport); - List cmdReponses = new LinkedList<>(); + List cmdResponses = new LinkedList<>(); for (SCMCommand cmd : commands) { - cmdReponses.add(getCommandResponse(cmd)); + cmdResponses.add(getCommandResponse(cmd)); } - return SCMHeartbeatResponseProto.newBuilder().addAllCommands(cmdReponses) + return SCMHeartbeatResponseProto.newBuilder().addAllCommands(cmdResponses) .build(); } @@ -501,8 +502,9 @@ public class StorageContainerManager sendContainerReport(ContainerReportsProto reports) throws IOException { // TODO : fix this in the server side code changes for handling this request // correctly. - return SCMHeartbeatResponseProto.newBuilder() - .addCommands(getNullCmdResponse()).build(); + List cmdResponses = new LinkedList<>(); + return SCMHeartbeatResponseProto.newBuilder().addAllCommands(cmdResponses) + .build(); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2ff806d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/CommandQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/CommandQueue.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/CommandQueue.java index 9a49fc5..2c6cae8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/CommandQueue.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/CommandQueue.java @@ -18,7 +18,6 @@ package org.apache.hadoop.ozone.scm.node; import org.apache.hadoop.hdfs.protocol.DatanodeID; -import org.apache.hadoop.ozone.protocol.commands.NullCommand; import org.apache.hadoop.ozone.protocol.commands.SCMCommand; import java.util.HashMap; @@ -39,7 +38,7 @@ public class CommandQueue { private final Map> commandMap; private final Lock lock; - // This map is used as default return value containing one null command. + // This map is used as default return value. private static final List DEFAULT_LIST = new LinkedList<>(); /** @@ -48,12 +47,11 @@ public class CommandQueue { public CommandQueue() { commandMap = new HashMap<>(); lock = new ReentrantLock(); - DEFAULT_LIST.add(NullCommand.newBuilder().build()); } /** * Returns a list of Commands for the datanode to execute, if we have no - * commands returns a list with Null Command otherwise the current set of + * commands returns a empty list otherwise the current set of * commands are returned and command map set to empty list again. * * @param datanodeID DatanodeID @@ -67,8 +65,7 @@ public class CommandQueue { if (commandMap.containsKey(datanodeID)) { List temp = commandMap.get(datanodeID); if (temp.size() > 0) { - LinkedList emptyList = new LinkedList<>(); - commandMap.put(datanodeID, emptyList); + commandMap.put(datanodeID, DEFAULT_LIST); return temp; } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2ff806d/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto index 545f05f..db814c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto @@ -181,12 +181,6 @@ message ContainerNodeIDProto { } -/** - * Empty Command Response - */ -message NullCmdResponseProto { - -} /** This command tells the data node to send in the container report when possible @@ -198,7 +192,6 @@ message SendContainerReportProto { Type of commands supported by SCM to datanode protocol. */ enum Type { - nullCmd = 1; versionCommand = 2; registeredCommand = 3; sendContainerReport = 4; @@ -209,10 +202,9 @@ enum Type { */ message SCMCommandResponseProto { required Type cmdType = 2; // Type of the command - optional NullCmdResponseProto nullCommand = 3; - optional SCMRegisteredCmdResponseProto registeredProto = 4; - optional SCMVersionResponseProto versionProto = 5; - optional SendContainerReportProto sendReport = 6; + optional SCMRegisteredCmdResponseProto registeredProto = 3; + optional SCMVersionResponseProto versionProto = 4; + optional SendContainerReportProto sendReport = 5; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2ff806d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java index 5e8d13b..c0bdd9e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/ScmTestMock.java @@ -20,13 +20,16 @@ import com.google.common.base.Preconditions; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol; import org.apache.hadoop.ozone.protocol.VersionResponse; -import org.apache.hadoop.ozone.protocol.commands.NullCommand; import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto; +import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandResponseProto; import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState; import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMNodeReport; import org.apache.hadoop.ozone.scm.VersionInfo; import java.io.IOException; +import java.util.LinkedList; +import java.util.List; import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; @@ -137,25 +140,10 @@ public class ScmTestMock implements StorageContainerDatanodeProtocol { heartbeatCount.incrementAndGet(); this.reportState = reportState; sleepIfNeeded(); - return getNullRespose(); - } - - private StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto - getNullRespose() throws - com.google.protobuf.InvalidProtocolBufferException { - StorageContainerDatanodeProtocolProtos.SCMCommandResponseProto - cmdResponse = StorageContainerDatanodeProtocolProtos - .SCMCommandResponseProto - .newBuilder().setCmdType(StorageContainerDatanodeProtocolProtos - .Type.nullCmd) - .setNullCommand( - StorageContainerDatanodeProtocolProtos.NullCmdResponseProto - .parseFrom( - NullCommand.newBuilder().build().getProtoBufMessage())) + List + cmdResponses = new LinkedList<>(); + return SCMHeartbeatResponseProto.newBuilder().addAllCommands(cmdResponses) .build(); - return StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto - .newBuilder() - .addCommands(cmdResponse).build(); } /** @@ -188,13 +176,16 @@ public class ScmTestMock implements StorageContainerDatanodeProtocol { * @throws IOException */ @Override - public StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto + public SCMHeartbeatResponseProto sendContainerReport(StorageContainerDatanodeProtocolProtos .ContainerReportsProto reports) throws IOException { Preconditions.checkNotNull(reports); containerReportsCount.incrementAndGet(); closedContainerCount.addAndGet(reports.getReportsCount()); - return getNullRespose(); + List + cmdResponses = new LinkedList<>(); + return SCMHeartbeatResponseProto.newBuilder().addAllCommands(cmdResponses) + .build(); } public ReportState getReportState() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2ff806d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java index 3c2c8ca..3c3f3b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java @@ -47,8 +47,6 @@ import org.apache.hadoop.ozone.protocol.proto .StorageContainerDatanodeProtocolProtos.SCMStorageReport; import org.apache.hadoop.ozone.protocol.proto .StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto; -import org.apache.hadoop.ozone.protocol.proto - .StorageContainerDatanodeProtocolProtos.Type; import org.apache.hadoop.ozone.scm.VersionInfo; import org.apache.hadoop.test.PathUtils; import org.apache.hadoop.util.Time; @@ -286,10 +284,7 @@ public class TestEndPoint { SCMHeartbeatResponseProto responseProto = rpcEndPoint.getEndPoint() .sendHeartbeat(dataNode, nrb.build(), defaultReportState); Assert.assertNotNull(responseProto); - Assert.assertEquals(1, responseProto.getCommandsCount()); - Assert.assertNotNull(responseProto.getCommandsList().get(0)); - Assert.assertEquals(responseProto.getCommandsList().get(0).getCmdType(), - Type.nullCmd); + Assert.assertEquals(0, responseProto.getCommandsCount()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a2ff806d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerPlacement.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerPlacement.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerPlacement.java index d28866e..da0ed16 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerPlacement.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/scm/node/TestContainerPlacement.java @@ -176,8 +176,8 @@ public class TestContainerPlacement { thrown.expect(IOException.class); thrown.expectMessage( - startsWith("Unable to find enough nodes that meet the space " + - "requirement in healthy node set.")); + startsWith("Unable to find enough nodes that meet " + + "the space requirement")); String container2 = UUID.randomUUID().toString(); containerManager.allocateContainer(container2, ScmClient.ReplicationFactor.THREE); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org