Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A479F189AA for ; Fri, 15 Jan 2016 14:26:10 +0000 (UTC) Received: (qmail 57810 invoked by uid 500); 15 Jan 2016 14:26:10 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 57779 invoked by uid 500); 15 Jan 2016 14:26:10 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 57691 invoked by uid 99); 15 Jan 2016 14:26:10 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 15 Jan 2016 14:26:10 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3895DE04D9; Fri, 15 Jan 2016 14:26:10 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: mpapirkovskyy@apache.org To: commits@ambari.apache.org Date: Fri, 15 Jan 2016 14:26:11 -0000 Message-Id: <02b4a789036344f9b998bffcc8316269@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [2/2] ambari git commit: AMBARI-14669. Add yarn-site properties for Spark Shuffle Aux services (2.2->2.4 upgrade path). (Dmytro Grinenko via mpapirkovskyy) AMBARI-14669. Add yarn-site properties for Spark Shuffle Aux services (2.2->2.4 upgrade path). (Dmytro Grinenko via mpapirkovskyy) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e0957d39 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e0957d39 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e0957d39 Branch: refs/heads/trunk Commit: e0957d397a2780683cc3387c30359c68669a9343 Parents: 672fb60 Author: Myroslav Papirkovskyy Authored: Fri Jan 15 16:24:56 2016 +0200 Committer: Myroslav Papirkovskyy Committed: Fri Jan 15 16:25:41 2016 +0200 ---------------------------------------------------------------------- .../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml | 4 + .../stacks/HDP/2.2/upgrades/upgrade-2.4.xml | 1 + .../SparkShufflePropertyConfigTest.java | 169 +++++++++++++++++++ 3 files changed, 174 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e0957d39/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml index 1efeb6c..00b9e96 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml @@ -395,6 +395,10 @@ + + + +

http://git-wip-us.apache.org/repos/asf/ambari/blob/e0957d39/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml index acebdd8..7637830 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml @@ -566,6 +566,7 @@ + http://git-wip-us.apache.org/repos/asf/ambari/blob/e0957d39/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java new file mode 100644 index 0000000..edba6f0 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.serveraction.upgrades; + +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; + +import org.apache.ambari.server.ServiceNotFoundException; +import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; +import org.apache.ambari.server.actionmanager.HostRoleCommand; +import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; +import org.apache.ambari.server.state.ConfigImpl; +import org.apache.ambari.server.state.Service; +import org.easymock.EasyMock; +import org.junit.Before; +import org.junit.Test; + +import com.google.inject.Injector; + +/** + * Tests upgrade-related server side actions for SparkShufflePropertyConfig + */ +public class SparkShufflePropertyConfigTest { + private Injector m_injector; + private Clusters m_clusters; + private Cluster cluster; + private Field clusterField; + + @Before + public void setup() throws Exception { + m_injector = EasyMock.createMock(Injector.class); + m_clusters = EasyMock.createMock(Clusters.class); + cluster = EasyMock.createMock(Cluster.class); + + + Config adminConfig = new ConfigImpl("yarn-site") { + Map mockProperties = new HashMap() {{ + put("yarn.nodemanager.aux-services", "some_service"); + }}; + @Override + public Map getProperties() { + return mockProperties; + } + + @Override + public void setProperties(Map properties) { + mockProperties.putAll(properties); + } + + @Override + public void persist(boolean newConfig) { + // no-op + } + }; + + expect(cluster.getDesiredConfigByType("yarn-site")).andReturn(adminConfig).atLeastOnce(); + + expect(m_clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes(); + expect(m_injector.getInstance(Clusters.class)).andReturn(m_clusters).atLeastOnce(); + + replay(m_injector, m_clusters); + + clusterField = SparkShufflePropertyConfig.class.getDeclaredField("clusters"); + clusterField.setAccessible(true); + } + + @Test + public void testSparkPresent() throws Exception { + ExecutionCommand executionCommand = new ExecutionCommand(); + Map commandParams = new HashMap<>(); + HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class); + Service sparkService = EasyMock.createMock(Service.class); + SparkShufflePropertyConfig action = new SparkShufflePropertyConfig(); + + commandParams.put("clusterName", "c1"); + executionCommand.setCommandParams(commandParams); + executionCommand.setClusterName("c1"); + + // test calculations when spark service present + expect(cluster.getService("SPARK")).andReturn(sparkService).atLeastOnce(); + + expect(hrc.getRequestId()).andReturn(1L).anyTimes(); + expect(hrc.getStageId()).andReturn(2L).anyTimes(); + expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes(); + replay(cluster, hrc); + + clusterField.set(action, m_clusters); + action.setExecutionCommand(executionCommand); + action.setHostRoleCommand(hrc); + + CommandReport report = action.execute(null); + assertNotNull(report); + + Cluster c = m_clusters.getCluster("c1"); + Config config = c.getDesiredConfigByType("yarn-site"); + Map map = config.getProperties(); + + assertTrue(map.containsKey("yarn.nodemanager.aux-services")); + assertTrue(map.containsKey("yarn.nodemanager.aux-services.spark_shuffle.class")); + + assertEquals("some_service,spark_shuffle", map.get("yarn.nodemanager.aux-services")); + assertEquals("org.apache.spark.network.yarn.YarnShuffleService", map.get("yarn.nodemanager.aux-services.spark_shuffle.class")); + } + + @Test + public void testSparkNotPresent() throws Exception{ + SparkShufflePropertyConfig action = new SparkShufflePropertyConfig(); + Map commandParams = new HashMap<>(); + ExecutionCommand executionCommand = new ExecutionCommand(); + HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class); + ServiceNotFoundException notFoundException = new ServiceNotFoundException("c1", "SPARK"); + + commandParams.put("clusterName", "c1"); + executionCommand.setCommandParams(commandParams); + executionCommand.setClusterName("c1"); + + //no spark service present + expect(cluster.getService("SPARK")).andThrow(notFoundException).atLeastOnce(); + + expect(hrc.getRequestId()).andReturn(1L).anyTimes(); + expect(hrc.getStageId()).andReturn(2L).anyTimes(); + expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes(); + replay(cluster, hrc); + + clusterField.set(action, m_clusters); + action.setExecutionCommand(executionCommand); + action.setHostRoleCommand(hrc); + + CommandReport report = action.execute(null); + assertNotNull(report); + + Cluster c = m_clusters.getCluster("c1"); + Config config = c.getDesiredConfigByType("yarn-site"); + Map map = config.getProperties(); + + assertTrue(map.containsKey("yarn.nodemanager.aux-services")); + assertFalse(map.containsKey("yarn.nodemanager.aux-services.spark_shuffle.class")); + assertEquals("some_service", map.get("yarn.nodemanager.aux-services")); + } +}