ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mpapirkovs...@apache.org
Subject [2/2] ambari git commit: AMBARI-14669. Add yarn-site properties for Spark Shuffle Aux services (2.2->2.4 upgrade path). (Dmytro Grinenko via mpapirkovskyy)
Date Fri, 15 Jan 2016 14:26:11 GMT
AMBARI-14669. Add yarn-site properties for Spark Shuffle Aux services (2.2->2.4 upgrade
path). (Dmytro Grinenko via mpapirkovskyy)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e0957d39
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e0957d39
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e0957d39

Branch: refs/heads/trunk
Commit: e0957d397a2780683cc3387c30359c68669a9343
Parents: 672fb60
Author: Myroslav Papirkovskyy <mpapyrkovskyy@hortonworks.com>
Authored: Fri Jan 15 16:24:56 2016 +0200
Committer: Myroslav Papirkovskyy <mpapyrkovskyy@hortonworks.com>
Committed: Fri Jan 15 16:25:41 2016 +0200

----------------------------------------------------------------------
 .../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml |   4 +
 .../stacks/HDP/2.2/upgrades/upgrade-2.4.xml     |   1 +
 .../SparkShufflePropertyConfigTest.java         | 169 +++++++++++++++++++
 3 files changed, 174 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/e0957d39/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
index 1efeb6c..00b9e96 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
@@ -395,6 +395,10 @@
         <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/>
       </execute-stage>
 
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn
Properties for Spark">
+        <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark"
class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
+      </execute-stage>
+
       <!--HBASE-->
       <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes
for HBase Master">
         <task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_phoenix_scheduler_factory"/>

http://git-wip-us.apache.org/repos/asf/ambari/blob/e0957d39/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
index acebdd8..7637830 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
@@ -566,6 +566,7 @@
 
           <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/>
 
+          <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark"
class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
         </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task"/>

http://git-wip-us.apache.org/repos/asf/ambari/blob/e0957d39/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
new file mode 100644
index 0000000..edba6f0
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.upgrades;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.ambari.server.ServiceNotFoundException;
+import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
+import org.apache.ambari.server.actionmanager.HostRoleCommand;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ConfigImpl;
+import org.apache.ambari.server.state.Service;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.inject.Injector;
+
+/**
+ * Tests upgrade-related server side actions for SparkShufflePropertyConfig
+ */
+public class SparkShufflePropertyConfigTest {
+  private Injector m_injector;
+  private Clusters m_clusters;
+  private Cluster cluster;
+  private Field clusterField;
+
+  @Before
+  public void setup() throws Exception {
+    m_injector = EasyMock.createMock(Injector.class);
+    m_clusters = EasyMock.createMock(Clusters.class);
+    cluster = EasyMock.createMock(Cluster.class);
+
+
+    Config adminConfig = new ConfigImpl("yarn-site") {
+      Map<String, String> mockProperties = new HashMap<String, String>() {{
+        put("yarn.nodemanager.aux-services", "some_service");
+      }};
+      @Override
+      public Map<String, String> getProperties() {
+        return mockProperties;
+      }
+
+      @Override
+      public void setProperties(Map<String, String> properties) {
+        mockProperties.putAll(properties);
+      }
+
+      @Override
+      public void persist(boolean newConfig) {
+        // no-op
+      }
+    };
+
+    expect(cluster.getDesiredConfigByType("yarn-site")).andReturn(adminConfig).atLeastOnce();
+
+    expect(m_clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes();
+    expect(m_injector.getInstance(Clusters.class)).andReturn(m_clusters).atLeastOnce();
+
+    replay(m_injector, m_clusters);
+
+    clusterField = SparkShufflePropertyConfig.class.getDeclaredField("clusters");
+    clusterField.setAccessible(true);
+  }
+
+  @Test
+  public void testSparkPresent() throws Exception {
+    ExecutionCommand executionCommand = new ExecutionCommand();
+    Map<String, String> commandParams = new HashMap<>();
+    HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
+    Service sparkService = EasyMock.createMock(Service.class);
+    SparkShufflePropertyConfig action = new SparkShufflePropertyConfig();
+
+    commandParams.put("clusterName", "c1");
+    executionCommand.setCommandParams(commandParams);
+    executionCommand.setClusterName("c1");
+
+    // test calculations when spark service present
+    expect(cluster.getService("SPARK")).andReturn(sparkService).atLeastOnce();
+
+    expect(hrc.getRequestId()).andReturn(1L).anyTimes();
+    expect(hrc.getStageId()).andReturn(2L).anyTimes();
+    expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
+    replay(cluster, hrc);
+
+    clusterField.set(action, m_clusters);
+    action.setExecutionCommand(executionCommand);
+    action.setHostRoleCommand(hrc);
+
+    CommandReport report = action.execute(null);
+    assertNotNull(report);
+
+    Cluster c = m_clusters.getCluster("c1");
+    Config config = c.getDesiredConfigByType("yarn-site");
+    Map<String, String> map = config.getProperties();
+
+    assertTrue(map.containsKey("yarn.nodemanager.aux-services"));
+    assertTrue(map.containsKey("yarn.nodemanager.aux-services.spark_shuffle.class"));
+
+    assertEquals("some_service,spark_shuffle", map.get("yarn.nodemanager.aux-services"));
+    assertEquals("org.apache.spark.network.yarn.YarnShuffleService", map.get("yarn.nodemanager.aux-services.spark_shuffle.class"));
+  }
+
+  @Test
+  public void testSparkNotPresent() throws Exception{
+    SparkShufflePropertyConfig action = new SparkShufflePropertyConfig();
+    Map<String, String> commandParams = new HashMap<>();
+    ExecutionCommand executionCommand = new ExecutionCommand();
+    HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
+    ServiceNotFoundException notFoundException = new ServiceNotFoundException("c1", "SPARK");
+
+    commandParams.put("clusterName", "c1");
+    executionCommand.setCommandParams(commandParams);
+    executionCommand.setClusterName("c1");
+
+    //no spark service present
+    expect(cluster.getService("SPARK")).andThrow(notFoundException).atLeastOnce();
+
+    expect(hrc.getRequestId()).andReturn(1L).anyTimes();
+    expect(hrc.getStageId()).andReturn(2L).anyTimes();
+    expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
+    replay(cluster, hrc);
+
+    clusterField.set(action, m_clusters);
+    action.setExecutionCommand(executionCommand);
+    action.setHostRoleCommand(hrc);
+
+    CommandReport report = action.execute(null);
+    assertNotNull(report);
+
+    Cluster c = m_clusters.getCluster("c1");
+    Config config = c.getDesiredConfigByType("yarn-site");
+    Map<String, String> map = config.getProperties();
+
+    assertTrue(map.containsKey("yarn.nodemanager.aux-services"));
+    assertFalse(map.containsKey("yarn.nodemanager.aux-services.spark_shuffle.class"));
+    assertEquals("some_service", map.get("yarn.nodemanager.aux-services"));
+  }
+}


Mime
View raw message