ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject [2/2] ambari git commit: AMBARI-14721. yarn-site properties for Spark Shuffle Aux should be added unconditionally (aonishuk)
Date Tue, 19 Jan 2016 15:03:31 GMT
AMBARI-14721. yarn-site properties for Spark Shuffle Aux should be added unconditionally (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ce77df08
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ce77df08
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ce77df08

Branch: refs/heads/branch-2.2
Commit: ce77df08e9da72cffadb7ece2734cfbf31bd067b
Parents: bd4892b
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Tue Jan 19 17:03:18 2016 +0200
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Tue Jan 19 17:03:18 2016 +0200

----------------------------------------------------------------------
 .../upgrades/SparkShufflePropertyConfig.java    | 14 ++--
 .../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml |  2 +-
 .../stacks/HDP/2.2/upgrades/upgrade-2.4.xml     |  2 +-
 .../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml |  2 +-
 .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml     |  2 +-
 .../services/YARN/configuration/yarn-site.xml   | 14 +---
 .../stacks/HDP/2.4/services/stack_advisor.py    | 78 +-------------------
 .../SparkShufflePropertyConfigTest.java         | 43 +----------
 8 files changed, 12 insertions(+), 145 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
index efeb4a0..299a373 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
@@ -68,9 +68,6 @@ public class SparkShufflePropertyConfig extends AbstractServerAction {
       return  createCommandReport(0, HostRoleStatus.FAILED,"{}",
               String.format("Source type %s not found", YARN_SITE_CONFIG_TYPE), "");
     }
-
-    try {
-      cluster.getService("SPARK");  // check if SPARK service present
       Map<String, String> yarnSiteProperties = yarnSiteConfig.getProperties();
 
       final List<String> auxSevices;
@@ -82,7 +79,11 @@ public class SparkShufflePropertyConfig extends AbstractServerAction {
       } else {
         auxSevices = new ArrayList<>();
       }
-      auxSevices.add(SPARK_SHUFFLE_AUX_STR);
+
+      // check if spark is not already in the list
+      if (!auxSevices.contains(SPARK_SHUFFLE_AUX_STR)) {
+        auxSevices.add(SPARK_SHUFFLE_AUX_STR);
+      }
       newAuxServices = StringUtils.join(auxSevices, ",");
 
       yarnSiteProperties.put(YARN_NODEMANAGER_AUX_SERVICES, newAuxServices);
@@ -94,10 +95,5 @@ public class SparkShufflePropertyConfig extends AbstractServerAction {
         String.format("%s was set from %s to %s. %s was set to %s",
                 YARN_NODEMANAGER_AUX_SERVICES, oldAuxServices, newAuxServices,
                 YARN_NODEMANAGER_AUX_SERVICES_SPARK_SHUFFLE_CLASS_VALUE, YARN_NODEMANAGER_AUX_SERVICES_SPARK_SHUFFLE_CLASS_VALUE),
"");
-
-    } catch (ServiceNotFoundException e) {
-      return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
-          String.format("%s not updated as no SPARK service present on the cluster", YARN_NODEMANAGER_AUX_SERVICES),
"");
-    }
   }
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
index ff7e032..c5d74e2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
@@ -395,7 +395,7 @@
         <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/>
       </execute-stage>
 
-      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn
Properties for Spark">
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn
Properties for Spark Shuffle">
         <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark"
class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
       </execute-stage>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
index 34e4848..6b5bcb1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
@@ -566,7 +566,7 @@
 
           <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/>
 
-          <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark"
class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
+          <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark
Shuffle" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig"
/>
         </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task"/>

http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
index f58eb79..a3880b3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
@@ -269,7 +269,7 @@
       <supports-auto-skip-failure>false</supports-auto-skip-failure>
 
       <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn
Properties for Spark">
-        <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark"
class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
+        <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark
Shuffle" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig"
/>
       </execute-stage>
 
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes
for Tez">

http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index 573713c..c15769b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -488,7 +488,7 @@
 
       <component name="RESOURCEMANAGER">
         <pre-upgrade>
-          <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark"
class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
+          <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark
Shuffle" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig"
/>
         </pre-upgrade>
         <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
index 134d9b0..5e8eae6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
@@ -20,26 +20,14 @@
 <configuration supports_final="true">
     <property>
         <name>yarn.nodemanager.aux-services</name>
-        <value>mapreduce_shuffle</value>
+        <value>mapreduce_shuffle,spark_shuffle</value>
         <description>Auxilliary services of NodeManager. A valid service name should
only contain a-zA-Z0-9_ and can
           not start with numbers</description>
-        <depends-on>
-            <property>
-              <type>spark-default</type>
-              <name>spark.yarn.queue</name>
-            </property>
-        </depends-on>
     </property>
 
     <property>
         <name>yarn.nodemanager.aux-services.spark_shuffle.class</name>
         <value>org.apache.spark.network.yarn.YarnShuffleService</value>
         <description>The auxiliary service class to use for Spark</description>
-        <depends-on>
-            <property>
-                <type>spark-default</type>
-                <name>spark.yarn.queue</name>
-            </property>
-        </depends-on>
     </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py
index 3031ab3..985c101 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py
@@ -19,80 +19,4 @@ limitations under the License.
 
 
 class HDP24StackAdvisor(HDP23StackAdvisor):
-  def getServiceConfigurationRecommenderDict(self):
-    parent_recommend_conf_dict = super(HDP24StackAdvisor, self).getServiceConfigurationRecommenderDict()
-    child_recommend_conf_dict = {
-      "YARN": self.recommendYARNConfigurations
-    }
-    parent_recommend_conf_dict.update(child_recommend_conf_dict)
-    return parent_recommend_conf_dict
-
-  def getServiceConfigurationValidators(self):
-    parentValidators = super(HDP24StackAdvisor, self).getServiceConfigurationValidators()
-    childValidators = {
-      "YARN": {"yarn-site": self.validateYARNConfigurations}
-    }
-    self.mergeValidators(parentValidators, childValidators)
-    return parentValidators
-
-  def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
-    super(HDP24StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData,
services, hosts)
-
-    yarn_site_config = "yarn-site"
-    properties = services["configurations"] if yarn_site_config in services["configurations"]
else []
-    yarn_site_properties = properties[yarn_site_config]["properties"] if yarn_site_config
in properties and \
-                                                                         "properties" in
properties[yarn_site_config] else []
-    put_yarn_site_property = self.putProperty(configurations, yarn_site_config, services)
-    put_yarn_site_property_attributes = self.putPropertyAttribute(configurations, yarn_site_config)
-    services_list = [service["StackServices"]["service_name"] for service in services["services"]]
-
-    if 'SPARK' in services_list:
-      if "yarn.nodemanager.aux-services" in yarn_site_properties:
-        aux_services = yarn_site_properties["yarn.nodemanager.aux-services"].split(",")
-        if "spark_shuffle" not in aux_services:
-          aux_services.append("spark_shuffle")
-        put_yarn_site_property("yarn.nodemanager.aux-services", ",".join(aux_services))
-      else:
-        put_yarn_site_property("yarn.nodemanager.aux-services", "spark_shuffle")
-
-      put_yarn_site_property("yarn.nodemanager.aux-services.spark_shuffle.class",
-                             "org.apache.spark.network.yarn.YarnShuffleService")
-    else:
-      put_yarn_site_property_attributes("yarn.nodemanager.aux-services.spark_shuffle.class",
"delete", "true")
-
-  def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services,
hosts):
-    yarn_site = properties
-    validationItems = []
-    services_list = [service["StackServices"]["service_name"] for service in services["services"]]
-
-    if "SPARK" in services_list and 'YARN' in services_list:
-      # yarn.nodemanager.aux-services = ...,spark_shuffle,....
-      # yarn.nodemanager.aux-services.spark_shuffle.class = <not set>
-      if "yarn.nodemanager.aux-services" in yarn_site \
-        and "spark_shuffle" in yarn_site["yarn.nodemanager.aux-services"].lower() \
-        and "yarn.nodemanager.aux-services.spark_shuffle.class" not in yarn_site:
-        validationItems.append({
-          "config-name": "yarn.nodemanager.aux-services.spark_shuffle.class",
-          "item": self.getErrorItem("If spark_shuffle is listed in the aux-services, property
value for " +
-                                    "yarn.nodemanager.aux-services.spark_shuffle.class need
to be set")
-        })
-
-      # yarn.nodemanager.aux-services = <not set>
-      # yarn.nodemanager.aux-services.spark_shuffle.class = is set
-      spark_aux_service_warning = False
-      if "yarn.nodemanager.aux-services" in yarn_site and "spark_shuffle" not in yarn_site[
-        "yarn.nodemanager.aux-services"].lower() \
-        and "yarn.nodemanager.aux-services.spark_shuffle.class" in yarn_site:
-        spark_aux_service_warning = True
-
-      if "yarn.nodemanager.aux-services" not in yarn_site and "yarn.nodemanager.aux-services.spark_shuffle.class"
in yarn_site:
-        spark_aux_service_warning = True
-
-      if spark_aux_service_warning:
-        validationItems.append({
-          "config-name": "yarn.nodemanager.aux-services",
-          "item": self.getWarnItem("If yarn.nodemanager.aux-services.spark_shuffle.class
is set, probably " +
-                                   "aux-services property need to be updated to enable spark_shuffle")
-        })
-
-    return self.toConfigurationValidationProblems(validationItems, "yarn-site")
+ pass

http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
index edba6f0..e65a824 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
@@ -22,7 +22,6 @@ import static org.easymock.EasyMock.anyObject;
 import static org.easymock.EasyMock.expect;
 import static org.easymock.EasyMock.replay;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
@@ -30,7 +29,6 @@ import java.lang.reflect.Field;
 import java.util.HashMap;
 import java.util.Map;
 
-import org.apache.ambari.server.ServiceNotFoundException;
 import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
 import org.apache.ambari.server.actionmanager.HostRoleCommand;
 import org.apache.ambari.server.agent.CommandReport;
@@ -39,7 +37,6 @@ import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Config;
 import org.apache.ambari.server.state.ConfigImpl;
-import org.apache.ambari.server.state.Service;
 import org.easymock.EasyMock;
 import org.junit.Before;
 import org.junit.Test;
@@ -94,20 +91,16 @@ public class SparkShufflePropertyConfigTest {
   }
 
   @Test
-  public void testSparkPresent() throws Exception {
+  public void testAction() throws Exception {
     ExecutionCommand executionCommand = new ExecutionCommand();
     Map<String, String> commandParams = new HashMap<>();
     HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
-    Service sparkService = EasyMock.createMock(Service.class);
     SparkShufflePropertyConfig action = new SparkShufflePropertyConfig();
 
     commandParams.put("clusterName", "c1");
     executionCommand.setCommandParams(commandParams);
     executionCommand.setClusterName("c1");
 
-    // test calculations when spark service present
-    expect(cluster.getService("SPARK")).andReturn(sparkService).atLeastOnce();
-
     expect(hrc.getRequestId()).andReturn(1L).anyTimes();
     expect(hrc.getStageId()).andReturn(2L).anyTimes();
     expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
@@ -131,39 +124,5 @@ public class SparkShufflePropertyConfigTest {
     assertEquals("org.apache.spark.network.yarn.YarnShuffleService", map.get("yarn.nodemanager.aux-services.spark_shuffle.class"));
   }
 
-  @Test
-  public void testSparkNotPresent() throws Exception{
-    SparkShufflePropertyConfig action = new SparkShufflePropertyConfig();
-    Map<String, String> commandParams = new HashMap<>();
-    ExecutionCommand executionCommand = new ExecutionCommand();
-    HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
-    ServiceNotFoundException notFoundException = new ServiceNotFoundException("c1", "SPARK");
 
-    commandParams.put("clusterName", "c1");
-    executionCommand.setCommandParams(commandParams);
-    executionCommand.setClusterName("c1");
-
-    //no spark service present
-    expect(cluster.getService("SPARK")).andThrow(notFoundException).atLeastOnce();
-
-    expect(hrc.getRequestId()).andReturn(1L).anyTimes();
-    expect(hrc.getStageId()).andReturn(2L).anyTimes();
-    expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
-    replay(cluster, hrc);
-
-    clusterField.set(action, m_clusters);
-    action.setExecutionCommand(executionCommand);
-    action.setHostRoleCommand(hrc);
-
-    CommandReport report = action.execute(null);
-    assertNotNull(report);
-
-    Cluster c = m_clusters.getCluster("c1");
-    Config config = c.getDesiredConfigByType("yarn-site");
-    Map<String, String> map = config.getProperties();
-
-    assertTrue(map.containsKey("yarn.nodemanager.aux-services"));
-    assertFalse(map.containsKey("yarn.nodemanager.aux-services.spark_shuffle.class"));
-    assertEquals("some_service", map.get("yarn.nodemanager.aux-services"));
-  }
 }


Mime
View raw message