Return-Path: X-Original-To: apmail-ambari-commits-archive@www.apache.org Delivered-To: apmail-ambari-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E8CCB184BF for ; Tue, 19 Jan 2016 15:03:30 +0000 (UTC) Received: (qmail 46894 invoked by uid 500); 19 Jan 2016 15:03:30 -0000 Delivered-To: apmail-ambari-commits-archive@ambari.apache.org Received: (qmail 46780 invoked by uid 500); 19 Jan 2016 15:03:30 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 46764 invoked by uid 99); 19 Jan 2016 15:03:30 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 19 Jan 2016 15:03:30 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 61B6EDFBAF; Tue, 19 Jan 2016 15:03:30 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aonishuk@apache.org To: commits@ambari.apache.org Date: Tue, 19 Jan 2016 15:03:31 -0000 Message-Id: <5142f958646d42b886e7048e4f3add21@git.apache.org> In-Reply-To: <0df6e9cb35b443089628cdfdd7bc074e@git.apache.org> References: <0df6e9cb35b443089628cdfdd7bc074e@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/2] ambari git commit: AMBARI-14721. yarn-site properties for Spark Shuffle Aux should be added unconditionally (aonishuk) AMBARI-14721. yarn-site properties for Spark Shuffle Aux should be added unconditionally (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ce77df08 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ce77df08 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ce77df08 Branch: refs/heads/branch-2.2 Commit: ce77df08e9da72cffadb7ece2734cfbf31bd067b Parents: bd4892b Author: Andrew Onishuk Authored: Tue Jan 19 17:03:18 2016 +0200 Committer: Andrew Onishuk Committed: Tue Jan 19 17:03:18 2016 +0200 ---------------------------------------------------------------------- .../upgrades/SparkShufflePropertyConfig.java | 14 ++-- .../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml | 2 +- .../stacks/HDP/2.2/upgrades/upgrade-2.4.xml | 2 +- .../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml | 2 +- .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 2 +- .../services/YARN/configuration/yarn-site.xml | 14 +--- .../stacks/HDP/2.4/services/stack_advisor.py | 78 +------------------- .../SparkShufflePropertyConfigTest.java | 43 +---------- 8 files changed, 12 insertions(+), 145 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java index efeb4a0..299a373 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java @@ -68,9 +68,6 @@ public class SparkShufflePropertyConfig extends AbstractServerAction { return createCommandReport(0, HostRoleStatus.FAILED,"{}", String.format("Source type %s not found", YARN_SITE_CONFIG_TYPE), ""); } - - try { - cluster.getService("SPARK"); // check if SPARK service present Map yarnSiteProperties = yarnSiteConfig.getProperties(); final List auxSevices; @@ -82,7 +79,11 @@ public class SparkShufflePropertyConfig extends AbstractServerAction { } else { auxSevices = new ArrayList<>(); } - auxSevices.add(SPARK_SHUFFLE_AUX_STR); + + // check if spark is not already in the list + if (!auxSevices.contains(SPARK_SHUFFLE_AUX_STR)) { + auxSevices.add(SPARK_SHUFFLE_AUX_STR); + } newAuxServices = StringUtils.join(auxSevices, ","); yarnSiteProperties.put(YARN_NODEMANAGER_AUX_SERVICES, newAuxServices); @@ -94,10 +95,5 @@ public class SparkShufflePropertyConfig extends AbstractServerAction { String.format("%s was set from %s to %s. %s was set to %s", YARN_NODEMANAGER_AUX_SERVICES, oldAuxServices, newAuxServices, YARN_NODEMANAGER_AUX_SERVICES_SPARK_SHUFFLE_CLASS_VALUE, YARN_NODEMANAGER_AUX_SERVICES_SPARK_SHUFFLE_CLASS_VALUE), ""); - - } catch (ServiceNotFoundException e) { - return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", - String.format("%s not updated as no SPARK service present on the cluster", YARN_NODEMANAGER_AUX_SERVICES), ""); - } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml index ff7e032..c5d74e2 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml @@ -395,7 +395,7 @@ - + http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml index 34e4848..6b5bcb1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml @@ -566,7 +566,7 @@ - + http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml index f58eb79..a3880b3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml @@ -269,7 +269,7 @@ false - + http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml index 573713c..c15769b 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml @@ -488,7 +488,7 @@ - + http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml index 134d9b0..5e8eae6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml @@ -20,26 +20,14 @@ yarn.nodemanager.aux-services - mapreduce_shuffle + mapreduce_shuffle,spark_shuffle Auxilliary services of NodeManager. A valid service name should only contain a-zA-Z0-9_ and can not start with numbers - - - spark-default - spark.yarn.queue - - yarn.nodemanager.aux-services.spark_shuffle.class org.apache.spark.network.yarn.YarnShuffleService The auxiliary service class to use for Spark - - - spark-default - spark.yarn.queue - - http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py index 3031ab3..985c101 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py @@ -19,80 +19,4 @@ limitations under the License. class HDP24StackAdvisor(HDP23StackAdvisor): - def getServiceConfigurationRecommenderDict(self): - parent_recommend_conf_dict = super(HDP24StackAdvisor, self).getServiceConfigurationRecommenderDict() - child_recommend_conf_dict = { - "YARN": self.recommendYARNConfigurations - } - parent_recommend_conf_dict.update(child_recommend_conf_dict) - return parent_recommend_conf_dict - - def getServiceConfigurationValidators(self): - parentValidators = super(HDP24StackAdvisor, self).getServiceConfigurationValidators() - childValidators = { - "YARN": {"yarn-site": self.validateYARNConfigurations} - } - self.mergeValidators(parentValidators, childValidators) - return parentValidators - - def recommendYARNConfigurations(self, configurations, clusterData, services, hosts): - super(HDP24StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData, services, hosts) - - yarn_site_config = "yarn-site" - properties = services["configurations"] if yarn_site_config in services["configurations"] else [] - yarn_site_properties = properties[yarn_site_config]["properties"] if yarn_site_config in properties and \ - "properties" in properties[yarn_site_config] else [] - put_yarn_site_property = self.putProperty(configurations, yarn_site_config, services) - put_yarn_site_property_attributes = self.putPropertyAttribute(configurations, yarn_site_config) - services_list = [service["StackServices"]["service_name"] for service in services["services"]] - - if 'SPARK' in services_list: - if "yarn.nodemanager.aux-services" in yarn_site_properties: - aux_services = yarn_site_properties["yarn.nodemanager.aux-services"].split(",") - if "spark_shuffle" not in aux_services: - aux_services.append("spark_shuffle") - put_yarn_site_property("yarn.nodemanager.aux-services", ",".join(aux_services)) - else: - put_yarn_site_property("yarn.nodemanager.aux-services", "spark_shuffle") - - put_yarn_site_property("yarn.nodemanager.aux-services.spark_shuffle.class", - "org.apache.spark.network.yarn.YarnShuffleService") - else: - put_yarn_site_property_attributes("yarn.nodemanager.aux-services.spark_shuffle.class", "delete", "true") - - def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts): - yarn_site = properties - validationItems = [] - services_list = [service["StackServices"]["service_name"] for service in services["services"]] - - if "SPARK" in services_list and 'YARN' in services_list: - # yarn.nodemanager.aux-services = ...,spark_shuffle,.... - # yarn.nodemanager.aux-services.spark_shuffle.class = - if "yarn.nodemanager.aux-services" in yarn_site \ - and "spark_shuffle" in yarn_site["yarn.nodemanager.aux-services"].lower() \ - and "yarn.nodemanager.aux-services.spark_shuffle.class" not in yarn_site: - validationItems.append({ - "config-name": "yarn.nodemanager.aux-services.spark_shuffle.class", - "item": self.getErrorItem("If spark_shuffle is listed in the aux-services, property value for " + - "yarn.nodemanager.aux-services.spark_shuffle.class need to be set") - }) - - # yarn.nodemanager.aux-services = - # yarn.nodemanager.aux-services.spark_shuffle.class = is set - spark_aux_service_warning = False - if "yarn.nodemanager.aux-services" in yarn_site and "spark_shuffle" not in yarn_site[ - "yarn.nodemanager.aux-services"].lower() \ - and "yarn.nodemanager.aux-services.spark_shuffle.class" in yarn_site: - spark_aux_service_warning = True - - if "yarn.nodemanager.aux-services" not in yarn_site and "yarn.nodemanager.aux-services.spark_shuffle.class" in yarn_site: - spark_aux_service_warning = True - - if spark_aux_service_warning: - validationItems.append({ - "config-name": "yarn.nodemanager.aux-services", - "item": self.getWarnItem("If yarn.nodemanager.aux-services.spark_shuffle.class is set, probably " + - "aux-services property need to be updated to enable spark_shuffle") - }) - - return self.toConfigurationValidationProblems(validationItems, "yarn-site") + pass http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java index edba6f0..e65a824 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java @@ -22,7 +22,6 @@ import static org.easymock.EasyMock.anyObject; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.replay; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -30,7 +29,6 @@ import java.lang.reflect.Field; import java.util.HashMap; import java.util.Map; -import org.apache.ambari.server.ServiceNotFoundException; import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.agent.CommandReport; @@ -39,7 +37,6 @@ import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; import org.apache.ambari.server.state.ConfigImpl; -import org.apache.ambari.server.state.Service; import org.easymock.EasyMock; import org.junit.Before; import org.junit.Test; @@ -94,20 +91,16 @@ public class SparkShufflePropertyConfigTest { } @Test - public void testSparkPresent() throws Exception { + public void testAction() throws Exception { ExecutionCommand executionCommand = new ExecutionCommand(); Map commandParams = new HashMap<>(); HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class); - Service sparkService = EasyMock.createMock(Service.class); SparkShufflePropertyConfig action = new SparkShufflePropertyConfig(); commandParams.put("clusterName", "c1"); executionCommand.setCommandParams(commandParams); executionCommand.setClusterName("c1"); - // test calculations when spark service present - expect(cluster.getService("SPARK")).andReturn(sparkService).atLeastOnce(); - expect(hrc.getRequestId()).andReturn(1L).anyTimes(); expect(hrc.getStageId()).andReturn(2L).anyTimes(); expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes(); @@ -131,39 +124,5 @@ public class SparkShufflePropertyConfigTest { assertEquals("org.apache.spark.network.yarn.YarnShuffleService", map.get("yarn.nodemanager.aux-services.spark_shuffle.class")); } - @Test - public void testSparkNotPresent() throws Exception{ - SparkShufflePropertyConfig action = new SparkShufflePropertyConfig(); - Map commandParams = new HashMap<>(); - ExecutionCommand executionCommand = new ExecutionCommand(); - HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class); - ServiceNotFoundException notFoundException = new ServiceNotFoundException("c1", "SPARK"); - commandParams.put("clusterName", "c1"); - executionCommand.setCommandParams(commandParams); - executionCommand.setClusterName("c1"); - - //no spark service present - expect(cluster.getService("SPARK")).andThrow(notFoundException).atLeastOnce(); - - expect(hrc.getRequestId()).andReturn(1L).anyTimes(); - expect(hrc.getStageId()).andReturn(2L).anyTimes(); - expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes(); - replay(cluster, hrc); - - clusterField.set(action, m_clusters); - action.setExecutionCommand(executionCommand); - action.setHostRoleCommand(hrc); - - CommandReport report = action.execute(null); - assertNotNull(report); - - Cluster c = m_clusters.getCluster("c1"); - Config config = c.getDesiredConfigByType("yarn-site"); - Map map = config.getProperties(); - - assertTrue(map.containsKey("yarn.nodemanager.aux-services")); - assertFalse(map.containsKey("yarn.nodemanager.aux-services.spark_shuffle.class")); - assertEquals("some_service", map.get("yarn.nodemanager.aux-services")); - } }