ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject ambari git commit: AMBARI-21770 - Spark1 Shuffle Property Is Removed Incorrectly on a Stack Upgrade (jonathanhurley)
Date Mon, 21 Aug 2017 22:55:39 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.6 4f52f6d40 -> d84cda89a


AMBARI-21770 - Spark1 Shuffle Property Is Removed Incorrectly on a Stack Upgrade (jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d84cda89
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d84cda89
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d84cda89

Branch: refs/heads/branch-2.6
Commit: d84cda89a940fa61701972d9ff011010dc0be08f
Parents: 4f52f6d
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Mon Aug 21 16:09:57 2017 -0400
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Mon Aug 21 18:55:31 2017 -0400

----------------------------------------------------------------------
 .../stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml      | 7 ++++++-
 .../stacks/BigInsights/4.2/upgrades/config-upgrade.xml        | 6 +++++-
 2 files changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/d84cda89/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
index f55f9fb..f90a20e 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
@@ -76,7 +76,12 @@
         <changes>
           <definition xsi:type="configure" id="biginsights_4_2_yarn_config_update" summary="Update
Yarn configurations">
             <type>yarn-site</type>
-            <replace key="yarn.nodemanager.aux-services" find=",spark_shuffle" replace-with=""/>
+            <!-- adjust the spark shuffle values for spark1 and spark2 -->
+            <set key="yarn.nodemanager.aux-services" value="mapreduce_shuffle,spark_shuffle,spark2_shuffle"/>
+            <set key="yarn.nodemanager.aux-services.spark_shuffle.class" value="org.apache.spark.network.yarn.YarnShuffleService"/>
+            <set key="yarn.nodemanager.aux-services.spark_shuffle.classpath" value="{{stack_root}}/${hdp.version}/spark/aux/*"/>
+            <set key="yarn.nodemanager.aux-services.spark2_shuffle.class" value="org.apache.spark.network.yarn.YarnShuffleService"/>
+            <set key="yarn.nodemanager.aux-services.spark2_shuffle.classpath" value="{{stack_root}}/${hdp.version}/spark2/aux/*"/>
           </definition>
           <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding
YARN Security ACLs">
             <type>yarn-env</type>

http://git-wip-us.apache.org/repos/asf/ambari/blob/d84cda89/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
index ad20bf9..77390cf 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
@@ -164,7 +164,11 @@
         <changes>
           <definition xsi:type="configure" id="biginsights_4_2_yarn_config_update" summary="Update
Yarn configurations">
             <type>yarn-site</type>
-            <replace key="yarn.nodemanager.aux-services" find=",spark_shuffle" replace-with=""/>
+            <!-- adjust the spark shuffle values for spark1 based on whether spark is
installed -->
+            <set key="yarn.nodemanager.aux-services" value="mapreduce_shuffle,spark_shuffle"
if-type="spark-env" if-key="content" if-key-state="present"/>
+            <set key="yarn.nodemanager.aux-services.spark_shuffle.class" value="org.apache.spark.network.yarn.YarnShuffleService"
if-type="spark-env" if-key="content" if-key-state="present"/>
+            <set key="yarn.nodemanager.aux-services.spark_shuffle.classpath" value="{{stack_root}}/${hdp.version}/spark/aux/*"
if-type="spark-env" if-key="content" if-key-state="present"/>
+
             <insert key="yarn.application.classpath" insert-type="append" newline-after="false"
newline-before="false" value=",/usr/hdp/current/ext/hadoop/*"/>
           </definition>
           <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding
YARN Security ACLs">


Mime
View raw message