ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jai...@apache.org
Subject [34/50] [abbrv] ambari git commit: AMBARI-21770 - Spark1 Shuffle Property Is Removed Incorrectly on a Stack Upgrade (jonathanhurley)
Date Mon, 09 Oct 2017 18:34:48 GMT
AMBARI-21770 - Spark1 Shuffle Property Is Removed Incorrectly on a Stack Upgrade (jonathanhurley)

(cherry picked from commit 24debeed14654249055077ad5483dc45d127a074)

Change-Id: Ic475ed2895a14740b576aeaf801f320320017bfd


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a0b3f4fc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a0b3f4fc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a0b3f4fc

Branch: refs/heads/2.5-maint
Commit: a0b3f4fc738aa28e2ac39a175da4e53e31208b3f
Parents: 2054332
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Mon Aug 21 16:09:57 2017 -0400
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Mon Aug 21 19:19:29 2017 -0700

----------------------------------------------------------------------
 .../stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml      | 7 ++++++-
 .../stacks/BigInsights/4.2/upgrades/config-upgrade.xml        | 6 +++++-
 2 files changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/a0b3f4fc/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
index f55f9fb..f90a20e 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
@@ -76,7 +76,12 @@
         <changes>
           <definition xsi:type="configure" id="biginsights_4_2_yarn_config_update" summary="Update
Yarn configurations">
             <type>yarn-site</type>
-            <replace key="yarn.nodemanager.aux-services" find=",spark_shuffle" replace-with=""/>
+            <!-- adjust the spark shuffle values for spark1 and spark2 -->
+            <set key="yarn.nodemanager.aux-services" value="mapreduce_shuffle,spark_shuffle,spark2_shuffle"/>
+            <set key="yarn.nodemanager.aux-services.spark_shuffle.class" value="org.apache.spark.network.yarn.YarnShuffleService"/>
+            <set key="yarn.nodemanager.aux-services.spark_shuffle.classpath" value="{{stack_root}}/${hdp.version}/spark/aux/*"/>
+            <set key="yarn.nodemanager.aux-services.spark2_shuffle.class" value="org.apache.spark.network.yarn.YarnShuffleService"/>
+            <set key="yarn.nodemanager.aux-services.spark2_shuffle.classpath" value="{{stack_root}}/${hdp.version}/spark2/aux/*"/>
           </definition>
           <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding
YARN Security ACLs">
             <type>yarn-env</type>

http://git-wip-us.apache.org/repos/asf/ambari/blob/a0b3f4fc/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
index 02d3a01..d85c7ee 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
@@ -164,7 +164,11 @@
         <changes>
           <definition xsi:type="configure" id="biginsights_4_2_yarn_config_update" summary="Update
Yarn configurations">
             <type>yarn-site</type>
-            <replace key="yarn.nodemanager.aux-services" find=",spark_shuffle" replace-with=""/>
+            <!-- adjust the spark shuffle values for spark1 based on whether spark is
installed -->
+            <set key="yarn.nodemanager.aux-services" value="mapreduce_shuffle,spark_shuffle"
if-type="spark-env" if-key="content" if-key-state="present"/>
+            <set key="yarn.nodemanager.aux-services.spark_shuffle.class" value="org.apache.spark.network.yarn.YarnShuffleService"
if-type="spark-env" if-key="content" if-key-state="present"/>
+            <set key="yarn.nodemanager.aux-services.spark_shuffle.classpath" value="{{stack_root}}/${hdp.version}/spark/aux/*"
if-type="spark-env" if-key="content" if-key-state="present"/>
+
             <insert key="yarn.application.classpath" insert-type="append" newline-after="false"
newline-before="false" value=",/usr/hdp/current/ext/hadoop/*"/>
           </definition>
           <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding
YARN Security ACLs">


Mime
View raw message