ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alejan...@apache.org
Subject ambari git commit: AMBARI-16857. Support yarn aux service classpath isolation feature in spark (Bikas Saha via alejandro)
Date Tue, 31 May 2016 21:46:16 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.4 03d5f14ea -> 9b24744f9


AMBARI-16857. Support yarn aux service classpath isolation feature in spark (Bikas Saha via
alejandro)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9b24744f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9b24744f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9b24744f

Branch: refs/heads/branch-2.4
Commit: 9b24744f90d8fb6f1d0dc3474d1da8255849e1f6
Parents: 03d5f14
Author: Alejandro Fernandez <afernandez@hortonworks.com>
Authored: Tue May 31 14:47:14 2016 -0700
Committer: Alejandro Fernandez <afernandez@hortonworks.com>
Committed: Tue May 31 14:47:14 2016 -0700

----------------------------------------------------------------------
 .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml |  4 ++
 .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml     |  3 ++
 .../stacks/HDP/2.4/upgrades/config-upgrade.xml  | 10 +++++
 .../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml |  5 +++
 .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml     |  3 ++
 .../services/YARN/configuration/yarn-site.xml   | 42 ++++++++++++++++++++
 6 files changed, 67 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/9b24744f/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
index 4362fe0..c7d8b30 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
@@ -291,6 +291,10 @@
         <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"/>
       </execute-stage>
 
+      <execute-stage service="YARN" component="NODEMANAGER" title="Add Spark2 shuffle">
+        <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
+      </execute-stage>
+
       <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config
changes for Mapreduce2 client">
         <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce"
class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
       </execute-stage>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9b24744f/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index 0e0ea1d..e39f413 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -590,6 +590,9 @@
       </component>
 
       <component name="NODEMANAGER">
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
+        </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9b24744f/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
index b68354f..39fb9a6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
@@ -173,6 +173,16 @@
           </definition>
         </changes>
       </component>
+
+      <component name="NODEMANAGER">
+        <changes>
+          <definition xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle">
+            <type>yarn-site</type>
+            <set key="yarn.nodemanager.aux-services" value="mapreduce_shuffle,spark_shuffle,spark2_shuffle"/>
+            <!-- Ideally we need to append spark2_shuffle to the existing value -->
+          </definition>
+        </changes>
+      </component>
     </service>
 
     <service name="KAFKA">

http://git-wip-us.apache.org/repos/asf/ambari/blob/9b24744f/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
index 5ccb153..47e849c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
@@ -277,6 +277,11 @@
         <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"/>
       </execute-stage>
 
+      <!-- YARN -->
+      <execute-stage service="YARN" component="NODEMANAGER" title="Add Spark2 shuffle">
+        <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
+      </execute-stage>
+
       <!--TEZ-->
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path
for Tez">
         <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9b24744f/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
index bc45be6..644ca87 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
@@ -584,6 +584,9 @@
       </component>
 
       <component name="NODEMANAGER">
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
+        </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9b24744f/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
new file mode 100644
index 0000000..cd992bc
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<configuration supports_final="true">
+  <property>
+    <name>yarn.nodemanager.aux-services</name>
+    <value>mapreduce_shuffle,spark_shuffle,spark2_shuffle</value>
+    <description>Auxilliary services of NodeManager. A valid service name should only
contain a-zA-Z0-9_ and cannot start with numbers</description>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.aux-services.spark2_shuffle.class</name>
+    <value>org.apache.spark.network.yarn.YarnShuffleService</value>
+    <description>The auxiliary service class to use for Spark 2</description>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name>
+    <value>/usr/hdp/${hdp.version}/spark/aux/*</value>
+  </property>
+
+  <property>
+    <name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name>
+    <value>/usr/hdp/${hdp.version}/spark2/aux/*</value>
+  </property>
+</configuration>
\ No newline at end of file


Mime
View raw message