ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject ambari git commit: AMBARI-16857. Support yarn aux service classpath isolation feature in spark (Bikas Saha via smohanty)
Date Tue, 31 May 2016 01:06:16 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.4 7b427d66c -> 6411a0db6


AMBARI-16857. Support yarn aux service classpath isolation feature in spark (Bikas Saha via
smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6411a0db
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6411a0db
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6411a0db

Branch: refs/heads/branch-2.4
Commit: 6411a0db6333140751c8825528d410f0b4d3e500
Parents: 7b427d6
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Mon May 30 17:45:02 2016 -0700
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Mon May 30 18:05:53 2016 -0700

----------------------------------------------------------------------
 .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml |  4 ++
 .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml     |  3 ++
 .../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml |  5 +++
 .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml     |  3 ++
 .../services/YARN/configuration/yarn-site.xml   | 43 ++++++++++++++++++++
 .../stacks/HDP/2.5/upgrades/config-upgrade.xml  | 11 +++++
 6 files changed, 69 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/6411a0db/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
index 4362fe0..c7d8b30 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
@@ -291,6 +291,10 @@
         <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"/>
       </execute-stage>
 
+      <execute-stage service="YARN" component="NODEMANAGER" title="Add Spark2 shuffle">
+        <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
+      </execute-stage>
+
       <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config
changes for Mapreduce2 client">
         <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce"
class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>
       </execute-stage>

http://git-wip-us.apache.org/repos/asf/ambari/blob/6411a0db/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index 0e0ea1d..e39f413 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -590,6 +590,9 @@
       </component>
 
       <component name="NODEMANAGER">
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
+        </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/6411a0db/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
index 5ccb153..47e849c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
@@ -277,6 +277,11 @@
         <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"/>
       </execute-stage>
 
+      <!-- YARN -->
+      <execute-stage service="YARN" component="NODEMANAGER" title="Add Spark2 shuffle">
+        <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
+      </execute-stage>
+
       <!--TEZ-->
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path
for Tez">
         <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/>

http://git-wip-us.apache.org/repos/asf/ambari/blob/6411a0db/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
index bc45be6..644ca87 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
@@ -584,6 +584,9 @@
       </component>
 
       <component name="NODEMANAGER">
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
+        </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/6411a0db/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
new file mode 100644
index 0000000..0f0e44f
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+<configuration supports_final="true">
+    <property>
+        <name>yarn.nodemanager.aux-services</name>
+        <value>mapreduce_shuffle,spark_shuffle,spark2_shuffle</value>
+        <description>Auxilliary services of NodeManager. A valid service name should
only contain a-zA-Z0-9_ and can
+          not start with numbers</description>
+    </property>
+
+    <property>
+        <name>yarn.nodemanager.aux-services.spark2_shuffle.class</name>
+        <value>org.apache.spark.network.yarn.YarnShuffleService</value>
+        <description>The auxiliary service class to use for Spark 2</description>
+    </property>
+
+    <property>
+      <name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name>
+      <value>/usr/hdp/${hdp.version}/spark/aux/*</value>
+    </property>
+
+    <property>
+      <name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name>
+      <value>/usr/hdp/${hdp.version}/spark2/aux/*</value>
+    </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/6411a0db/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index 5211276..aaa2d21 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -50,6 +50,17 @@
         </changes>
       </component>
     </service>
+    <service name="YARN">
+      <component name="NODEMANAGER">
+        <changes>
+          <definition xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle">
+            <type>yarn-site</type>
+            <set key="yarn.nodemanager.aux-services" value="mapreduce_shuffle,spark_shuffle,spark2_shuffle"/>
+            <!-- Ideally we need to append spark2_shuffle to the existing value -->
+          </definition>
+        </changes>
+      </component>
+    </service>
   </services>
 
 </upgrade-config-changes>


Mime
View raw message