ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject ambari git commit: AMBARI-17489. Remove spark.yarn.max.executor.failures configuration in Spark Ambari definition (Saisai Shao via smohanty)
Date Thu, 20 Oct 2016 04:54:09 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.4 47affcdba -> 45ccfaf01


AMBARI-17489. Remove spark.yarn.max.executor.failures configuration in Spark Ambari definition
(Saisai Shao via smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/45ccfaf0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/45ccfaf0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/45ccfaf0

Branch: refs/heads/branch-2.4
Commit: 45ccfaf01a1607a9813b79d39fb7c02a882ab6f3
Parents: 47affcd
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Wed Oct 19 21:54:05 2016 -0700
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Wed Oct 19 21:54:05 2016 -0700

----------------------------------------------------------------------
 .../1.6.0/configuration/spark-defaults.xml      |  5 +++++
 .../stacks/HDP/2.3/upgrades/config-upgrade.xml  |  2 ++
 .../stacks/HDP/2.4/upgrades/config-upgrade.xml  | 22 +++++++++++++++++++-
 .../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 20 +++++++++++++-----
 .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml     | 10 +++++++--
 5 files changed, 51 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/45ccfaf0/ambari-server/src/main/resources/common-services/SPARK/1.6.0/configuration/spark-defaults.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.6.0/configuration/spark-defaults.xml
b/ambari-server/src/main/resources/common-services/SPARK/1.6.0/configuration/spark-defaults.xml
index 6465b72..0bcf10f 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.6.0/configuration/spark-defaults.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.6.0/configuration/spark-defaults.xml
@@ -25,6 +25,11 @@
     <on-ambari-upgrade add="false"/>
   </property>
   <property>
+    <name>spark.yarn.max.executor.failures</name>
+    <deleted>true</deleted>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
     <name>spark.history.provider</name>
     <value>org.apache.spark.deploy.history.FsHistoryProvider</value>
     <description>Name of history provider class</description>

http://git-wip-us.apache.org/repos/asf/ambari/blob/45ccfaf0/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
index cdbe319..b63115b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
@@ -370,6 +370,7 @@
             <transfer operation="delete" delete-key="spark.yarn.services" />
             <transfer operation="delete" delete-key="spark.driver.extraJavaOptions" />
             <transfer operation="delete" delete-key="spark.yarn.am.extraJavaOptions" />
+            <transfer operation="delete" delete-key="spark.yarn.max.executor.failures"
/>
             <set key="spark.history.provider" value="org.apache.spark.deploy.history.FsHistoryProvider"/>
           </definition>
           <definition xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts">
@@ -384,6 +385,7 @@
             <type>spark-defaults</type>
             <transfer operation="delete" delete-key="spark.driver.extraJavaOptions" />
             <transfer operation="delete" delete-key="spark.yarn.am.extraJavaOptions" />
+            <transfer operation="delete" delete-key="spark.yarn.max.executor.failures"
/>
           </definition>
           <definition xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts">
             <type>spark-javaopts-properties</type>

http://git-wip-us.apache.org/repos/asf/ambari/blob/45ccfaf0/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
index 9045443..8a9996c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
@@ -282,7 +282,7 @@
             <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir"
/>
           </definition>
 
-          
+
           <definition xsi:type="configure" id="hdp_2_5_0_0_upgrade_storm_1.0">
             <type>storm-site</type>
             <replace key="_storm.thrift.nonsecure.transport" find="backtype.storm.security.auth.SimpleTransportPlugin"
@@ -381,6 +381,26 @@
         </changes>
       </component>
     </service>
+
+    <service name="SPARK">
+      <component name="SPARK_JOBHISTORYSERVER">
+        <changes>
+          <definition xsi:type="configure" id="hdp_2_5_0_0_spark_jobhistoryserver">
+            <type>spark-defaults</type>
+            <transfer operation="delete" delete-key="spark.yarn.max.executor.failures"
/>
+          </definition>
+        </changes>
+      </component>
+      <component name="SPARK_CLIENT">
+        <changes>
+          <definition xsi:type="configure" id="hdp_2_5_0_0_spark_client">
+            <type>spark-defaults</type>
+            <transfer operation="delete" delete-key="spark.yarn.max.executor.failures"
/>
+          </definition>
+        </changes>
+      </component>
+    </service>
+
   </services>
 
 </upgrade-config-changes>

http://git-wip-us.apache.org/repos/asf/ambari/blob/45ccfaf0/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
index 3478603..2924455 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml
@@ -32,6 +32,7 @@
 
     <check>org.apache.ambari.server.checks.AtlasPresenceCheck</check>
   </prerequisite-checks>
+
   <order>
     <group xsi:type="cluster" name="PRE_CLUSTER" title="Prepare Upgrade">
       <direction>UPGRADE</direction>
@@ -86,7 +87,7 @@
         <component>ACCUMULO_MONITOR</component>
         <component>ACCUMULO_MASTER</component>
       </service>
-      
+
       <service name="STORM">
         <component>DRPC_SERVER</component>
         <component>STORM_UI_SERVER</component>
@@ -402,6 +403,15 @@
       <execute-stage service="RANGER_KMS" component="RANGER_KMS_SERVER" title="Apply config
changes for Ranger KMS Server">
         <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_kms_audit_db"/>
       </execute-stage>
+
+      <!-- SPARK -->
+      <execute-stage service="SPARK" component="SPARK_JOBHISTORYSERVER" title="Apply config
changes for Spark JobHistoryServer">
+        <task xsi:type="configure" id="hdp_2_5_0_0_spark_jobhistoryserver"/>
+      </execute-stage>
+      <execute-stage service="SPARK" component="SPARK_CLIENT" title="Apply config changes
for Spark">
+        <task xsi:type="configure" id="hdp_2_5_0_0_spark_client"/>
+      </execute-stage>
+
     </group>
 
     <!--
@@ -588,7 +598,7 @@
         <service>HBASE</service>
       </priority>
     </group>
-    
+
     <group xsi:type="restart" name="HIVE_MASTERS" title="Hive Masters">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -663,7 +673,7 @@
         <service>SPARK</service>
       </priority>
     </group>
-    
+
     <group xsi:type="restart" name="FALCON" title="Falcon">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -727,7 +737,7 @@
         <component>FLUME_HANDLER</component>
       </service>
     </group>
-    
+
     <group xsi:type="restart" name="ACCUMULO" title="Accumulo">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -757,7 +767,7 @@
 
     <group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}}
Pre-Check">
       <direction>UPGRADE</direction>
-      
+
       <execute-stage title="Check Component Versions">
         <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction"
/>
       </execute-stage>

http://git-wip-us.apache.org/repos/asf/ambari/blob/45ccfaf0/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
index 826cf16..e4f56e9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml
@@ -503,7 +503,7 @@
           <message>-- If keytabs are managed manually, create the required ranger keytabs
in the locations specified in ranger config and restart Ranger components.</message>
         </task>
       </execute-stage>
-      
+
       <execute-stage service="KAFKA" component="KAFKA_BROKER" title="Kafka Property Removal">
         <task xsi:type="manual">
           <message>Kafka Brokers have been upgraded with version compatability.</message>
@@ -653,7 +653,7 @@
           <task xsi:type="restart-task" />
         </upgrade>
       </component>
-      
+
       <component name="NFS_GATEWAY">
         <upgrade>
           <task xsi:type="restart-task" />
@@ -876,6 +876,9 @@
 
     <service name="SPARK">
       <component name="SPARK_JOBHISTORYSERVER">
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_5_0_0_spark_jobhistoryserver"/>
+        </pre-upgrade>
         <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
         <upgrade>
           <task xsi:type="restart-task" />
@@ -888,6 +891,9 @@
         </upgrade>
       </component>
       <component name="SPARK_CLIENT">
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_5_0_0_spark_client"/>
+        </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>


Mime
View raw message