ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject ambari git commit: AMBARI-21226. Add two new Spark thrift server related configurations in Spark stack (Saisai Shao via smohanty)
Date Wed, 14 Jun 2017 02:57:55 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.5 558d9e3da -> 2461ddbcd


AMBARI-21226. Add two new Spark thrift server related configurations in Spark stack (Saisai
Shao via smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2461ddbc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2461ddbc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2461ddbc

Branch: refs/heads/branch-2.5
Commit: 2461ddbcd1c4fa00b628395b60625648a9074cd9
Parents: 558d9e3
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Tue Jun 13 19:57:06 2017 -0700
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Tue Jun 13 19:57:48 2017 -0700

----------------------------------------------------------------------
 .../configuration/spark-thrift-sparkconf.xml    | 40 ++++++++++++++++++++
 .../configuration/spark2-thrift-sparkconf.xml   | 40 ++++++++++++++++++++
 .../stacks/HDP/2.6/upgrades/config-upgrade.xml  | 24 ++++++++++++
 .../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml | 26 +++++++++----
 .../stacks/HDP/2.6/upgrades/upgrade-2.6.xml     | 32 ++++++++++------
 5 files changed, 142 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/2461ddbc/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/spark-thrift-sparkconf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/spark-thrift-sparkconf.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/spark-thrift-sparkconf.xml
new file mode 100644
index 0000000..001ef1e
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK/configuration/spark-thrift-sparkconf.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+  <property>
+    <name>spark.yarn.executor.failuresValidityInterval</name>
+    <value>2h</value>
+    <description>
+      Defines the validity interval for executor failure tracking.
+      Executor failures which are older than the validity interval will be ignored.
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>spark.yarn.maxAppAttempts</name>
+    <value>1</value>
+    <description>
+      The maximum number of attempts that will be made to submit the application.
+      It should be no larger than the global number of max attempts in the YARN configuration.
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/2461ddbc/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-thrift-sparkconf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-thrift-sparkconf.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-thrift-sparkconf.xml
new file mode 100644
index 0000000..001ef1e
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/SPARK2/configuration/spark2-thrift-sparkconf.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+  <property>
+    <name>spark.yarn.executor.failuresValidityInterval</name>
+    <value>2h</value>
+    <description>
+      Defines the validity interval for executor failure tracking.
+      Executor failures which are older than the validity interval will be ignored.
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>spark.yarn.maxAppAttempts</name>
+    <value>1</value>
+    <description>
+      The maximum number of attempts that will be made to submit the application.
+      It should be no larger than the global number of max attempts in the YARN configuration.
+    </description>
+    <on-ambari-upgrade add="false"/>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/2461ddbc/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
index 1610bb5..564dead 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
@@ -168,5 +168,29 @@
       </component>
     </service>
 
+    <service name="SPARK">
+      <component name="SPARK_THRIFTSERVER">
+        <changes>
+          <definition xsi:type="configure" id="hdp_2_6_0_0_spark_thriftserver">
+            <type>spark-thrift-sparkconf</type>
+            <set key="spark.yarn.executor.failuresValidityInterval" value="2h"/>
+            <set key="spark.yarn.maxAppAttempts" value="1"/>
+          </definition>
+        </changes>
+      </component>
+    </service>
+
+    <service name="SPARK2">
+      <component name="SPARK2_THRIFTSERVER">
+        <changes>
+          <definition xsi:type="configure" id="hdp_2_6_0_0_spark2_thriftserver">
+            <type>spark2-thrift-sparkconf</type>
+            <set key="spark.yarn.executor.failuresValidityInterval" value="2h"/>
+            <set key="spark.yarn.maxAppAttempts" value="1"/>
+          </definition>
+        </changes>
+      </component>
+    </service>
+
   </services>
 </upgrade-config-changes>

http://git-wip-us.apache.org/repos/asf/ambari/blob/2461ddbc/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
index 1cdd184..6b827c9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
@@ -75,7 +75,7 @@
       <service name="FLUME">
         <component>FLUME_HANDLER</component>
       </service>
-      
+
       <service name="ACCUMULO">
         <component>ACCUMULO_TRACER</component>
         <component>ACCUMULO_GC</component>
@@ -284,7 +284,7 @@
       </execute-stage>
     </group>
 
-    
+
     <group xsi:type="cluster" name="Upgrade service configs" title="Upgrade service configs">
       <direction>UPGRADE</direction> <!--  prevent config changes on downgrade
-->
       <skippable>true</skippable> <!-- May fix configuration problems manually
-->
@@ -322,7 +322,7 @@
       <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes
for Ranger Yarn plugin">
         <task xsi:type="configure" id="hdp_2_6_maint_ranger_yarn_plugin_cluster_name"/>
       </execute-stage>
-      
+
       <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes
for YARN Preemption">
         <task xsi:type="configure" id="hdp_2_6_yarn_preemption"/>
       </execute-stage>
@@ -353,8 +353,18 @@
         <task xsi:type="configure" id="hdp_2_6_maint_ranger_kms_plugin_cluster_name"/>
       </execute-stage>
 
+      <!--SPARK-->
+      <execute-stage service="SPARK" component="SPARK_THRIFTSERVER" title="Apply config
changes for Spark ThriftServer">
+        <task xsi:type="configure" id="hdp_2_6_0_0_spark_thriftserver"/>
+      </execute-stage>
+
+      <!--SPARK2-->
+      <execute-stage service="SPARK2" component="SPARK2_THRIFTSERVER" title="Apply config
changes for Spark2 ThriftServer">
+        <task xsi:type="configure" id="hdp_2_6_0_0_spark2_thriftserver"/>
+      </execute-stage>
+
     </group>
-    
+
 
     <!--
     After processing this group, the user-specified Kerberos descriptor will be updated to
work with
@@ -383,7 +393,7 @@
         </task>
       </execute-stage>
     </group>
-    
+
     <!-- Now, restart all of the services. -->
     <group xsi:type="restart" name="ZOOKEEPER" title="ZooKeeper">
       <service-check>false</service-check>
@@ -747,7 +757,7 @@
         <component>FLUME_HANDLER</component>
       </service>
     </group>
-    
+
     <group xsi:type="restart" name="ACCUMULO" title="Accumulo">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -790,7 +800,7 @@
 
     <group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}}
Pre-Check">
       <direction>UPGRADE</direction>
-      
+
       <execute-stage title="Check Component Versions">
         <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction"
/>
       </execute-stage>
@@ -870,7 +880,7 @@
             <function>setup_ranger_java_patches</function>
           </task>
         </pre-upgrade>
-        
+
         <pre-downgrade copy-upgrade="true" />
 
         <upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/2461ddbc/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
index 3e7e3d7..9227669 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
@@ -168,7 +168,7 @@
         <component>HBASE_MASTER</component>
       </service>
     </group>
-    
+
     <group name="SERVICE_CHECK_1" title="All Service Checks" xsi:type="service-check">
       <direction>UPGRADE</direction>
       <skippable>true</skippable>
@@ -184,7 +184,7 @@
         <service>LOGSEARCH</service>
       </exclude>
     </group>
-    
+
     <group name="CORE_SLAVES" title="Core Slaves" xsi:type="colocated">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -192,7 +192,7 @@
         <component>DATANODE</component>
         <component>NFS_GATEWAY</component>
       </service>
-      
+
       <service name="HBASE">
         <component>HBASE_REGIONSERVER</component>
         <component>PHOENIX_QUERY_SERVER</component>
@@ -208,7 +208,7 @@
         <message>The initial batch of {{components}} hosts have been {{direction.past}}.
You are advised to check the hosts and perform cluster/workload-specific tests against your
cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining
services.</message>
       </batch>
     </group>
-    
+
     <group name="SERVICE_CHECK_2" title="All Service Checks" xsi:type="service-check">
       <direction>UPGRADE</direction>
       <skippable>true</skippable>
@@ -460,7 +460,7 @@
 
     <group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}}
Pre-Check">
       <direction>UPGRADE</direction>
-      
+
       <execute-stage title="Check Component Versions">
         <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction"
/>
       </execute-stage>
@@ -469,7 +469,7 @@
     <group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
       <skippable>true</skippable>
       <supports-auto-skip-failure>false</supports-auto-skip-failure>
-      
+
       <execute-stage title="Confirm Finalize">
         <direction>UPGRADE</direction>
         <task xsi:type="manual">
@@ -493,7 +493,7 @@
           <function>finalize_rolling_upgrade</function>
         </task>
       </execute-stage>
-      
+
       <execute-stage title="Save Cluster State">
         <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction">
         </task>
@@ -544,7 +544,7 @@
             <function>setup_ranger_java_patches</function>
           </task>
         </pre-upgrade>
-        
+
         <pre-downgrade copy-upgrade="true" />
 
         <upgrade>
@@ -645,7 +645,7 @@
         </upgrade>
       </component>
     </service>
-    
+
     <service name="SLIDER">
       <component name="SLIDER">
         <upgrade>
@@ -667,7 +667,7 @@
             <summary>Verifying LZO codec path for mapreduce</summary>
           </task>
         </pre-upgrade>
-        
+
         <pre-downgrade copy-upgrade="true" />
 
         <upgrade>
@@ -748,9 +748,9 @@
             <summary>Verifying LZO codec path for Tez</summary>
           </task>
         </pre-upgrade>
-        
+
         <pre-downgrade copy-upgrade="true" />
-        
+
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>
@@ -830,6 +830,10 @@
         </upgrade>
       </component>
       <component name="SPARK_THRIFTSERVER">
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_6_0_0_spark_thriftserver"/>
+        </pre-upgrade>
+        <pre-downgrade />
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>
@@ -853,6 +857,10 @@
         </upgrade>
       </component>
       <component name="SPARK2_THRIFTSERVER">
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_6_0_0_spark2_thriftserver"/>
+        </pre-upgrade>
+        <pre-downgrade />
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>


Mime
View raw message