ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject [1/3] ambari git commit: AMBARI-13650. Bump the Ambari Spark definition to Spark 1.5.1 (Saisai Shao via smohanty)
Date Mon, 09 Nov 2015 01:29:22 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 66e2eb424 -> 618d843c1


AMBARI-13650. Bump the Ambari Spark definition to Spark 1.5.1 (Saisai Shao via smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/db224500
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/db224500
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/db224500

Branch: refs/heads/trunk
Commit: db224500f253f2bd6c842f9097efe572943001cf
Parents: 66e2eb4
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Sun Nov 8 17:13:35 2015 -0800
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Sun Nov 8 17:13:35 2015 -0800

----------------------------------------------------------------------
 .../1.2.0.2.2/configuration/spark-defaults.xml  | 17 ++-----
 .../SPARK/1.5.1.2.3/metainfo.xml                | 50 ++++++++++++++++++++
 .../configuration/spark-thrift-sparkconf.xml    | 13 +----
 .../stacks/HDP/2.3/services/SPARK/metainfo.xml  | 45 ++----------------
 4 files changed, 60 insertions(+), 65 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/db224500/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
index 8b13779..05d1a3a 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
@@ -42,15 +42,6 @@
   </property>
 
   <property>
-    <name>spark.yarn.applicationMaster.waitTries</name>
-    <value>10</value>
-    <description>
-      Set the number of times the ApplicationMaster waits for the the Spark master and then
-      also the number of tries it waits for the SparkContext to be initialized.
-    </description>
-  </property>
-
-  <property>
     <name>spark.yarn.scheduler.heartbeat.interval-ms</name>
     <value>5000</value>
     <description>
@@ -95,7 +86,7 @@
     <name>spark.yarn.preserve.staging.files</name>
     <value>false</value>
     <description>
-      Set to true to preserve the staged files (Spark jar, app jar, distributed cache files)
at the 
+      Set to true to preserve the staged files (Spark jar, app jar, distributed cache files)
at the
       end of the job rather then delete them.
     </description>
   </property>
@@ -161,19 +152,19 @@
       Location of the kerberos keytab file for the Spark History Server.
     </description>
   </property>
-  
+
   <property>
     <name>spark.history.provider</name>
     <value>org.apache.spark.deploy.yarn.history.YarnHistoryProvider</value>
     <description>Name of history provider class</description>
   </property>
-  
+
   <property>
     <name>spark.yarn.historyServer.address</name>
     <value>{{spark_history_server_host}}:{{spark_history_ui_port}}</value>
     <description>The address of the Spark history server (i.e. host.com:18080). The
address should not contain a scheme (http://). Defaults to not being set since the history
server is an optional service. This address is given to the YARN ResourceManager when the
Spark application finishes to link the application from the ResourceManager UI to the Spark
history server UI.</description>
   </property>
-  
+
   <property>
     <name>spark.yarn.max.executor.failures</name>
     <value>3</value>

http://git-wip-us.apache.org/repos/asf/ambari/blob/db224500/ambari-server/src/main/resources/common-services/SPARK/1.5.1.2.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.5.1.2.3/metainfo.xml
b/ambari-server/src/main/resources/common-services/SPARK/1.5.1.2.3/metainfo.xml
new file mode 100644
index 0000000..190d892
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.5.1.2.3/metainfo.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0"?>
+<!--Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+-->
+<metainfo>
+  <schemaVersion>2.0</schemaVersion>
+  <services>
+    <service>
+      <name>SPARK</name>
+      <extends>common-services/SPARK/1.4.1.2.3</extends>
+      <displayName>Spark</displayName>
+      <comment>Apache Spark is a fast and general engine for large-scale data processing.</comment>
+      <version>1.5.1.2.3</version>
+      <!-- No new components added for 1.5.1 -->
+      <configuration-dependencies>
+        <config-type>spark-defaults</config-type>
+        <config-type>spark-env</config-type>
+        <config-type>spark-log4j-properties</config-type>
+        <config-type>spark-metrics-properties</config-type>
+        <config-type>spark-javaopts-properties</config-type>
+        <config-type>spark-thrift-sparkconf</config-type>
+        <config-type>spark-hive-site-override</config-type>
+      </configuration-dependencies>
+
+      <commandScript>
+        <script>scripts/service_check.py</script>
+        <scriptType>PYTHON</scriptType>
+        <timeout>300</timeout>
+      </commandScript>
+
+      <requiredServices>
+        <service>YARN</service>
+      </requiredServices>
+    </service>
+  </services>
+</metainfo>

http://git-wip-us.apache.org/repos/asf/ambari/blob/db224500/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
index c42841f..41d2e21 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
@@ -42,15 +42,6 @@
   </property>
 
   <property>
-    <name>spark.yarn.applicationMaster.waitTries</name>
-    <value>10</value>
-    <description>
-      Set the number of times the ApplicationMaster waits for the the Spark master and then
-      also the number of tries it waits for the SparkContext to be initialized.
-    </description>
-  </property>
-
-  <property>
     <name>spark.yarn.scheduler.heartbeat.interval-ms</name>
     <value>5000</value>
     <description>
@@ -95,7 +86,7 @@
     <name>spark.yarn.preserve.staging.files</name>
     <value>false</value>
     <description>
-      Set to true to preserve the staged files (Spark jar, app jar, distributed cache files)
at the 
+      Set to true to preserve the staged files (Spark jar, app jar, distributed cache files)
at the
       end of the job rather then delete them.
     </description>
   </property>
@@ -115,7 +106,7 @@
       Specifies the parameters that are passed to the JVM of the Spark Application Master.
     </description>
   </property>
-  
+
   <property>
     <name>spark.yarn.max.executor.failures</name>
     <value>3</value>

http://git-wip-us.apache.org/repos/asf/ambari/blob/db224500/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
index 14161b4..0e20794 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
@@ -23,8 +23,8 @@
     <services>
         <service>
           <name>SPARK</name>
-          <version>1.4.1.2.3</version>
-          <extends>common-services/SPARK/1.3.1.2.3</extends>
+          <version>1.5.1.2.3</version>
+          <extends>common-services/SPARK/1.4.1.2.3</extends>
           <osSpecifics>
             <osSpecific>
               <osFamily>redhat7,redhat6,suse11</osFamily>
@@ -52,44 +52,7 @@
           <requiredServices>
             <service>YARN</service>
           </requiredServices>
-          <components>
-              <component>
-                <name>SPARK_THRIFTSERVER</name>
-                <displayName>Spark Thrift Server</displayName>
-                <deleted>true</deleted>
-                <category>MASTER</category>
-                <cardinality>0+</cardinality>
-                <versionAdvertised>true</versionAdvertised>
-                <dependencies>
-                  <dependency>
-                    <name>HDFS/HDFS_CLIENT</name>
-                    <scope>host</scope>
-                    <auto-deploy>
-                      <enabled>true</enabled>
-                    </auto-deploy>
-                  </dependency>
-                  <dependency>
-                     <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name>
-                     <scope>host</scope>
-                     <auto-deploy>
-                       <enabled>true</enabled>
-                     </auto-deploy>
-                  </dependency>
-                  <dependency>
-                    <name>YARN/YARN_CLIENT</name>
-                    <scope>host</scope>
-                    <auto-deploy>
-                      <enabled>true</enabled>
-                   </auto-deploy>
-                 </dependency>
-                </dependencies>
-                <commandScript>
-                  <script>scripts/spark_thrift_server.py</script>
-                  <scriptType>PYTHON</scriptType>
-                  <timeout>600</timeout>
-                </commandScript>
-              </component>
-          </components>
+          <!-- No new components compared to 1.4.1 -->
           <configuration-dependencies>
             <config-type>spark-defaults</config-type>
             <config-type>spark-env</config-type>
@@ -98,7 +61,7 @@
             <config-type>spark-javaopts-properties</config-type>
             <config-type>spark-thrift-sparkconf</config-type>
             <config-type>spark-hive-site-override</config-type>
-          </configuration-dependencies>          
+          </configuration-dependencies>
         </service>
     </services>
 </metainfo>


Mime
View raw message