ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [25/39] ambari git commit: AMBARI-14839 - DEA is not enabled due to configuration conflict and history log directory can not be changed (Jeff Zhang via jonathanhurley)
Date Mon, 01 Feb 2016 15:42:02 GMT
AMBARI-14839 - DEA is not enabled due to configuration conflict and history log directory can
not be changed (Jeff Zhang via jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6bac8ec1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6bac8ec1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6bac8ec1

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 6bac8ec15e774349d454aa05ec38e23d762b3ce8
Parents: 516912b
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Fri Jan 29 11:27:35 2016 -0500
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Fri Jan 29 11:34:41 2016 -0500

----------------------------------------------------------------------
 .../SPARK/1.2.0.2.2/package/scripts/params.py   |  2 +-
 .../SPARK/configuration/spark-defaults.xml      |  4 +--
 .../configuration/spark-thrift-sparkconf.xml    | 31 ++++++++++++++++----
 .../stacks/HDP/2.4/upgrades/config-upgrade.xml  | 15 ----------
 4 files changed, 29 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/6bac8ec1/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
index 935c47e..68c4f37 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
@@ -86,7 +86,7 @@ hive_user = status_params.hive_user
 spark_group = status_params.spark_group
 user_group = status_params.user_group
 spark_hdfs_user_dir = format("/user/{spark_user}")
-spark_history_dir = 'hdfs:///spark-history'
+spark_history_dir = default('/configurations/spark-defaults/spark.history.fs.logDirectory',
"hdfs:///spark-history")
 
 spark_history_server_pid_file = status_params.spark_history_server_pid_file
 spark_thrift_server_pid_file = status_params.spark_thrift_server_pid_file

http://git-wip-us.apache.org/repos/asf/ambari/blob/6bac8ec1/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml
b/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml
index d8af790..1a6552f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml
@@ -31,7 +31,7 @@
     </property>
     <property>
         <name>spark.history.fs.logDirectory</name>
-        <value>{{spark_history_dir}}</value>
+        <value>hdfs:///spark-history</value>
         <description>
             Base directory for history spark application log.
         </description>
@@ -45,7 +45,7 @@
     </property>
     <property>
         <name>spark.eventLog.dir</name>
-        <value>{{spark_history_dir}}</value>
+        <value>hdfs:///spark-history</value>
         <description>
             Base directory in which Spark events are logged, if spark.eventLog.enabled is
true.
         </description>

http://git-wip-us.apache.org/repos/asf/ambari/blob/6bac8ec1/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml
b/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml
index 3b13496..2dbfe51 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml
@@ -75,14 +75,17 @@
   <property>
     <name>spark.history.fs.logDirectory</name>
     <value>{{spark_history_dir}}</value>
+    <final>true</final>
     <description>
-      Base directory for history spark application log.
+      Base directory for history spark application log. It is the same value
+      as in spark-defaults.xml.
     </description>
   </property>
 
   <property>
     <name>spark.eventLog.enabled</name>
     <value>true</value>
+    <final>true</final>
     <description>
       Whether to log Spark events, useful for reconstructing the Web UI after the application
has finished.
     </description>
@@ -91,8 +94,10 @@
   <property>
     <name>spark.eventLog.dir</name>
     <value>{{spark_history_dir}}</value>
+    <final>true</final>
     <description>
-      Base directory in which Spark events are logged, if spark.eventLog.enabled is true.
+      Base directory in which Spark events are logged, if spark.eventLog.enabled is true.
It is the same value
+      as in spark-defaults.xml.
     </description>
   </property>
 
@@ -138,10 +143,26 @@
   </property>
 
   <property>
-    <name>spark.executor.instances</name>
-    <value>2</value>
+    <name>spark.dynamicAllocation.initialExecutors</name>
+    <value>0</value>
     <description>
-      The number of executor.
+      Initial number of executors to run if dynamic allocation is enabled.
+    </description>
+  </property>
+
+  <property>
+    <name>spark.dynamicAllocation.maxExecutors</name>
+    <value>10</value>
+    <description>
+      Upper bound for the number of executors if dynamic allocation is enabled.
+    </description>
+  </property>
+
+  <property>
+    <name>spark.dynamicAllocation.minExecutors</name>
+    <value>0</value>
+    <description>
+      Lower bound for the number of executors if dynamic allocation is enabled.
     </description>
   </property>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/6bac8ec1/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
index 5949486..ee06577 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
@@ -172,9 +172,6 @@
             <type>spark-defaults</type>
             <transfer operation="delete" delete-key="spark.yarn.services" />
             <set key="spark.history.provider" value="org.apache.spark.deploy.history.FsHistoryProvider"/>
-            <set key="spark.history.fs.logDirectory" value="{{spark_history_dir}}"/>
-            <set key="spark.eventLog.enabled" value="true"/>
-            <set key="spark.eventLog.dir" value="{{spark_history_dir}}"/>
           </definition>
         </changes>
       </component>
@@ -190,18 +187,6 @@
             <transfer operation="delete" delete-key="spark.yarn.submit.file.replication"
/>
             <transfer operation="delete" delete-key="spark.yarn.preserve.staging.files"
/>
             <transfer operation="delete" delete-key="spark.yarn.max.executor.failures"
/>
-            <set key="spark.history.provider" value="org.apache.spark.deploy.history.FsHistoryProvider"/>
-            <set key="spark.history.fs.logDirectory" value="{{spark_history_dir}}"/>
-            <set key="spark.eventLog.enabled" value="true"/>
-            <set key="spark.eventLog.dir" value="{{spark_history_dir}}"/>
-            <set key="spark.master" value="{{spark_thrift_master}}"/>
-            <set key="spark.scheduler.allocation.file" value="{{spark_conf}}/spark-thrift-fairscheduler.xml"/>
-            <set key="spark.scheduler.mode" value="FAIR"/>
-            <set key="spark.shuffle.service.enabled" value="true"/>
-            <set key="spark.dynamicAllocation.enabled" value="true"/>
-            <set key="spark.executor.instances" value="2"/>
-            <set key="spark.yarn.am.memory" value="512m"/>
-            <set key="spark.executor.memory" value="1g"/>
           </definition>
         </changes>
       </component>


Mime
View raw message