ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jlun...@apache.org
Subject ambari git commit: AMBARI-17954: Fix Spark hdp.version issues in upgrading and fresh install (Saisai Shao via jluniya)
Date Wed, 03 Aug 2016 17:41:33 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 15df97317 -> 9bb772392


AMBARI-17954: Fix Spark hdp.version issues in upgrading and fresh install (Saisai Shao via
jluniya)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9bb77239
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9bb77239
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9bb77239

Branch: refs/heads/trunk
Commit: 9bb77239292f99b935634eafe5e807d61e1d8cb1
Parents: 15df973
Author: Jayush Luniya <jluniya@hortonworks.com>
Authored: Wed Aug 3 10:41:27 2016 -0700
Committer: Jayush Luniya <jluniya@hortonworks.com>
Committed: Wed Aug 3 10:41:27 2016 -0700

----------------------------------------------------------------------
 .../libraries/functions/constants.py            |  1 +
 .../1.2.1/configuration/spark-defaults.xml      |  4 +--
 .../SPARK/1.2.1/package/scripts/params.py       | 10 +++++-
 .../SPARK/1.2.1/package/scripts/setup_spark.py  | 23 +++++++------
 .../configuration/spark-thrift-sparkconf.xml    | 16 +++++++++
 .../HDP/2.0.6/properties/stack_features.json    |  6 ++++
 .../stacks/HDP/2.3/upgrades/config-upgrade.xml  | 27 +++++++++++++++-
 .../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml | 20 +++++++++---
 .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 22 ++++++++++---
 .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml     |  9 +++++-
 .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml     |  9 +++++-
 .../SPARK/configuration/spark-defaults.xml      | 32 ++++++++++++++++++
 .../configuration/spark-thrift-sparkconf.xml    | 32 ++++++++++++++++++
 .../SPARK/configuration/spark-defaults.xml      | 34 --------------------
 .../stacks/2.2/SPARK/test_job_history_server.py | 21 ++++++------
 .../stacks/2.2/SPARK/test_spark_client.py       | 22 ++++++-------
 .../2.3/SPARK/test_spark_thrift_server.py       | 12 +++----
 17 files changed, 213 insertions(+), 87 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
index 6537e43..0d6f2b2 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py
@@ -95,3 +95,4 @@ class StackFeature:
   RANGER_KMS_PID_SUPPORT = "ranger_kms_pid_support"
   RANGER_ADMIN_PASSWD_CHANGE = "ranger_admin_password_change"
   STORM_METRICS_APACHE_CLASSES = "storm_metrics_apache_classes"
+  SPARK_JAVA_OPTS_SUPPORT = "spark_java_opts_support"

http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml
b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml
index 646239e..ecb103c 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml
@@ -121,7 +121,7 @@
   </property>
   <property>
     <name>spark.driver.extraJavaOptions</name>
-    <value></value>
+    <value>-Dhdp.version={{full_stack_version}}</value>
     <description>
       Specifies parameters that are passed to the JVM of the Spark driver.
     </description>
@@ -132,7 +132,7 @@
   </property>
   <property>
     <name>spark.yarn.am.extraJavaOptions</name>
-    <value></value>
+    <value>-Dhdp.version={{full_stack_version}}</value>
     <description>
       Specifies the parameters that are passed to the JVM of the Spark Application Master.
     </description>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
index 86aeb70..eded4bd 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py
@@ -124,8 +124,16 @@ hive_server_host = default("/clusterHostInfo/hive_server_host", [])
 is_hive_installed = not len(hive_server_host) == 0
 
 full_stack_version = get_stack_version('spark-client')
-spark_javaopts_properties = default("/configurations/spark-javaopts-properties/content",
" ")
 
+spark_javaopts_properties = default("/configurations/spark-javaopts-properties/content",
" ")
+if spark_javaopts_properties.find('-Dhdp.version') == -1:
+  spark_javaopts_properties = spark_javaopts_properties+ ' -Dhdp.version=' + str(full_stack_version)
+else:
+  lists = spark_javaopts_properties.split(" ")
+  for idx, val in enumerate(lists):
+    if (val.startswith("-Dhdp.version=")):
+        lists[idx] = "-Dhdp.version=" + str(full_stack_version)
+  spark_javaopts_properties = " ".join(lists)
 
 security_enabled = config['configurations']['cluster-env']['security_enabled']
 kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',
None))

http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
index 2b5d36b..50c1555 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py
@@ -95,17 +95,10 @@ def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None):
        mode=0644
   )
 
-  File(os.path.join(params.spark_conf, 'java-opts'),
-      owner=params.spark_user,
-      group=params.spark_group,
-      content=InlineTemplate(params.spark_javaopts_properties),
-      mode=0644
-  )
-
   Directory(params.spark_logs_dir,
        owner=params.spark_user,
        group=params.spark_group,
-       mode=0755,   
+       mode=0755,
   )
 
   if params.is_hive_installed:
@@ -129,6 +122,18 @@ def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None):
   if effective_version:
     effective_version = format_stack_version(effective_version)
 
+  if effective_version and check_stack_feature(StackFeature.SPARK_JAVA_OPTS_SUPPORT, effective_version):
+    File(os.path.join(params.spark_conf, 'java-opts'),
+      owner=params.spark_user,
+      group=params.spark_group,
+      content=InlineTemplate(params.spark_javaopts_properties),
+      mode=0644
+    )
+  else:
+    File(os.path.join(params.spark_conf, 'java-opts'),
+      action="delete"
+    )
+
   if params.spark_thrift_fairscheduler_content and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS,
effective_version):
     # create spark-thrift-fairscheduler.xml
     File(os.path.join(config_dir,"spark-thrift-fairscheduler.xml"),
@@ -136,4 +141,4 @@ def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None):
       group=params.spark_group,
       mode=0755,
       content=InlineTemplate(params.spark_thrift_fairscheduler_content)
-    )
\ No newline at end of file
+    )

http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml
b/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml
index e27c986..865aa73 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml
@@ -38,6 +38,22 @@
     <on-ambari-upgrade add="true"/>
   </property>
   <property>
+    <name>spark.driver.extraJavaOptions</name>
+    <value>-Dhdp.version={{full_stack_version}}</value>
+    <description>
+      Specifies parameters that are passed to the JVM of the Spark driver.
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>spark.yarn.am.extraJavaOptions</name>
+    <value>-Dhdp.version={{full_stack_version}}</value>
+    <description>
+      Specifies the parameters that are passed to the JVM of the Spark Application Master.
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
     <name>spark.yarn.driver.memoryOverhead</name>
     <value>384</value>
     <description>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
index b297fa6..da577ac 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json
@@ -276,6 +276,12 @@
       "name": "storm_metrics_apache_classes",
       "description": "Metrics sink for Storm that uses Apache class names",
       "min_version": "2.5.0.0"
+    },
+    {
+      "name": "spark_java_opts_support",
+      "description": "Allow Spark to generate java-opts file",
+      "min_version": "2.2.0.0",
+      "max_version": "2.4.0.0"
     }
   ]
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
index b295cc9..7661205 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
@@ -363,8 +363,27 @@
           <definition xsi:type="configure" id="hdp_2_4_0_0_spark_jobhistoryserver">
             <type>spark-defaults</type>
             <transfer operation="delete" delete-key="spark.yarn.services" />
+            <transfer operation="delete" delete-key="spark.driver.extraJavaOptions" />
+            <transfer operation="delete" delete-key="spark.yarn.am.extraJavaOptions" />
             <set key="spark.history.provider" value="org.apache.spark.deploy.history.FsHistoryProvider"/>
           </definition>
+          <definition xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts">
+            <type>spark-javaopts-properties</type>
+            <transfer operation="delete" delete-key="content" />
+          </definition>
+        </changes>
+      </component>
+      <component name="SPARK_CLIENT">
+        <changes>
+          <definition xsi:type="configure" id="hdp_2_4_0_0_remove_spark_properties_extraJavaOptions">
+            <type>spark-defaults</type>
+            <transfer operation="delete" delete-key="spark.driver.extraJavaOptions" />
+            <transfer operation="delete" delete-key="spark.yarn.am.extraJavaOptions" />
+          </definition>
+          <definition xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts">
+            <type>spark-javaopts-properties</type>
+            <transfer operation="delete" delete-key="content" />
+          </definition>
         </changes>
       </component>
       <component name="SPARK_THRIFTSERVER">
@@ -379,6 +398,12 @@
             <transfer operation="delete" delete-key="spark.yarn.submit.file.replication"
/>
             <transfer operation="delete" delete-key="spark.yarn.preserve.staging.files"
/>
             <transfer operation="delete" delete-key="spark.yarn.max.executor.failures"
/>
+            <transfer operation="delete" delete-key="spark.driver.extraJavaOptions" />
+            <transfer operation="delete" delete-key="spark.yarn.am.extraJavaOptions" />
+          </definition>
+          <definition xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts">
+            <type>spark-javaopts-properties</type>
+            <transfer operation="delete" delete-key="content" />
           </definition>
         </changes>
       </component>
@@ -442,7 +467,7 @@
             <transfer operation="delete" delete-key="xasecure.audit.credential.provider.file"
/>
             <transfer operation="delete" delete-key="xasecure.audit.destination.db.batch.filespool.dir"
/>
           </definition>
-          
+
           <definition xsi:type="configure" id="hdp_2_5_0_0_upgrade_storm_1.0">
             <type>storm-site</type>
             <replace key="_storm.thrift.nonsecure.transport" find="backtype.storm.security.auth.SimpleTransportPlugin"

http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
index 5b20a72..dbdecc7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
@@ -71,7 +71,7 @@
       <service name="FLUME">
         <component>FLUME_HANDLER</component>
       </service>
-      
+
       <service name="ACCUMULO">
         <component>ACCUMULO_TRACER</component>
         <component>ACCUMULO_GC</component>
@@ -304,9 +304,21 @@
       <execute-stage service="SPARK" component="SPARK_JOBHISTORYSERVER" title="Apply config
changes for Spark JobHistoryServer">
         <task xsi:type="configure" id="hdp_2_4_0_0_spark_jobhistoryserver"/>
       </execute-stage>
+      <execute-stage service="SPARK" component="SPARK_JOBHISTORYSERVER" title="Apply config
changes for Spark JobHistoryServer">
+        <task xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts"/>
+      </execute-stage>
       <execute-stage service="SPARK" component="SPARK_THRIFTSERVER" title="Apply config
changes for Spark ThriftServer">
         <task xsi:type="configure" id="hdp_2_4_0_0_spark_thriftserver"/>
       </execute-stage>
+      <execute-stage service="SPARK" component="SPARK_THRIFTSERVER" title="Apply config
changes for Spark ThriftServer">
+        <task xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts"/>
+      </execute-stage>
+      <execute-stage service="SPARK" component="SPARK_CLIENT" title="Apply config changes
for Spark">
+        <task xsi:type="configure" id="hdp_2_4_0_0_remove_spark_properties_extraJavaOptions"/>
+      </execute-stage>
+      <execute-stage service="SPARK" component="SPARK_CLIENT" title="Apply config changes
for Spark">
+        <task xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts"/>
+      </execute-stage>
     </group>
 
     <!--
@@ -618,7 +630,7 @@
         <component>FLUME_HANDLER</component>
       </service>
     </group>
-    
+
     <group xsi:type="restart" name="ACCUMULO" title="Accumulo">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -648,7 +660,7 @@
 
     <group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}}
Pre-Check">
       <direction>UPGRADE</direction>
-      
+
       <execute-stage title="Check Component Versions">
         <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction"
/>
       </execute-stage>
@@ -1050,7 +1062,7 @@
             <function>delete_storm_local_data</function>
           </task>
         </pre-downgrade>
-        
+
         <upgrade>
           <task xsi:type="restart-task"/>
         </upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
index e046849..4c09f43 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml
@@ -80,7 +80,7 @@
       <service name="FLUME">
         <component>FLUME_HANDLER</component>
       </service>
-      
+
       <service name="ACCUMULO">
         <component>ACCUMULO_TRACER</component>
         <component>ACCUMULO_GC</component>
@@ -364,9 +364,21 @@
       <execute-stage service="SPARK" component="SPARK_JOBHISTORYSERVER" title="Apply config
changes for Spark JobHistoryServer">
         <task xsi:type="configure" id="hdp_2_4_0_0_spark_jobhistoryserver"/>
       </execute-stage>
+      <execute-stage service="SPARK" component="SPARK_JOBHISTORYSERVER" title="Apply config
changes for Spark JobHistoryServer">
+        <task xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts"/>
+      </execute-stage>
       <execute-stage service="SPARK" component="SPARK_THRIFTSERVER" title="Apply config
changes for Spark ThriftServer">
         <task xsi:type="configure" id="hdp_2_4_0_0_spark_thriftserver"/>
       </execute-stage>
+      <execute-stage service="SPARK" component="SPARK_THRIFTSERVER" title="Apply config
changes for Spark ThriftServer">
+        <task xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts"/>
+      </execute-stage>
+      <execute-stage service="SPARK" component="SPARK_CLIENT" title="Apply config changes
for Spark">
+        <task xsi:type="configure" id="hdp_2_4_0_0_remove_spark_properties_extraJavaOptions"/>
+      </execute-stage>
+      <execute-stage service="SPARK" component="SPARK_CLIENT" title="Apply config changes
for Spark">
+        <task xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts"/>
+      </execute-stage>
 
       <!-- RANGER -->
       <execute-stage service="RANGER" component="RANGER_ADMIN" title="Apply config changes
for Ranger Admin">
@@ -437,7 +449,7 @@
         </task>
       </execute-stage>
     </group>
-    
+
     <!-- Now, restart all of the services. -->
     <group xsi:type="restart" name="ZOOKEEPER" title="ZooKeeper">
       <service-check>false</service-check>
@@ -732,7 +744,7 @@
         <component>FLUME_HANDLER</component>
       </service>
     </group>
-    
+
     <group xsi:type="restart" name="ACCUMULO" title="Accumulo">
       <service-check>false</service-check>
       <skippable>true</skippable>
@@ -762,7 +774,7 @@
 
     <group xsi:type="cluster" name="FINALIZE_PRE_CHECK" title="Finalize {{direction.text.proper}}
Pre-Check">
       <direction>UPGRADE</direction>
-      
+
       <execute-stage title="Check Component Versions">
         <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.ComponentVersionCheckAction"
/>
       </execute-stage>
@@ -1160,7 +1172,7 @@
             <function>delete_storm_local_data</function>
           </task>
         </pre-downgrade>
-        
+
         <upgrade>
           <task xsi:type="restart-task"/>
         </upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index c9f9823..6ba3d32 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -573,7 +573,7 @@
           <task xsi:type="restart-task" />
         </upgrade>
       </component>
-      
+
       <component name="NFS_GATEWAY">
         <upgrade>
           <task xsi:type="restart-task" />
@@ -787,6 +787,7 @@
         <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_4_0_0_spark_jobhistoryserver"/>
+          <task xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts"/>
         </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task" />
@@ -796,12 +797,18 @@
         <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_4_0_0_spark_thriftserver"/>
+          <task xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts"/>
         </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task"/>
         </upgrade>
       </component>
       <component name="SPARK_CLIENT">
+        <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_4_0_0_remove_spark_properties_extraJavaOptions"/>
+          <task xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts"/>
+        </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
index 6390f1f..cf0be76 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml
@@ -639,7 +639,7 @@
           <task xsi:type="restart-task" />
         </upgrade>
       </component>
-      
+
       <component name="NFS_GATEWAY">
         <upgrade>
           <task xsi:type="restart-task" />
@@ -874,6 +874,7 @@
         <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_4_0_0_spark_jobhistoryserver"/>
+          <task xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts"/>
         </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task" />
@@ -883,12 +884,18 @@
         <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
         <pre-upgrade>
           <task xsi:type="configure" id="hdp_2_4_0_0_spark_thriftserver"/>
+          <task xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts"/>
         </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task"/>
         </upgrade>
       </component>
       <component name="SPARK_CLIENT">
+        <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_4_0_0_remove_spark_properties_extraJavaOptions"/>
+          <task xsi:type="configure" id="hdp_2_4_0_0_spark_java_opts"/>
+        </pre-upgrade>
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml
b/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml
new file mode 100644
index 0000000..0f5c126
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+  <property>
+    <name>spark.driver.extraJavaOptions</name>
+    <deleted>true</deleted>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>spark.yarn.am.extraJavaOptions</name>
+    <deleted>true</deleted>
+    <on-ambari-upgrade add="false"/>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml
b/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml
new file mode 100644
index 0000000..0f5c126
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+  <property>
+    <name>spark.driver.extraJavaOptions</name>
+    <deleted>true</deleted>
+    <on-ambari-upgrade add="false"/>
+  </property>
+  <property>
+    <name>spark.yarn.am.extraJavaOptions</name>
+    <deleted>true</deleted>
+    <on-ambari-upgrade add="false"/>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/configuration/spark-defaults.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/configuration/spark-defaults.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/configuration/spark-defaults.xml
deleted file mode 100644
index b3d05d8..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/configuration/spark-defaults.xml
+++ /dev/null
@@ -1,34 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration supports_final="true">
-  <property>
-    <name>spark.driver.extraJavaOptions</name>
-    <value></value>
-    <deleted>true</deleted>
-    <on-ambari-upgrade add="false"/>
-  </property>
-  <property>
-    <name>spark.yarn.am.extraJavaOptions</name>
-    <value></value>
-    <deleted>true</deleted>
-    <on-ambari-upgrade add="false"/>
-  </property>
-</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
index b2ed5df..186ff72 100644
--- a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
+++ b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
@@ -227,16 +227,16 @@ class TestJobHistoryServer(RMFTestCase):
         group = 'spark',
         mode = 0644
     )
-    self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
-        content = InlineTemplate(' '),
+    self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs',
         owner = 'spark',
         group = 'spark',
-        mode = 0644
+        mode = 0755,
     )
-    self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs',
+    self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
+        content = InlineTemplate('  -Dhdp.version=None'),
         owner = 'spark',
         group = 'spark',
-        mode = 0755,
+        mode = 0644
     )
 
   def assert_configure_secured(self):
@@ -308,19 +308,18 @@ class TestJobHistoryServer(RMFTestCase):
         group = 'spark',
         mode = 0644
     )
-    self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
-        content = InlineTemplate(' '),
+    self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs',
         owner = 'spark',
         group = 'spark',
-        mode = 0644
+        mode = 0755,
     )
-    self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs',
+    self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
+        content = InlineTemplate('  -Dhdp.version=None'),
         owner = 'spark',
         group = 'spark',
-        mode = 0755,
+        mode = 0644
     )
 
-
   @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
   def test_pre_upgrade_restart_23(self, copy_to_hdfs_mock):
     config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json"

http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py b/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py
index ed331ea..0921edc 100644
--- a/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py
+++ b/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py
@@ -89,19 +89,18 @@ class TestSparkClient(RMFTestCase):
         group = 'spark',
         mode = 0644
     )
-    self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
-        content = InlineTemplate(' '),
+    self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs',
         owner = 'spark',
         group = 'spark',
-        mode = 0644
+        mode = 0755,
     )
-    self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs',
+    self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
+        content = InlineTemplate('  -Dhdp.version=None'),
         owner = 'spark',
         group = 'spark',
-        mode = 0755,
+        mode = 0644
     )
 
-
   def assert_configure_secured(self):
     self.assertResourceCalled('Directory', '/var/run/spark',
         owner = 'spark',
@@ -140,19 +139,18 @@ class TestSparkClient(RMFTestCase):
         group = 'spark',
         mode = 0644
     )
-    self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
-        content = InlineTemplate(' '),
+    self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs',
         owner = 'spark',
         group = 'spark',
-        mode = 0644
+        mode = 0755,
     )
-    self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs',
+    self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
+        content = InlineTemplate('  -Dhdp.version=None'),
         owner = 'spark',
         group = 'spark',
-        mode = 0755,
+        mode = 0644
     )
 
-
   def test_pre_upgrade_restart_23(self):
     config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json"
     with open(config_file, "r") as f:

http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
index a1abdfa..e8e9320 100644
--- a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
+++ b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
@@ -147,12 +147,6 @@ class TestSparkThriftServer(RMFTestCase):
         group = 'spark',
         mode = 0644
     )
-    self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
-        content = InlineTemplate(' '),
-        owner = 'spark',
-        group = 'spark',
-        mode = 0644
-    )
     self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs',
         owner = 'spark',
         group = 'spark',
@@ -165,6 +159,12 @@ class TestSparkThriftServer(RMFTestCase):
         properties = self.getConfig()['configurations']['spark-thrift-sparkconf'],
         mode = 0644
     )
+    self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
+        content = InlineTemplate('  -Dhdp.version=None'),
+        owner = 'spark',
+        group = 'spark',
+        mode = 0644
+    )
 
   @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
   def test_pre_upgrade_restart_23(self, copy_to_hdfs_mock):


Mime
View raw message