ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject ambari git commit: AMBARI-13899. Remove hdp.version configuration in Ambari Spark definition (Saisai Shao via smohanty)
Date Mon, 16 Nov 2015 04:37:31 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 66e90bcfa -> cacbb5532


AMBARI-13899. Remove hdp.version configuration in Ambari Spark definition (Saisai Shao via
smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cacbb553
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cacbb553
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cacbb553

Branch: refs/heads/trunk
Commit: cacbb5532611131ebdc6f8ca4bc6a71d6998f7db
Parents: 66e90bc
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Sun Nov 15 20:37:20 2015 -0800
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Sun Nov 15 20:37:20 2015 -0800

----------------------------------------------------------------------
 .../1.2.0.2.2/configuration/spark-defaults.xml  | 22 --------------
 .../configuration/spark-javaopts-properties.xml | 30 --------------------
 .../SPARK/1.2.0.2.2/metainfo.xml                |  1 -
 .../SPARK/1.2.0.2.2/package/scripts/params.py   | 15 ----------
 .../1.2.0.2.2/package/scripts/setup_spark.py    | 14 +++------
 .../SPARK/1.3.1.2.3/metainfo.xml                |  1 -
 .../SPARK/1.4.1.2.3/metainfo.xml                |  1 -
 .../configuration/spark-thrift-sparkconf.xml    | 16 -----------
 .../stacks/HDP/2.3/services/SPARK/metainfo.xml  |  1 -
 .../stacks/2.2/SPARK/test_job_history_server.py | 13 ++-------
 .../stacks/2.2/SPARK/test_spark_client.py       | 12 ++------
 .../2.3/SPARK/test_spark_thrift_server.py       |  5 ----
 12 files changed, 8 insertions(+), 123 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
index 6def8b2..b507d5e 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
@@ -116,28 +116,6 @@
   </property>
 
   <property>
-    <name>spark.driver.extraJavaOptions</name>
-    <value>-Dhdp.version={{hdp_full_version}}</value>
-    <description>
-      Specifies parameters that are passed to the JVM of the Spark driver.
-    </description>
-    <value-attributes>
-      <empty-value-valid>true</empty-value-valid>
-    </value-attributes>
-  </property>
-
-  <property>
-    <name>spark.yarn.am.extraJavaOptions</name>
-    <value>-Dhdp.version={{hdp_full_version}}</value>
-    <description>
-      Specifies the parameters that are passed to the JVM of the Spark Application Master.
-    </description>
-    <value-attributes>
-      <empty-value-valid>true</empty-value-valid>
-    </value-attributes>
-  </property>
-
-  <property>
     <name>spark.history.kerberos.principal</name>
     <value>none</value>
     <description>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-javaopts-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-javaopts-properties.xml
b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-javaopts-properties.xml
deleted file mode 100644
index f48d79b..0000000
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-javaopts-properties.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration supports_final="true">
-  <property>
-    <name>content</name>
-    <description>Spark-javaopts-properties</description>
-    <value> </value>
-    <value-attributes>
-      <show-property-name>false</show-property-name>
-    </value-attributes>
-  </property>
-</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/metainfo.xml
b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/metainfo.xml
index f84fb73..61016ab 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/metainfo.xml
@@ -131,7 +131,6 @@
         <config-type>spark-env</config-type>
         <config-type>spark-log4j-properties</config-type>
         <config-type>spark-metrics-properties</config-type>
-        <config-type>spark-javaopts-properties</config-type>
       </configuration-dependencies>
 
       <commandScript>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
index 7706650..985c2e3 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
@@ -112,25 +112,10 @@ spark_history_ui_port = config['configurations']['spark-defaults']['spark.histor
 spark_env_sh = config['configurations']['spark-env']['content']
 spark_log4j_properties = config['configurations']['spark-log4j-properties']['content']
 spark_metrics_properties = config['configurations']['spark-metrics-properties']['content']
-spark_javaopts_properties = config['configurations']['spark-javaopts-properties']['content']
 
 hive_server_host = default("/clusterHostInfo/hive_server_host", [])
 is_hive_installed = not len(hive_server_host) == 0
 
-hdp_full_version = functions.get_hdp_version('spark-client')
-
-spark_driver_extraJavaOptions = str(config['configurations']['spark-defaults']['spark.driver.extraJavaOptions'])
-if spark_driver_extraJavaOptions.find('-Dhdp.version') == -1:
-  spark_driver_extraJavaOptions = spark_driver_extraJavaOptions + ' -Dhdp.version=' + str(hdp_full_version)
-
-spark_yarn_am_extraJavaOptions = str(config['configurations']['spark-defaults']['spark.yarn.am.extraJavaOptions'])
-if spark_yarn_am_extraJavaOptions.find('-Dhdp.version') == -1:
-  spark_yarn_am_extraJavaOptions = spark_yarn_am_extraJavaOptions + ' -Dhdp.version=' + str(hdp_full_version)
-
-spark_javaopts_properties = str(spark_javaopts_properties)
-if spark_javaopts_properties.find('-Dhdp.version') == -1:
-  spark_javaopts_properties = spark_javaopts_properties+ ' -Dhdp.version=' + str(hdp_full_version)
-
 security_enabled = config['configurations']['cluster-env']['security_enabled']
 kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',
None))
 spark_kerberos_keytab =  config['configurations']['spark-defaults']['spark.history.kerberos.keytab']

http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
index 9969a9b..debed40 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
@@ -44,12 +44,12 @@ def setup_spark(env, type, action = None):
                        mode=0775
     )
     params.HdfsResource(None, action="execute")
-    
+
   PropertiesFile(format("{spark_conf}/spark-defaults.conf"),
     properties = params.config['configurations']['spark-defaults'],
-    key_value_delimiter = " ", 
+    key_value_delimiter = " ",
     owner=params.spark_user,
-    group=params.spark_group,              
+    group=params.spark_group,
   )
 
   # create spark-env.sh in etc/conf dir
@@ -73,12 +73,6 @@ def setup_spark(env, type, action = None):
        content=InlineTemplate(params.spark_metrics_properties)
   )
 
-  File(os.path.join(params.spark_conf, 'java-opts'),
-       owner=params.spark_user,
-       group=params.spark_group,
-       content=params.spark_javaopts_properties
-  )
-
   if params.is_hive_installed:
     XmlConfig("hive-site.xml",
           conf_dir=params.spark_conf,
@@ -92,5 +86,5 @@ def setup_spark(env, type, action = None):
       and 'spark-thrift-sparkconf' in params.config['configurations']:
     PropertiesFile(params.spark_thrift_server_conf_file,
       properties = params.config['configurations']['spark-thrift-sparkconf'],
-      key_value_delimiter = " ",             
+      key_value_delimiter = " ",
     )

http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/common-services/SPARK/1.3.1.2.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.3.1.2.3/metainfo.xml
b/ambari-server/src/main/resources/common-services/SPARK/1.3.1.2.3/metainfo.xml
index 306d16c..2869f75 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.3.1.2.3/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.3.1.2.3/metainfo.xml
@@ -125,7 +125,6 @@
         <config-type>spark-env</config-type>
         <config-type>spark-log4j-properties</config-type>
         <config-type>spark-metrics-properties</config-type>
-        <config-type>spark-javaopts-properties</config-type>
       </configuration-dependencies>
 
       <commandScript>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/metainfo.xml
b/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/metainfo.xml
index 839bbac..0923dda 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/metainfo.xml
@@ -68,7 +68,6 @@
         <config-type>spark-env</config-type>
         <config-type>spark-log4j-properties</config-type>
         <config-type>spark-metrics-properties</config-type>
-        <config-type>spark-javaopts-properties</config-type>
         <config-type>spark-thrift-sparkconf</config-type>
         <config-type>spark-hive-site-override</config-type>
       </configuration-dependencies>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
index 41d2e21..b5742ea 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
@@ -92,22 +92,6 @@
   </property>
 
   <property>
-    <name>spark.driver.extraJavaOptions</name>
-    <value>-Dhdp.version={{hdp_full_version}}</value>
-    <description>
-      Specifies parameters that are passed to the JVM of the Spark driver.
-    </description>
-  </property>
-
-  <property>
-    <name>spark.yarn.am.extraJavaOptions</name>
-    <value>-Dhdp.version={{hdp_full_version}}</value>
-    <description>
-      Specifies the parameters that are passed to the JVM of the Spark Application Master.
-    </description>
-  </property>
-
-  <property>
     <name>spark.yarn.max.executor.failures</name>
     <value>3</value>
     <description>The maximum number of executor failures before failing the application.</description>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
index d86afdf..2c9680c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
@@ -58,7 +58,6 @@
             <config-type>spark-env</config-type>
             <config-type>spark-log4j-properties</config-type>
             <config-type>spark-metrics-properties</config-type>
-            <config-type>spark-javaopts-properties</config-type>
             <config-type>spark-thrift-sparkconf</config-type>
             <config-type>spark-hive-site-override</config-type>
           </configuration-dependencies>

http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
index 2a45c09..90d3f0a 100644
--- a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
+++ b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
@@ -215,12 +215,7 @@ class TestJobHistoryServer(RMFTestCase):
         owner = 'spark',
         group = 'spark',
     )
-    self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
-        content = '  -Dhdp.version=2.3.0.0-1597',
-        owner = 'spark',
-        group = 'spark',
-    )
-      
+
   def assert_configure_secured(self):
     self.assertResourceCalled('Directory', '/var/run/spark',
         owner = 'spark',
@@ -282,11 +277,7 @@ class TestJobHistoryServer(RMFTestCase):
         owner = 'spark',
         group = 'spark',
     )
-    self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
-        content = '  -Dhdp.version=2.3.0.0-1597',
-        owner = 'spark',
-        group = 'spark',
-    )
+
 
   @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
   def test_pre_upgrade_restart_23(self, copy_to_hdfs_mock):

http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py b/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py
index 9516d2f..798feb8 100644
--- a/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py
+++ b/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py
@@ -83,11 +83,7 @@ class TestSparkClient(RMFTestCase):
         owner = 'spark',
         group = 'spark',
     )
-    self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
-        content = '  -Dhdp.version=2.3.0.0-1597',
-        owner = 'spark',
-        group = 'spark',
-    )
+
       
   def assert_configure_secured(self):
     self.assertResourceCalled('Directory', '/var/run/spark',
@@ -121,11 +117,7 @@ class TestSparkClient(RMFTestCase):
         owner = 'spark',
         group = 'spark',
     )
-    self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
-        content = '  -Dhdp.version=2.3.0.0-1597',
-        owner = 'spark',
-        group = 'spark',
-    )
+
 
   def test_pre_upgrade_restart_23(self):
     config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json"

http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
index 0fcfbf0..05745ae 100644
--- a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
+++ b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
@@ -138,11 +138,6 @@ class TestSparkThriftServer(RMFTestCase):
         owner = 'spark',
         group = 'spark',
     )
-    self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
-        content = '  -Dhdp.version=2.3.2.0-1597',
-        owner = 'spark',
-        group = 'spark',
-    )
     self.assertResourceCalled('PropertiesFile', '/usr/hdp/current/spark-client/conf/spark-thrift-sparkconf.conf',
         key_value_delimiter = ' ',
         properties = self.getConfig()['configurations']['spark-thrift-sparkconf']


Mime
View raw message