Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 1522F200B49 for ; Wed, 3 Aug 2016 19:41:37 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 13D38160A86; Wed, 3 Aug 2016 17:41:37 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 8DE0C160A5D for ; Wed, 3 Aug 2016 19:41:35 +0200 (CEST) Received: (qmail 4611 invoked by uid 500); 3 Aug 2016 17:41:33 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 4533 invoked by uid 99); 3 Aug 2016 17:41:33 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 03 Aug 2016 17:41:33 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 43D8CE3839; Wed, 3 Aug 2016 17:41:33 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jluniya@apache.org To: commits@ambari.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-17954: Fix Spark hdp.version issues in upgrading and fresh install (Saisai Shao via jluniya) Date: Wed, 3 Aug 2016 17:41:33 +0000 (UTC) archived-at: Wed, 03 Aug 2016 17:41:37 -0000 Repository: ambari Updated Branches: refs/heads/trunk 15df97317 -> 9bb772392 AMBARI-17954: Fix Spark hdp.version issues in upgrading and fresh install (Saisai Shao via jluniya) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9bb77239 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9bb77239 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9bb77239 Branch: refs/heads/trunk Commit: 9bb77239292f99b935634eafe5e807d61e1d8cb1 Parents: 15df973 Author: Jayush Luniya Authored: Wed Aug 3 10:41:27 2016 -0700 Committer: Jayush Luniya Committed: Wed Aug 3 10:41:27 2016 -0700 ---------------------------------------------------------------------- .../libraries/functions/constants.py | 1 + .../1.2.1/configuration/spark-defaults.xml | 4 +-- .../SPARK/1.2.1/package/scripts/params.py | 10 +++++- .../SPARK/1.2.1/package/scripts/setup_spark.py | 23 +++++++------ .../configuration/spark-thrift-sparkconf.xml | 16 +++++++++ .../HDP/2.0.6/properties/stack_features.json | 6 ++++ .../stacks/HDP/2.3/upgrades/config-upgrade.xml | 27 +++++++++++++++- .../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml | 20 +++++++++--- .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 22 ++++++++++--- .../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 9 +++++- .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 9 +++++- .../SPARK/configuration/spark-defaults.xml | 32 ++++++++++++++++++ .../configuration/spark-thrift-sparkconf.xml | 32 ++++++++++++++++++ .../SPARK/configuration/spark-defaults.xml | 34 -------------------- .../stacks/2.2/SPARK/test_job_history_server.py | 21 ++++++------ .../stacks/2.2/SPARK/test_spark_client.py | 22 ++++++------- .../2.3/SPARK/test_spark_thrift_server.py | 12 +++---- 17 files changed, 213 insertions(+), 87 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-common/src/main/python/resource_management/libraries/functions/constants.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py index 6537e43..0d6f2b2 100644 --- a/ambari-common/src/main/python/resource_management/libraries/functions/constants.py +++ b/ambari-common/src/main/python/resource_management/libraries/functions/constants.py @@ -95,3 +95,4 @@ class StackFeature: RANGER_KMS_PID_SUPPORT = "ranger_kms_pid_support" RANGER_ADMIN_PASSWD_CHANGE = "ranger_admin_password_change" STORM_METRICS_APACHE_CLASSES = "storm_metrics_apache_classes" + SPARK_JAVA_OPTS_SUPPORT = "spark_java_opts_support" http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml index 646239e..ecb103c 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-defaults.xml @@ -121,7 +121,7 @@ spark.driver.extraJavaOptions - + -Dhdp.version={{full_stack_version}} Specifies parameters that are passed to the JVM of the Spark driver. @@ -132,7 +132,7 @@ spark.yarn.am.extraJavaOptions - + -Dhdp.version={{full_stack_version}} Specifies the parameters that are passed to the JVM of the Spark Application Master. http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py index 86aeb70..eded4bd 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/params.py @@ -124,8 +124,16 @@ hive_server_host = default("/clusterHostInfo/hive_server_host", []) is_hive_installed = not len(hive_server_host) == 0 full_stack_version = get_stack_version('spark-client') -spark_javaopts_properties = default("/configurations/spark-javaopts-properties/content", " ") +spark_javaopts_properties = default("/configurations/spark-javaopts-properties/content", " ") +if spark_javaopts_properties.find('-Dhdp.version') == -1: + spark_javaopts_properties = spark_javaopts_properties+ ' -Dhdp.version=' + str(full_stack_version) +else: + lists = spark_javaopts_properties.split(" ") + for idx, val in enumerate(lists): + if (val.startswith("-Dhdp.version=")): + lists[idx] = "-Dhdp.version=" + str(full_stack_version) + spark_javaopts_properties = " ".join(lists) security_enabled = config['configurations']['cluster-env']['security_enabled'] kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py index 2b5d36b..50c1555 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py @@ -95,17 +95,10 @@ def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None): mode=0644 ) - File(os.path.join(params.spark_conf, 'java-opts'), - owner=params.spark_user, - group=params.spark_group, - content=InlineTemplate(params.spark_javaopts_properties), - mode=0644 - ) - Directory(params.spark_logs_dir, owner=params.spark_user, group=params.spark_group, - mode=0755, + mode=0755, ) if params.is_hive_installed: @@ -129,6 +122,18 @@ def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None): if effective_version: effective_version = format_stack_version(effective_version) + if effective_version and check_stack_feature(StackFeature.SPARK_JAVA_OPTS_SUPPORT, effective_version): + File(os.path.join(params.spark_conf, 'java-opts'), + owner=params.spark_user, + group=params.spark_group, + content=InlineTemplate(params.spark_javaopts_properties), + mode=0644 + ) + else: + File(os.path.join(params.spark_conf, 'java-opts'), + action="delete" + ) + if params.spark_thrift_fairscheduler_content and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version): # create spark-thrift-fairscheduler.xml File(os.path.join(config_dir,"spark-thrift-fairscheduler.xml"), @@ -136,4 +141,4 @@ def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None): group=params.spark_group, mode=0755, content=InlineTemplate(params.spark_thrift_fairscheduler_content) - ) \ No newline at end of file + ) http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml b/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml index e27c986..865aa73 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml +++ b/ambari-server/src/main/resources/common-services/SPARK/1.5.2/configuration/spark-thrift-sparkconf.xml @@ -38,6 +38,22 @@ + spark.driver.extraJavaOptions + -Dhdp.version={{full_stack_version}} + + Specifies parameters that are passed to the JVM of the Spark driver. + + + + + spark.yarn.am.extraJavaOptions + -Dhdp.version={{full_stack_version}} + + Specifies the parameters that are passed to the JVM of the Spark Application Master. + + + + spark.yarn.driver.memoryOverhead 384 http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json index b297fa6..da577ac 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/properties/stack_features.json @@ -276,6 +276,12 @@ "name": "storm_metrics_apache_classes", "description": "Metrics sink for Storm that uses Apache class names", "min_version": "2.5.0.0" + }, + { + "name": "spark_java_opts_support", + "description": "Allow Spark to generate java-opts file", + "min_version": "2.2.0.0", + "max_version": "2.4.0.0" } ] } http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml index b295cc9..7661205 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml @@ -363,8 +363,27 @@ spark-defaults + + + + spark-javaopts-properties + + + + + + + + spark-defaults + + + + + spark-javaopts-properties + + @@ -379,6 +398,12 @@ + + + + + spark-javaopts-properties + @@ -442,7 +467,7 @@ - + storm-site FLUME_HANDLER - + ACCUMULO_TRACER ACCUMULO_GC @@ -304,9 +304,21 @@ + + + + + + + + + + + + @@ -437,7 +449,7 @@ - + false @@ -732,7 +744,7 @@ FLUME_HANDLER - + false true @@ -762,7 +774,7 @@ UPGRADE - + @@ -1160,7 +1172,7 @@ delete_storm_local_data - + http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml index c9f9823..6ba3d32 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml @@ -573,7 +573,7 @@ - + @@ -787,6 +787,7 @@ + @@ -796,12 +797,18 @@ + + + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml index 6390f1f..cf0be76 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml @@ -639,7 +639,7 @@ - + @@ -874,6 +874,7 @@ + @@ -883,12 +884,18 @@ + + + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml new file mode 100644 index 0000000..0f5c126 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-defaults.xml @@ -0,0 +1,32 @@ + + + + + spark.driver.extraJavaOptions + true + + + + spark.yarn.am.extraJavaOptions + true + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml new file mode 100644 index 0000000..0f5c126 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/SPARK/configuration/spark-thrift-sparkconf.xml @@ -0,0 +1,32 @@ + + + + + spark.driver.extraJavaOptions + true + + + + spark.yarn.am.extraJavaOptions + true + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/configuration/spark-defaults.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/configuration/spark-defaults.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/configuration/spark-defaults.xml deleted file mode 100644 index b3d05d8..0000000 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/configuration/spark-defaults.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - spark.driver.extraJavaOptions - - true - - - - spark.yarn.am.extraJavaOptions - - true - - - http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py index b2ed5df..186ff72 100644 --- a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py +++ b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py @@ -227,16 +227,16 @@ class TestJobHistoryServer(RMFTestCase): group = 'spark', mode = 0644 ) - self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts', - content = InlineTemplate(' '), + self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs', owner = 'spark', group = 'spark', - mode = 0644 + mode = 0755, ) - self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs', + self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts', + content = InlineTemplate(' -Dhdp.version=None'), owner = 'spark', group = 'spark', - mode = 0755, + mode = 0644 ) def assert_configure_secured(self): @@ -308,19 +308,18 @@ class TestJobHistoryServer(RMFTestCase): group = 'spark', mode = 0644 ) - self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts', - content = InlineTemplate(' '), + self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs', owner = 'spark', group = 'spark', - mode = 0644 + mode = 0755, ) - self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs', + self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts', + content = InlineTemplate(' -Dhdp.version=None'), owner = 'spark', group = 'spark', - mode = 0755, + mode = 0644 ) - @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") def test_pre_upgrade_restart_23(self, copy_to_hdfs_mock): config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json" http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py b/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py index ed331ea..0921edc 100644 --- a/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py +++ b/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py @@ -89,19 +89,18 @@ class TestSparkClient(RMFTestCase): group = 'spark', mode = 0644 ) - self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts', - content = InlineTemplate(' '), + self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs', owner = 'spark', group = 'spark', - mode = 0644 + mode = 0755, ) - self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs', + self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts', + content = InlineTemplate(' -Dhdp.version=None'), owner = 'spark', group = 'spark', - mode = 0755, + mode = 0644 ) - def assert_configure_secured(self): self.assertResourceCalled('Directory', '/var/run/spark', owner = 'spark', @@ -140,19 +139,18 @@ class TestSparkClient(RMFTestCase): group = 'spark', mode = 0644 ) - self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts', - content = InlineTemplate(' '), + self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs', owner = 'spark', group = 'spark', - mode = 0644 + mode = 0755, ) - self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs', + self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts', + content = InlineTemplate(' -Dhdp.version=None'), owner = 'spark', group = 'spark', - mode = 0755, + mode = 0644 ) - def test_pre_upgrade_restart_23(self): config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json" with open(config_file, "r") as f: http://git-wip-us.apache.org/repos/asf/ambari/blob/9bb77239/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py index a1abdfa..e8e9320 100644 --- a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py +++ b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py @@ -147,12 +147,6 @@ class TestSparkThriftServer(RMFTestCase): group = 'spark', mode = 0644 ) - self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts', - content = InlineTemplate(' '), - owner = 'spark', - group = 'spark', - mode = 0644 - ) self.assertResourceCalled('Directory', '/usr/hdp/current/spark-client/logs', owner = 'spark', group = 'spark', @@ -165,6 +159,12 @@ class TestSparkThriftServer(RMFTestCase): properties = self.getConfig()['configurations']['spark-thrift-sparkconf'], mode = 0644 ) + self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts', + content = InlineTemplate(' -Dhdp.version=None'), + owner = 'spark', + group = 'spark', + mode = 0644 + ) @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") def test_pre_upgrade_restart_23(self, copy_to_hdfs_mock):