ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mradhakrish...@apache.org
Subject [12/51] [abbrv] ambari git commit: AMBARI-22525 - ATS start failed during patch upgrade due to CNF SparkATSPlugin (jonathanhurley)
Date Tue, 05 Dec 2017 20:35:38 GMT
AMBARI-22525 - ATS start failed during patch upgrade due to CNF SparkATSPlugin (jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3fa1289e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3fa1289e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3fa1289e

Branch: refs/heads/branch-feature-AMBARI-14714
Commit: 3fa1289eaaf9c30985bcf5a50725080a9eb9b368
Parents: 393fdb8
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Mon Nov 27 13:58:38 2017 -0500
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Tue Nov 28 10:24:31 2017 -0500

----------------------------------------------------------------------
 .../libraries/functions/component_version.py          |  9 +++++----
 .../YARN/2.1.0.2.0/package/scripts/params_linux.py    |  6 ++++++
 .../YARN/3.0.0.3.0/package/scripts/params_linux.py    |  6 ++++++
 .../HDP/2.5/services/YARN/configuration/yarn-site.xml |  4 ++--
 .../stacks/HDP/2.5/services/stack_advisor.py          |  2 +-
 .../stacks/HDP/2.6/upgrades/config-upgrade.xml        | 14 ++++++++++++--
 .../HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml       | 10 ++++++++--
 .../resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml |  4 ++++
 .../HDP/3.0/services/YARN/configuration/yarn-site.xml |  4 ++--
 .../1.0/services/FAKEYARN/configuration/yarn-site.xml |  4 ++--
 .../python/stacks/2.5/common/test_stack_advisor.py    |  2 +-
 11 files changed, 49 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py
b/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py
index 169b339..fd18232 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/component_version.py
@@ -20,7 +20,7 @@ limitations under the License.
 
 from resource_management.libraries.script.script import Script
 
-def get_component_repository_version(service_name = None, component_name = None):
+def get_component_repository_version(service_name = None, component_name = None, default_value
= None):
   """
   Gets the version associated with the specified component from the structure in the command.
   Every command should contain a mapping of service/component to the desired repository it's
set
@@ -28,22 +28,23 @@ def get_component_repository_version(service_name = None, component_name
= None)
 
   :service_name: the name of the service
   :component_name: the name of the component
+  :default_value: the value to return if either the service or the component are not found
   """
   config = Script.get_config()
 
   versions = _get_component_repositories(config)
   if versions is None:
-    return None
+    return default_value
 
   if service_name is None:
     service_name = config['serviceName'] if config is not None and 'serviceName' in config
else None
 
   if service_name is None or service_name not in versions:
-    return None
+    return default_value
 
   component_versions = versions[service_name]
   if len(component_versions) == 0:
-    return None
+    return default_value
 
   if component_name is None:
     component_name = config["role"] if config is not None and "role" in config else None

http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 3e4504d..b244af0 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -84,6 +84,12 @@ stack_supports_timeline_state_store = check_stack_feature(StackFeature.TIMELINE_
 # It cannot be used during the initial Cluser Install because the version is not yet known.
 version = default("/commandParams/version", None)
 
+# these are used to render the classpath for picking up Spark classes
+# in the event that spark is not installed, then we must default to the vesrion of YARN installed
+# since it will still load classes from its own spark version
+spark_version = component_version.get_component_repository_version("SPARK", "SPARK_CLIENT",
default_value = version)
+spark2_version = component_version.get_component_repository_version("SPARK2", "SPARK2_CLIENT",
default_value = version)
+
 stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT,
version_for_stack_feature_checks)
 stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT,
version_for_stack_feature_checks)
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
index 617dc3b..372ae85 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/params_linux.py
@@ -84,6 +84,12 @@ stack_supports_timeline_state_store = check_stack_feature(StackFeature.TIMELINE_
 # It cannot be used during the initial Cluser Install because the version is not yet known.
 version = default("/commandParams/version", None)
 
+# these are used to render the classpath for picking up Spark classes
+# in the event that spark is not installed, then we must default to the vesrion of YARN installed
+# since it will still load classes from its own spark version
+spark_version = component_version.get_component_repository_version("SPARK", "SPARK_CLIENT",
default_value = version)
+spark2_version = component_version.get_component_repository_version("SPARK2", "SPARK2_CLIENT",
default_value = version)
+
 stack_supports_ranger_kerberos = check_stack_feature(StackFeature.RANGER_KERBEROS_SUPPORT,
version_for_stack_feature_checks)
 stack_supports_ranger_audit_db = check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT,
version_for_stack_feature_checks)
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
index b6fadcb..29833fb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml
@@ -31,13 +31,13 @@
   </property>
   <property>
     <name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name>
-    <value>{{stack_root}}/${hdp.version}/spark/aux/*</value>
+    <value>{{stack_root}}/{{spark_version}}/spark/aux/*</value>
     <description>The auxiliary service classpath to use for Spark</description>
     <on-ambari-upgrade add="false"/>
   </property>
   <property>
     <name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name>
-    <value>{{stack_root}}/${hdp.version}/spark2/aux/*</value>
+    <value>{{stack_root}}/{{spark2_version}}/spark2/aux/*</value>
     <description>The auxiliary service classpath to use for Spark 2</description>
     <on-ambari-upgrade add="false"/>
   </property>

http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
index b6f2478..6f409cd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/stack_advisor.py
@@ -823,7 +823,7 @@ class HDP25StackAdvisor(HDP24StackAdvisor):
 
     if self.isServiceDeployed(services, "SPARK"):
       timeline_plugin_classes_values.append('org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin')
-      timeline_plugin_classpath_values.append(stack_root + "/${hdp.version}/spark/hdpLib/*")
+      timeline_plugin_classpath_values.append(stack_root + "/{{spark_version}}/spark/hdpLib/*")
 
     putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes',
",".join(timeline_plugin_classes_values))
     putYarnSiteProperty('yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath',
":".join(timeline_plugin_classpath_values))

http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
index bcb13bc..9311877 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
@@ -162,14 +162,13 @@
               if-key-state="absent" />
           </definition>
 
-
           <definition xsi:type="configure" id="hdp_2_6_yarn_app_classpath_parameterization"
summary="Application Classpath Parameterization">
             <type>yarn-site</type>
             <replace key="yarn.application.classpath" find="/usr/hdp/current/hadoop-client"
replace-with="{{hadoop_home}}"/>
           </definition>
-
         </changes>
       </component>
+
       <component name="NODEMANAGER">
         <changes>
           <definition xsi:type="configure" id="hdp_2_6_0_0_yarn_nodemanager_llap_mem">
@@ -179,6 +178,17 @@
           </definition>
         </changes>
       </component>
+
+      <component name="APP_TIMELINE_SERVER">
+        <changes>
+          <definition xsi:type="configure" id="hdp_2_6_yarn_spark_versioned_classpaths">
+            <type>yarn-site</type>
+            <set key="yarn.nodemanager.aux-services.spark_shuffle.classpath" value="{{stack_root}}/{{spark_version}}/spark/aux/*"/>
+            <set key="yarn.nodemanager.aux-services.spark2_shuffle.classpath" value="{{stack_root}}/{{spark2_version}}/spark2/aux/*"/>
+            <replace key="yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath"
find="${hdp.version}/spark/" replace-with="{{spark_version}}/spark/"/>
+          </definition>
+        </changes>
+      </component>
     </service>
 
     <service name="KAFKA">

http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
index 0355362..2184060 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/nonrolling-upgrade-2.6.xml
@@ -147,6 +147,7 @@
         <component>DRUID_SUPERSET</component>
       </service>
 
+
     </group>
 
     <group xsi:type="cluster" name="Backups" title="Perform Backups">
@@ -351,7 +352,12 @@
         <task xsi:type="configure" id="hdp_2_6_yarn_app_classpath_parameterization" supports-patch="true"/>
       </execute-stage>
 
-      <!-- YARN -->
+      <execute-stage service="YARN" component="APP_TIMELINE_SERVER" title="Spark Versioned
Classpaths">
+        <task xsi:type="configure" id="hdp_2_6_yarn_spark_versioned_classpaths" supports-patch="true">
+          <summary>Updating the Spark classpath entries to use the specific version
of Spark which is installed.</summary>
+        </task>
+      </execute-stage>
+
       <execute-stage service="YARN" component="NODEMANAGER" title="Apply config changes
for YARN NM">
         <task xsi:type="configure" id="hdp_2_6_0_0_yarn_nodemanager_llap_mem">
           <summary>Updating YARN NodeManager config for LLAP</summary>
@@ -414,7 +420,7 @@
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Updating LD Library Classpath
for Tarball" >
         <task xsi:type="configure" id="hdp_2_6_tez_tarball_ld_library" supports-patch="true"/>
       </execute-stage>
-      
+
       <!-- MapR -->
       <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Updating
LD Library Classpath for Tarball">
         <task xsi:type="configure" id="hdp_2_6_mapreduce_tarball_ld_library" supports-patch="true"/>

http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
index 5aa2d20..70f84ab 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/upgrade-2.6.xml
@@ -690,6 +690,10 @@
 
     <service name="YARN">
       <component name="APP_TIMELINE_SERVER">
+        <pre-upgrade>
+          <task xsi:type="configure" id="hdp_2_6_yarn_spark_versioned_classpaths" supports-patch="true"/>
+        </pre-upgrade>
+        <pre-downgrade/>
         <upgrade>
           <task xsi:type="restart-task" />
         </upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
index 01551c4..d32e2de 100644
--- a/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/YARN/configuration/yarn-site.xml
@@ -20,7 +20,7 @@
 
   <property>
     <name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name>
-    <value>{{stack_root}}/${hdp.version}/spark/aux/*</value>
+    <value>{{stack_root}}/{{spark_version}}/spark/aux/*</value>
     <description>The auxiliary service classpath to use for Spark</description>
     <on-ambari-upgrade add="false"/>
   </property>
@@ -41,7 +41,7 @@
   <!-- These configs were inherited from HDP 2.5 -->
   <property>
     <name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name>
-    <value>{{stack_root}}/${hdp.version}/spark2/aux/*</value>
+    <value>{{stack_root}}/{{spark2_version}}/spark2/aux/*</value>
     <description>The auxiliary service classpath to use for Spark 2</description>
     <on-ambari-upgrade add="false"/>
   </property>

http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEYARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEYARN/configuration/yarn-site.xml
b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEYARN/configuration/yarn-site.xml
index 57d03a8..392d0de 100644
--- a/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEYARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/PERF/1.0/services/FAKEYARN/configuration/yarn-site.xml
@@ -747,13 +747,13 @@
   </property>
   <property>
     <name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name>
-    <value>{{stack_root}}/${hdp.version}/spark/aux/*</value>
+    <value>{{stack_root}}/{{spark_version}}/spark/aux/*</value>
     <description>The auxiliary service classpath to use for Spark</description>
     <on-ambari-upgrade add="false"/>
   </property>
   <property>
     <name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name>
-    <value>{{stack_root}}/${hdp.version}/spark2/aux/*</value>
+    <value>{{stack_root}}/{{spark2_version}}/spark2/aux/*</value>
     <description>The auxiliary service classpath to use for Spark 2</description>
     <on-ambari-upgrade add="false"/>
   </property>

http://git-wip-us.apache.org/repos/asf/ambari/blob/3fa1289e/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
index 407e78d..cc6bd9d 100644
--- a/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.5/common/test_stack_advisor.py
@@ -1206,7 +1206,7 @@ class TestHDP25StackAdvisor(TestCase):
     self.assertEquals(configurations['hive-interactive-site']['properties']['hive.llap.daemon.queue.name'],
'default')
     self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classes'],
                       'org.apache.tez.dag.history.logging.ats.TimelineCachePluginImpl,org.apache.spark.deploy.history.yarn.plugin.SparkATSPlugin')
-    self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath'],
'/usr/hdp/${hdp.version}/spark/hdpLib/*')
+    self.assertEquals(configurations['yarn-site']['properties']['yarn.timeline-service.entity-group-fs-store.group-id-plugin-classpath'],
'/usr/hdp/{{spark_version}}/spark/hdpLib/*')
     self.assertTrue('hive-interactive-env' not in configurations)
     self.assertTrue('property_attributes' not in configurations)
 


Mime
View raw message