ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-15053: Parameterize distro-specific stack information for YARN/MR (Juanjo Marron via dili)
Date Mon, 29 Feb 2016 15:29:14 GMT
Repository: ambari
Updated Branches:
  refs/heads/AMBARI-13364 3e02731ab -> 7a77bfe52


AMBARI-15053: Parameterize distro-specific stack information for YARN/MR (Juanjo Marron via
dili)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7a77bfe5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7a77bfe5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7a77bfe5

Branch: refs/heads/AMBARI-13364
Commit: 7a77bfe52060920678d50aebc6d7f1e24ef01858
Parents: 3e02731
Author: Di Li <dili@apache.org>
Authored: Mon Feb 29 10:28:27 2016 -0500
Committer: Di Li <dili@apache.org>
Committed: Mon Feb 29 10:28:27 2016 -0500

----------------------------------------------------------------------
 .../scripts/application_timeline_server.py      |  5 ++--
 .../2.1.0.2.0/package/scripts/historyserver.py  |  9 ++++----
 .../package/scripts/mapreduce2_client.py        |  5 ++--
 .../2.1.0.2.0/package/scripts/nodemanager.py    |  5 ++--
 .../2.1.0.2.0/package/scripts/params_linux.py   | 24 +++++++++++---------
 .../package/scripts/resourcemanager.py          |  5 ++--
 .../2.1.0.2.0/package/scripts/service_check.py  |  6 ++---
 .../YARN/2.1.0.2.0/package/scripts/yarn.py      |  6 ++---
 .../2.1.0.2.0/package/scripts/yarn_client.py    |  5 ++--
 9 files changed, 39 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py
index 7644225..61e4d66 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/application_timeline_server.py
@@ -63,14 +63,15 @@ class ApplicationTimelineServerWindows(ApplicationTimelineServer):
 @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
 class ApplicationTimelineServerDefault(ApplicationTimelineServer):
   def get_stack_to_component(self):
-    return {"HDP": "hadoop-yarn-timelineserver"}
+    import params
+    return {params.stack_name: "hadoop-yarn-timelineserver"}
 
   def pre_upgrade_restart(self, env, upgrade_type=None):
     Logger.info("Executing Stack Upgrade pre-restart")
     import params
     env.set_params(params)
 
-    if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0')
>= 0:
+    if params.version and compare_versions(format_hdp_stack_version(params.version), params.stack_version_ru_support)
>= 0:
       conf_select.select(params.stack_name, "hadoop", params.version)
       hdp_select.select("hadoop-yarn-timelineserver", params.version)
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
index 5d95c5c..c8b1bcd 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/historyserver.py
@@ -70,14 +70,15 @@ class HistoryserverWindows(HistoryServer):
 @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
 class HistoryServerDefault(HistoryServer):
   def get_stack_to_component(self):
-    return {"HDP": "hadoop-mapreduce-historyserver"}
+    import params
+    return {params.stack_name: "hadoop-mapreduce-historyserver"}
 
   def pre_upgrade_restart(self, env, upgrade_type=None):
     Logger.info("Executing Stack Upgrade pre-restart")
     import params
     env.set_params(params)
 
-    if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0')
>= 0:
+    if params.version and compare_versions(format_hdp_stack_version(params.version), params.stack_version_ru_support)
>= 0:
       conf_select.select(params.stack_name, "hadoop", params.version)
       hdp_select.select("hadoop-mapreduce-historyserver", params.version)
       # MC Hammer said, "Can't touch this"
@@ -91,7 +92,7 @@ class HistoryServerDefault(HistoryServer):
     env.set_params(params)
     self.configure(env) # FOR SECURITY
 
-    if params.hdp_stack_version_major and compare_versions(params.hdp_stack_version_major,
'2.2.0.0') >= 0:
+    if params.stack_version_formatted and compare_versions(params.stack_version_formatted,
params.stack_version_ru_support) >= 0:
       # MC Hammer said, "Can't touch this"
       resource_created = copy_to_hdfs(
         "mapreduce",
@@ -111,7 +112,7 @@ class HistoryServerDefault(HistoryServer):
       if resource_created:
         params.HdfsResource(None, action="execute")
     else:
-      # In HDP 2.1, tez.tar.gz was copied to a different folder in HDFS.
+      # In version < stack_version_ru_support tez.tar.gz was copied to a different folder
in HDFS.
       install_tez_jars()
 
     service('historyserver', action='start', serviceName='mapreduce')

http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapreduce2_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapreduce2_client.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapreduce2_client.py
index 7ceadf0..085ad10 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapreduce2_client.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/mapreduce2_client.py
@@ -50,13 +50,14 @@ class MapReduce2ClientWindows(MapReduce2Client):
 @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
 class MapReduce2ClientDefault(MapReduce2Client):
   def get_stack_to_component(self):
-    return {"HDP": "hadoop-client"}
+    import params
+    return {params.stack_name: "hadoop-client"}
 
   def pre_upgrade_restart(self, env, upgrade_type=None):
     import params
     env.set_params(params)
 
-    if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0')
>= 0:
+    if params.version and compare_versions(format_hdp_stack_version(params.version), params.stack_version_ru_support)
>= 0:
       conf_select.select(params.stack_name, "hadoop", params.version)
       hdp_select.select("hadoop-client", params.version)
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
index d508d55..9ea7713 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager.py
@@ -65,14 +65,15 @@ class NodemanagerWindows(Nodemanager):
 @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
 class NodemanagerDefault(Nodemanager):
   def get_stack_to_component(self):
-    return {"HDP": "hadoop-yarn-nodemanager"}
+    import params
+    return {params.stack_name: "hadoop-yarn-nodemanager"}
 
   def pre_upgrade_restart(self, env, upgrade_type=None):
     Logger.info("Executing NodeManager Stack Upgrade pre-restart")
     import params
     env.set_params(params)
 
-    if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0')
>= 0:
+    if params.version and compare_versions(format_hdp_stack_version(params.version), params.stack_version_ru_support)
>= 0:
       conf_select.select(params.stack_name, "hadoop", params.version)
       hdp_select.select("hadoop-yarn-nodemanager", params.version)
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index f0b6927..c57a4d9 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -33,7 +33,7 @@ from resource_management.libraries import functions
 import status_params
 
 # a map of the Ambari role to the component name
-# for use with /usr/hdp/current/<component>
+# for use with <stack_dir>/current/<component>
 MAPR_SERVER_ROLE_DIRECTORY_MAP = {
   'HISTORYSERVER' : 'hadoop-mapreduce-historyserver',
   'MAPREDUCE2_CLIENT' : 'hadoop-mapreduce-client',
@@ -51,11 +51,13 @@ config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
 stack_name = default("/hostLevelParams/stack_name", None)
+stack_dir = config['configurations']['cluster-env']['stack_dir']
 
 # This is expected to be of the form #.#.#.#
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
-hdp_stack_version_major = format_hdp_stack_version(stack_version_unformatted)
-hdp_stack_version = functions.get_hdp_version('hadoop-yarn-resourcemanager')
+stack_version_formatted = format_hdp_stack_version(stack_version_unformatted)
+stack_version = functions.get_hdp_version('hadoop-yarn-resourcemanager')
+stack_version_ru_support = config['configurations']['cluster-env']['stack_version_ru_support']
 
 # New Cluster Stack Version that is defined during the RESTART of a Stack Upgrade.
 # It cannot be used during the initial Cluser Install because the version is not yet known.
@@ -75,8 +77,8 @@ yarn_bin = "/usr/lib/hadoop-yarn/sbin"
 yarn_container_bin = "/usr/lib/hadoop-yarn/bin"
 hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir")
 
-# hadoop parameters for 2.2+
-if Script.is_hdp_stack_greater_or_equal("2.2"):
+# hadoop parameters for stack_version_ru_support+
+if Script.is_hdp_stack_greater_or_equal(stack_version_ru_support):
   # MapR directory root
   mapred_role_root = "hadoop-mapreduce-client"
   command_role = default("/role", "")
@@ -88,14 +90,14 @@ if Script.is_hdp_stack_greater_or_equal("2.2"):
   if command_role in YARN_SERVER_ROLE_DIRECTORY_MAP:
     yarn_role_root = YARN_SERVER_ROLE_DIRECTORY_MAP[command_role]
 
-  hadoop_mapred2_jar_location = format("/usr/hdp/current/{mapred_role_root}")
-  mapred_bin = format("/usr/hdp/current/{mapred_role_root}/sbin")
+  hadoop_mapred2_jar_location = format("{stack_dir}/current/{mapred_role_root}")
+  mapred_bin = format("{stack_dir}/current/{mapred_role_root}/sbin")
 
-  hadoop_yarn_home = format("/usr/hdp/current/{yarn_role_root}")
-  yarn_bin = format("/usr/hdp/current/{yarn_role_root}/sbin")
-  yarn_container_bin = format("/usr/hdp/current/{yarn_role_root}/bin")
+  hadoop_yarn_home = format("{stack_dir}/current/{yarn_role_root}")
+  yarn_bin = format("{stack_dir}/current/{yarn_role_root}/sbin")
+  yarn_container_bin = format("{stack_dir}/current/{yarn_role_root}/bin")
 
-  # Timeline Service property that was added in 2.2
+  # Timeline Service property that was added in tack_version_ru_support
   ats_leveldb_state_store_dir = config['configurations']['yarn-site']['yarn.timeline-service.leveldb-state-store.path']
 
 # ats 1.5 properties

http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
index ec7799e..27ed804 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
@@ -95,14 +95,15 @@ class ResourcemanagerWindows(Resourcemanager):
 @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
 class ResourcemanagerDefault(Resourcemanager):
   def get_stack_to_component(self):
-    return {"HDP": "hadoop-yarn-resourcemanager"}
+    import params
+    return {params.stack_name: "hadoop-yarn-resourcemanager"}
 
   def pre_upgrade_restart(self, env, upgrade_type=None):
     Logger.info("Executing Stack Upgrade post-restart")
     import params
     env.set_params(params)
 
-    if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0')
>= 0:
+    if params.version and compare_versions(format_hdp_stack_version(params.version), params.stack_version_ru_support)
>= 0:
       conf_select.select(params.stack_name, "hadoop", params.version)
       hdp_select.select("hadoop-yarn-resourcemanager", params.version)
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
index cd4d558..c25c8f4 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py
@@ -85,10 +85,10 @@ class ServiceCheckDefault(ServiceCheck):
     import params
     env.set_params(params)
 
-    if params.hdp_stack_version_major != "" and compare_versions(params.hdp_stack_version_major,
'2.2') >= 0:
-      path_to_distributed_shell_jar = "/usr/hdp/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar"
+    if params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted,
params.stack_version_ru_support) >= 0:
+      path_to_distributed_shell_jar = format("{stack_dir}/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar")
     else:
-      path_to_distributed_shell_jar = "/usr/lib/hadoop-yarn/hadoop-yarn-applications-distributedshell*.jar"
+      path_to_distributed_shell_jar = format("{stack_dir}/hadoop-yarn/hadoop-yarn-applications-distributedshell*.jar")
 
     yarn_distrubuted_shell_check_cmd = format("yarn org.apache.hadoop.yarn.applications.distributedshell.Client
"
                                               "-shell_command ls -num_containers {number_of_nm}
-jar {path_to_distributed_shell_jar}")

http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
index edfbe44..97891de 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
@@ -189,7 +189,7 @@ def yarn(name = None):
 
   # During RU, Core Masters and Slaves need hdfs-site.xml
   # TODO, instead of specifying individual configs, which is susceptible to breaking when
new configs are added,
-  # RU should rely on all available in /usr/hdp/<version>/hadoop/conf
+  # RU should rely on all available in {stack_dir}/<version>/hadoop/conf
   if 'hdfs-site' in params.config['configurations']:
     XmlConfig("hdfs-site.xml",
               conf_dir=params.hadoop_conf_dir,
@@ -252,8 +252,8 @@ def yarn(name = None):
        cd_access="a",
     )
 
-    # if HDP stack is greater than/equal to 2.2, mkdir for state store property (added in
2.2)
-    if (Script.is_hdp_stack_greater_or_equal("2.2")):
+    # if stack is greater than/equal to stack_version_ru_support, mkdir for state store property
(added in stack_version_ru_support)
+    if (Script.is_hdp_stack_greater_or_equal(params.stack_version_ru_support)):
       Directory(params.ats_leveldb_state_store_dir,
        owner=params.yarn_user,
        group=params.user_group,

http://git-wip-us.apache.org/repos/asf/ambari/blob/7a77bfe5/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn_client.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn_client.py
index 0c6115f..8c4c063 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn_client.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn_client.py
@@ -50,13 +50,14 @@ class YarnClientWindows(YarnClient):
 @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
 class YarnClientDefault(YarnClient):
   def get_stack_to_component(self):
-    return {"HDP": "hadoop-client"}
+    import params
+    return {params.stack_name: "hadoop-client"}
 
   def pre_upgrade_restart(self, env, upgrade_type=None):
     import params
     env.set_params(params)
 
-    if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0')
>= 0:
+    if params.version and compare_versions(format_hdp_stack_version(params.version), params.stack_version_ru_support)
>= 0:
       conf_select.select(params.stack_name, "hadoop", params.version)
       hdp_select.select("hadoop-client", params.version)
 


Mime
View raw message