ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject ambari git commit: AMBARI-11049. Some Hadoop Directory Parameters Are Wrong On Runnings Processes After Upgrade (ncole)
Date Mon, 11 May 2015 21:45:43 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 66a4bfb26 -> 3da48c232


AMBARI-11049. Some Hadoop Directory Parameters Are Wrong On Runnings Processes After Upgrade
(ncole)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3da48c23
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3da48c23
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3da48c23

Branch: refs/heads/trunk
Commit: 3da48c232f08dea8f536356e0b0b8f6e25a0bf2b
Parents: 66a4bfb
Author: Nate Cole <ncole@hortonworks.com>
Authored: Mon May 11 13:31:13 2015 -0400
Committer: Nate Cole <ncole@hortonworks.com>
Committed: Mon May 11 17:43:46 2015 -0400

----------------------------------------------------------------------
 .../libraries/functions/conf_select.py          | 57 ++++++++++++---
 .../1.6.1.2.2.0/package/scripts/params.py       |  3 +-
 .../0.5.0.2.1/package/scripts/params_linux.py   |  5 +-
 .../0.96.0.2.0/package/scripts/params_linux.py  |  3 +-
 .../2.1.0.2.0/package/scripts/params_linux.py   |  9 +--
 .../0.12.0.2.0/package/scripts/params_linux.py  |  2 +-
 .../0.12.0.2.0/package/scripts/status_params.py |  1 +
 .../MAHOUT/1.0.0.2.3/package/scripts/params.py  |  2 +-
 .../4.0.0.2.0/package/scripts/params_linux.py   |  8 +--
 .../0.12.0.2.0/package/scripts/params_linux.py  |  3 +-
 .../SPARK/1.2.0.2.2/package/scripts/params.py   |  2 +-
 .../0.4.0.2.1/package/scripts/params_linux.py   |  3 +-
 .../2.1.0.2.0/package/scripts/params_linux.py   | 10 +--
 .../2.0.6/hooks/after-INSTALL/scripts/params.py |  7 +-
 .../2.0.6/hooks/before-ANY/scripts/params.py    |  3 +-
 .../2.0.6/hooks/before-START/scripts/params.py  | 10 ++-
 .../python/stacks/2.0.6/HDFS/test_namenode.py   | 75 ++++++++++++++++++++
 17 files changed, 149 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
index 45888d7..63b474f 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/conf_select.py
@@ -18,13 +18,21 @@ limitations under the License.
 
 """
 
-__all__ = ["select", "create"]
+__all__ = ["select", "create", "get_hadoop_conf_dir", "get_hadoop_dir"]
 
 import version
 from resource_management.core import shell
+from resource_management.core.exceptions import Fail
 from resource_management.libraries.script.script import Script
 
 TEMPLATE = "conf-select {0} --package {1} --stack-version {2} --conf-version 0"
+HADOOP_DIR_TEMPLATE = "/usr/hdp/{0}/{1}/{2}"
+HADOOP_DIR_DEFAULTS = {
+  "libexec": "/usr/lib/hadoop/libexec",
+  "sbin": "/usr/lib/hadoop/sbin",
+  "bin": "/usr/bin",
+  "lib": "/usr/lib/hadoop/lib"
+}
 
 def _valid(stack_name, package, ver):
   if stack_name != "HDP":
@@ -35,6 +43,17 @@ def _valid(stack_name, package, ver):
 
   return True
 
+def _is_upgrade():
+  from resource_management.libraries.functions.default import default
+  direction = default("/commandParams/upgrade_direction", None)
+  stack_name = default("/hostLevelParams/stack_name", None)
+  ver = default("/commandParams/version", None)
+
+  if direction and stack_name and ver:
+    return (stack_name, ver)
+
+  return None
+
 def create(stack_name, package, version):
   """
   Creates a config version for the specified package
@@ -76,22 +95,40 @@ def get_hadoop_conf_dir():
       the configs are written in the correct place
   """
 
-  config = Script.get_config()
   hadoop_conf_dir = "/etc/hadoop/conf"
 
   if Script.is_hdp_stack_greater_or_equal("2.2"):
-    from resource_management.libraries.functions.default import default
-
     hadoop_conf_dir = "/usr/hdp/current/hadoop-client/conf"
 
-    direction = default("/commandParams/upgrade_direction", None)
-    ver = default("/commandParams/version", None)
-    stack_name = default("/hostLevelParams/stack_name", None)
+    res = _is_upgrade()
 
-    if direction and ver and stack_name and Script.is_hdp_stack_greater_or_equal("2.3"):
-      select(stack_name, "hadoop", ver)
-      hadoop_conf_dir = "/usr/hdp/{0}/hadoop/conf".format(ver)
+    if res is not None and Script.is_hdp_stack_greater_or_equal("2.3"):
+      select(res[0], "hadoop", res[1])
+      hadoop_conf_dir = "/usr/hdp/{0}/hadoop/conf".format(res[1])
 
   return hadoop_conf_dir
 
+def get_hadoop_dir(target):
+  """
+  Return the hadoop shared directory in the following override order
+  1. Use default for 2.1 and lower
+  2. If 2.2 and higher, use /usr/hdp/current/hadoop-client/{target}
+  3. If 2.2 and higher AND for an upgrade, use /usr/hdp/<version>/hadoop/{target}
+  :target: the target directory
+  """
+
+  if not target in HADOOP_DIR_DEFAULTS:
+    raise Fail("Target {0} not defined".format(target))
+
+  hadoop_dir = HADOOP_DIR_DEFAULTS[target]
+
+  if Script.is_hdp_stack_greater_or_equal("2.2"):
+    hadoop_dir = HADOOP_DIR_TEMPLATE.format("current", "hadoop-client", target)
+
+    res = _is_upgrade()
+
+    if res is not None:
+      hadoop_dir = HADOOP_DIR_TEMPLATE.format(res[1], "hadoop", target)
+
+  return hadoop_dir
     

http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
index 3d2d36d..ff5a874 100644
--- a/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ACCUMULO/1.6.1.2.2.0/package/scripts/params.py
@@ -46,8 +46,7 @@ conf_dir = status_params.conf_dir
 server_conf_dir = status_params.server_conf_dir
 
 # service locations
-hadoop_prefix = "/usr/hdp/current/hadoop-client"
-hadoop_bin_dir = format("{hadoop_prefix}/bin")
+hadoop_bin_dir = conf_select.get_hadoop_dir("bin")
 zookeeper_home = "/usr/hdp/current/zookeeper-client"
 
 # the configuration direction for HDFS/YARN/MapR is the hadoop config

http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
index 27ced1d..9a5fdb5 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
@@ -18,6 +18,7 @@ limitations under the License.
 """
 import status_params
 
+from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import format
 from resource_management.libraries.functions.version import format_hdp_stack_version
 from resource_management.libraries.functions.default import default
@@ -37,8 +38,9 @@ hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 etc_prefix_dir = "/etc/falcon"
 
 # hadoop params
+hadoop_bin_dir = conf_select.get_hadoop_dir("bin")
+
 if Script.is_hdp_stack_greater_or_equal("2.2"):
-  hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
 
   # if this is a server action, then use the server binaries; smoke tests
   # use the client binaries
@@ -53,7 +55,6 @@ if Script.is_hdp_stack_greater_or_equal("2.2"):
   falcon_webapp_dir = format('/usr/hdp/current/{falcon_root}/webapp')
   falcon_home = format('/usr/hdp/current/{falcon_root}')
 else:
-  hadoop_bin_dir = "/usr/bin"
   falcon_webapp_dir = '/var/lib/falcon/webapp'
   falcon_home = '/usr/lib/falcon'
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
index aec8e23..8f23bb2 100644
--- a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py
@@ -48,7 +48,7 @@ stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 
 # hadoop default parameters
-hadoop_bin_dir = "/usr/bin"
+hadoop_bin_dir = conf_select.get_hadoop_dir("bin")
 hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
 daemon_script = "/usr/lib/hbase/bin/hbase-daemon.sh"
 region_mover = "/usr/lib/hbase/bin/region_mover.rb"
@@ -57,7 +57,6 @@ hbase_cmd = "/usr/lib/hbase/bin/hbase"
 
 # hadoop parameters for 2.2+
 if Script.is_hdp_stack_greater_or_equal("2.2"):
-  hadoop_bin_dir = format("/usr/hdp/current/hadoop-client/bin")
   daemon_script = format('/usr/hdp/current/hbase-client/bin/hbase-daemon.sh')
   region_mover = format('/usr/hdp/current/hbase-client/bin/region_mover.rb')
   region_drainer = format('/usr/hdp/current/hbase-client/bin/draining_servers.rb')

http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
index 82a6351..18e0fa0 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
@@ -62,9 +62,9 @@ secure_dn_ports_are_in_use = False
 
 # hadoop default parameters
 mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
-hadoop_libexec_dir = "/usr/lib/hadoop/libexec"
-hadoop_bin = "/usr/lib/hadoop/sbin"
-hadoop_bin_dir = "/usr/bin"
+hadoop_libexec_dir = conf_select.get_hadoop_dir("libexec")
+hadoop_bin = conf_select.get_hadoop_dir("sbin")
+hadoop_bin_dir = conf_select.get_hadoop_dir("bin")
 hadoop_home = "/usr/lib/hadoop"
 hadoop_secure_dn_user = hdfs_user
 hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
@@ -72,9 +72,6 @@ hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
 # hadoop parameters for 2.2+
 if Script.is_hdp_stack_greater_or_equal("2.2"):
   mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce-client/*"
-  hadoop_libexec_dir = "/usr/hdp/current/hadoop-client/libexec"
-  hadoop_bin = "/usr/hdp/current/hadoop-client/sbin"
-  hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
   hadoop_home = "/usr/hdp/current/hadoop-client"
 
   if not security_enabled:

http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index bd6f70e..ed12a65 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -71,6 +71,7 @@ webhcat_bin_dir = '/usr/lib/hcatalog/sbin'
 # use the directories from status_params as they are already calculated for
 # the correct version of HDP
 hadoop_conf_dir = status_params.hadoop_conf_dir
+hadoop_bin_dir = status_params.hadoop_bin_dir
 webhcat_conf_dir = status_params.webhcat_conf_dir
 hive_conf_dir = status_params.hive_conf_dir
 hive_config_dir = status_params.hive_config_dir
@@ -87,7 +88,6 @@ if Script.is_hdp_stack_greater_or_equal("2.2"):
   hive_specific_configs_supported = True
 
   component_directory = status_params.component_directory
-  hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
   hadoop_home = '/usr/hdp/current/hadoop-client'
   hive_bin = format('/usr/hdp/current/{component_directory}/bin')
   hive_lib = format('/usr/hdp/current/{component_directory}/lib')

http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/status_params.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/status_params.py
index d0acb59..1cf1793 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/status_params.py
@@ -71,6 +71,7 @@ else:
 
   # default configuration directories
   hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+  hadoop_bin_dir = conf_select.get_hadoop_dir("bin")
   webhcat_conf_dir = '/etc/hive-webhcat/conf'
   hive_etc_dir_prefix = "/etc/hive"
   hive_conf_dir = "/etc/hive/conf"

http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/params.py
index 47ce98a..ab0dd04 100644
--- a/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/params.py
@@ -45,7 +45,7 @@ mahout_conf_dir = "/usr/hdp/current/mahout-client/conf"
 mahout_user = config['configurations']['mahout-env']['mahout_user']
 
 #hadoop params
-hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
+hadoop_bin_dir = conf_select.get_hadoop_dir("bin")
 hadoop_home = '/usr/hdp/current/hadoop-client'
 
 # the configuration direction for HDFS/YARN/MapR is the hadoop config

http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
index 6909cd7..9732aab 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
@@ -47,13 +47,11 @@ stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 
 hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+hadoop_bin_dir = conf_select.get_hadoop_dir("bin")
+hadoop_lib_home = conf_select.get_hadoop_dir("lib")
 
 #hadoop params
 if Script.is_hdp_stack_greater_or_equal("2.2"):
-  # start out assuming client libraries
-  hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
-  hadoop_lib_home = "/usr/hdp/current/hadoop-client/lib"
-
   # oozie-server or oozie-client, depending on role
   oozie_root = status_params.component_directory
 
@@ -74,8 +72,6 @@ if Script.is_hdp_stack_greater_or_equal("2.2"):
   hive_conf_dir = format("{conf_dir}/action-conf/hive")
 
 else:
-  hadoop_bin_dir = "/usr/bin"
-  hadoop_lib_home = "/usr/lib/hadoop/lib"
   oozie_lib_dir = "/var/lib/oozie"
   oozie_setup_sh = "/usr/lib/oozie/bin/oozie-setup.sh"
   oozie_webapps_dir = "/var/lib/oozie/oozie-server/webapps/"

http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py
index 93c8c71..53762b6 100644
--- a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py
@@ -40,15 +40,14 @@ version = default("/commandParams/version", None)
 
 # hadoop default parameters
 hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+hadoop_bin_dir = conf_select.get_hadoop_dir("bin")
 pig_conf_dir = "/etc/pig/conf"
-hadoop_bin_dir = "/usr/bin"
 hadoop_home = '/usr'
 pig_bin_dir = ""
 
 # hadoop parameters for 2.2+
 if Script.is_hdp_stack_greater_or_equal("2.2"):
   pig_conf_dir = "/usr/hdp/current/pig-client/conf"
-  hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
   hadoop_home = '/usr/hdp/current/hadoop-client'
   pig_bin_dir = '/usr/hdp/current/pig-client/bin'
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
index 4130472..41bac8a 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
@@ -61,10 +61,10 @@ version = default("/commandParams/version", None)
 
 spark_conf = '/etc/spark/conf'
 hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+hadoop_bin_dir = conf_select.get_hadoop_dir("bin")
 
 if Script.is_hdp_stack_greater_or_equal("2.2"):
   hadoop_home = "/usr/hdp/current/hadoop-client"
-  hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
   spark_conf = format("/usr/hdp/current/{component_directory}/conf")
   spark_log_dir = config['configurations']['spark-env']['spark_log_dir']
   spark_pid_dir = status_params.spark_pid_dir

http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
index 7f17a26..e2aa660 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
@@ -42,7 +42,7 @@ version = default("/commandParams/version", None)
 
 # default hadoop parameters
 hadoop_home = '/usr'
-hadoop_bin_dir = "/usr/bin"
+hadoop_bin_dir = conf_select.get_hadoop_dir("bin")
 hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
 tez_etc_dir = "/etc/tez"
 config_dir = "/etc/tez/conf"
@@ -50,7 +50,6 @@ path_to_tez_examples_jar = "/usr/lib/tez/tez-mapreduce-examples*.jar"
 
 # hadoop parameters for 2.2+
 if Script.is_hdp_stack_greater_or_equal("2.2"):
-  hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
   path_to_tez_examples_jar = "/usr/hdp/{hdp_version}/tez/tez-examples*.jar"
 
 # tez only started linking /usr/hdp/x.x.x.x/tez-client/conf in HDP 2.3+

http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index ee454b6..6e43192 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -60,9 +60,9 @@ version = default("/commandParams/version", None)
 hostname = config['hostname']
 
 # hadoop default parameters
-hadoop_libexec_dir = "/usr/lib/hadoop/libexec"
-hadoop_bin = "/usr/lib/hadoop/sbin"
-hadoop_bin_dir = "/usr/bin"
+hadoop_libexec_dir = conf_select.get_hadoop_dir("libexec")
+hadoop_bin = conf_select.get_hadoop_dir("sbin")
+hadoop_bin_dir = conf_select.get_hadoop_dir("bin")
 hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
 hadoop_yarn_home = '/usr/lib/hadoop-yarn'
 hadoop_mapred2_jar_location = "/usr/lib/hadoop-mapreduce"
@@ -84,10 +84,6 @@ if Script.is_hdp_stack_greater_or_equal("2.2"):
   if command_role in YARN_SERVER_ROLE_DIRECTORY_MAP:
     yarn_role_root = YARN_SERVER_ROLE_DIRECTORY_MAP[command_role]
 
-  hadoop_libexec_dir = "/usr/hdp/current/hadoop-client/libexec"
-  hadoop_bin = "/usr/hdp/current/hadoop-client/sbin"
-  hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
-
   hadoop_mapred2_jar_location = format("/usr/hdp/current/{mapred_role_root}")
   mapred_bin = format("/usr/hdp/current/{mapred_role_root}/sbin")
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py
index 5c6c5bd..3031a9f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py
@@ -18,6 +18,7 @@ limitations under the License.
 """
 
 from ambari_commons.constants import AMBARI_SUDO_BINARY
+from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
 from resource_management import *
 from resource_management.core.system import System
@@ -31,15 +32,13 @@ hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 
 # default hadoop params
 mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
-hadoop_libexec_dir = "/usr/lib/hadoop/libexec"
-hadoop_conf_dir = "/etc/hadoop/conf"
+hadoop_libexec_dir = conf_select.get_hadoop_dir("libexec")
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
 hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"
 
 # HDP 2.2+ params
 if Script.is_hdp_stack_greater_or_equal("2.2"):
   mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce-client/*"
-  hadoop_libexec_dir = "/usr/hdp/current/hadoop-client/libexec"
-  hadoop_conf_dir = "/usr/hdp/current/hadoop-client/conf"
 
   # not supported in HDP 2.2+
   hadoop_conf_empty_dir = None

http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
index 9054133..9d57c18 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py
@@ -72,18 +72,17 @@ def is_secure_port(port):
 
 # hadoop default params
 mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
-hadoop_libexec_dir = "/usr/lib/hadoop/libexec"
 hadoop_home = "/usr/lib/hadoop"
 hadoop_secure_dn_user = hdfs_user
 hadoop_dir = "/etc/hadoop"
 versioned_hdp_root = '/usr/hdp/current'
 hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+hadoop_libexec_dir = conf_select.get_hadoop_dir("libexec")
 hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"
 
 # HDP 2.2+ params
 if Script.is_hdp_stack_greater_or_equal("2.2"):
   mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce-client/*"
-  hadoop_libexec_dir = "/usr/hdp/current/hadoop-client/libexec"
   hadoop_home = "/usr/hdp/current/hadoop-client"
 
   # not supported in HDP 2.2+

http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py
index 059835d..6f5f666 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py
@@ -31,9 +31,10 @@ hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 
 # hadoop default params
 mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
-hadoop_libexec_dir = "/usr/lib/hadoop/libexec"
-hadoop_lib_home = "/usr/lib/hadoop/lib"
-hadoop_bin = "/usr/lib/hadoop/sbin"
+
+hadoop_libexec_dir = conf_select.get_hadoop_dir("libexec")
+hadoop_lib_home = conf_select.get_hadoop_dir("lib")
+hadoop_bin = conf_select.get_hadoop_dir("sbin")
 hadoop_home = '/usr'
 create_lib_snappy_symlinks = True
 hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
@@ -42,9 +43,6 @@ default_topology_script_file_path = "/etc/hadoop/conf/topology_script.py"
 # HDP 2.2+ params
 if Script.is_hdp_stack_greater_or_equal("2.2"):
   mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce-client/*"
-  hadoop_libexec_dir = "/usr/hdp/current/hadoop-client/libexec"
-  hadoop_lib_home = "/usr/hdp/current/hadoop-client/lib"
-  hadoop_bin = "/usr/hdp/current/hadoop-client/sbin"
   hadoop_home = '/usr/hdp/current/hadoop-client'
   create_lib_snappy_symlinks = False
   

http://git-wip-us.apache.org/repos/asf/ambari/blob/3da48c23/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
index 2e7af32..417a163 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py
@@ -1193,6 +1193,81 @@ class TestNamenode(RMFTestCase):
                               )
     self.assertNoMoreResources()
 
+  @patch("resource_management.core.shell.call")
+  def test_pre_rolling_restart_21_and_lower_params(self, call_mock):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/nn_ru_lzo.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    json_content['hostLevelParams']['stack_name'] = 'HDP'
+    json_content['hostLevelParams']['stack_version'] = '2.0'
+
+    mocks_dict = {}
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
+                       classname = "NameNode",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES,
+                       call_mocks = [(0, None), (0, None), (0, None), (0, None), (0, None),
(0, None), (0, None)],
+                       mocks_dict = mocks_dict)
+    import sys
+    self.assertEquals("/etc/hadoop/conf", sys.modules["params"].hadoop_conf_dir)
+    self.assertEquals("/usr/lib/hadoop/libexec", sys.modules["params"].hadoop_libexec_dir)
+    self.assertEquals("/usr/bin", sys.modules["params"].hadoop_bin_dir)
+    self.assertEquals("/usr/lib/hadoop/sbin", sys.modules["params"].hadoop_bin)
+
+  @patch("resource_management.core.shell.call")
+  def test_pre_rolling_restart_22_params(self, call_mock):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/nn_ru_lzo.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.2.0.0-1234'
+    del json_content['commandParams']['version']
+    json_content['hostLevelParams']['stack_name'] = 'HDP'
+    json_content['hostLevelParams']['stack_version'] = '2.2'
+
+    mocks_dict = {}
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
+                       classname = "NameNode",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES,
+                       call_mocks = [(0, None), (0, None), (0, None), (0, None), (0, None),
(0, None), (0, None)],
+                       mocks_dict = mocks_dict)
+    import sys
+    self.assertEquals("/usr/hdp/current/hadoop-client/conf", sys.modules["params"].hadoop_conf_dir)
+    self.assertEquals("/usr/hdp/current/hadoop-client/libexec", sys.modules["params"].hadoop_libexec_dir)
+    self.assertEquals("/usr/hdp/current/hadoop-client/bin", sys.modules["params"].hadoop_bin_dir)
+    self.assertEquals("/usr/hdp/current/hadoop-client/sbin", sys.modules["params"].hadoop_bin)
+
+  @patch("resource_management.core.shell.call")
+  def test_pre_rolling_restart_23_params(self, call_mock):
+    config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/nn_ru_lzo.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    version = '2.3.0.0-1234'
+    json_content['commandParams']['version'] = version
+    json_content['commandParams']['upgrade_direction'] = 'upgrade'
+    json_content['hostLevelParams']['stack_name'] = 'HDP'
+    json_content['hostLevelParams']['stack_version'] = '2.3'
+
+    mocks_dict = {}
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
+                       classname = "NameNode",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES,
+                       call_mocks = [(0, None), (0, None), (0, None), (0, None), (0, None),
(0, None), (0, None)],
+                       mocks_dict = mocks_dict)
+    import sys
+    self.assertEquals("/usr/hdp/2.3.0.0-1234/hadoop/conf", sys.modules["params"].hadoop_conf_dir)
+    self.assertEquals("/usr/hdp/2.3.0.0-1234/hadoop/libexec", sys.modules["params"].hadoop_libexec_dir)
+    self.assertEquals("/usr/hdp/2.3.0.0-1234/hadoop/bin", sys.modules["params"].hadoop_bin_dir)
+    self.assertEquals("/usr/hdp/2.3.0.0-1234/hadoop/sbin", sys.modules["params"].hadoop_bin)
+
+
 
 class Popen_Mock:
   return_value = 1


Mime
View raw message