Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm
Precedence: bulk
Reply-To: ambari-dev@ambari.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: oleewere@apache.org
To: commits@ambari.apache.org
Date: Wed, 29 Mar 2017 13:57:46 -0000
Message-Id: <fc480965a77441f2880cf7e2fb6e6428@git.apache.org>
In-Reply-To: <4d60496393ea4efdbd9a6f61570bbe9d@git.apache.org>
References: <4d60496393ea4efdbd9a6f61570bbe9d@git.apache.org>
Subject: [38/67] [abbrv] ambari git commit: AMBARI-20400 Yarn should not copy
 Tez and Slider tar ball if Tez and Sliders are not installed on the cluster
 (dili)
archived-at: Wed, 29 Mar 2017 13:57:17 -0000

AMBARI-20400 Yarn should not copy Tez and Slider tar ball if Tez and Sliders are not installed on the cluster (dili)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3ab4c8d2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3ab4c8d2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3ab4c8d2

Branch: refs/heads/branch-dev-logsearch
Commit: 3ab4c8d2ae0a7f28bc73d8438f79d2f18f96cc01
Parents: c5ccb1a
Author: Di Li <dili@apache.org>
Authored: Mon Mar 27 15:39:33 2017 -0400
Committer: Di Li <dili@apache.org>
Committed: Mon Mar 27 15:39:33 2017 -0400

----------------------------------------------------------------------
 .../libraries/functions/copy_tarball.py         | 24 ++++++++++++++++++-
 .../python/stacks/2.0.6/configs/default.json    | 25 ++++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/3ab4c8d2/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
index 31a9be4..63b6926 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
@@ -64,6 +64,18 @@ TARBALL_MAP = {
              "/{0}/apps/{1}/spark2/spark2-{0}-yarn-archive.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN))
 }
 
+SERVICE_MAP = {
+  "slider": "SLIDER",
+  "tez": "TEZ_CLIENT",
+  "pig": "PIG",
+  "sqoop": "SQOOP_CLIENT",
+  "hive": "HIVE_CLIENT",
+  "mapreduce": "HADOOP_CLIENT",
+  "hadoop_streaming": "MAPREDUCE2_CLIENT",
+  "tez_hive2": "HIVE_CLIENT",
+  "spark": "SPARK_CLIENT",
+  "spark2": "SPARK2_CLIENT"
+}
 
 def get_sysprep_skip_copy_tarballs_hdfs():
   import params
@@ -199,7 +211,7 @@ def _get_single_version_from_stack_select():
 
 
 def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=None, custom_dest_file=None, force_execute=False,
-                 use_upgrading_version_during_upgrade=True, replace_existing_files=False, skip=False):
+                 use_upgrading_version_during_upgrade=True, replace_existing_files=False, skip=False, skip_component_check=False):
   """
   :param name: Tarball name, e.g., tez, hive, pig, sqoop.
   :param user_group: Group to own the directory.
@@ -210,6 +222,8 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=Non
   :param force_execute: If true, will execute the HDFS commands immediately, otherwise, will defer to the calling function.
   :param use_upgrading_version_during_upgrade: If true, will use the version going to during upgrade. Otherwise, use the CURRENT (source) version.
   :param skip: If true, tarballs will not be copied as the cluster deployment uses prepped VMs.
+  :param skip_component_check: If true, will skip checking if a given component is installed on the node for a file under its dir to be copied.
+                               This is in case the file is not mapped to a component but rather to a specific location (JDK jar, Ambari jar, etc).
   :return: Will return True if successful, otherwise, False.
   """
   import params
@@ -226,6 +240,14 @@ def copy_to_hdfs(name, user_group, owner, file_mode=0444, custom_source_file=Non
     Logger.warning("Skipping copying {0} to {1} for {2} as it is a sys prepped host.".format(str(source_file), str(dest_file), str(name)))
     return True
 
+  if not skip_component_check:
+    #Use components installed on the node to check if a file can be copied into HDFS
+    local_components = default("/localComponents", [])
+    component = SERVICE_MAP.get(name)
+    if component not in local_components:
+      Logger.info("{0} is not installed on the host. Skip copying {1}".format(component, source_file))
+      return False
+
   Logger.info("Source file: {0} , Dest file in HDFS: {1}".format(source_file, dest_file))
 
   if not os.path.exists(source_file):

http://git-wip-us.apache.org/repos/asf/ambari/blob/3ab4c8d2/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
index fa7419f..ceb0ca0 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
@@ -1,4 +1,26 @@
 {
+    "localComponents": [
+        "NAMENODE",
+        "SECONDARY_NAMENODE",
+        "ZOOKEEPER_SERVER",
+        "DATANODE",
+        "HDFS_CLIENT",
+        "ZOOKEEPER_CLIENT",
+        "RESOURCEMANAGER",
+        "HISTORYSERVER",
+        "NODEMANAGER",
+        "YARN_CLIENT",
+        "MAPREDUCE2_CLIENT",
+        "SLIDER",
+        "PIG",
+        "SQOOP_CLIENT",
+        "HIVE_CLIENT",
+        "TEZ_CLIENT",
+        "HADOOP_CLIENT",
+        "HIVE_CLIENT",
+        "SPARK_CLIENT",
+        "SPARK2_CLIENT"
+    ],
     "roleCommand": "SERVICE_CHECK",
     "clusterName": "c1",
     "hostname": "c6401.ambari.apache.org",
@@ -544,6 +566,9 @@
             "tez.staging-dir": "/tmp/${user.name}/staging",
             "tez.am.am-rm.heartbeat.interval-ms.max": "250"
         },
+        "slider-env": {
+            "content": "export JAVA_HOME={{java64_home}}\nexport HADOOP_CONF_DIR={{hadoop_conf_dir}}"
+        },
         "yarn-env": {
             "yarn_pid_dir_prefix": "/var/run/hadoop-yarn",
             "apptimelineserver_heapsize": "1024",