ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject ambari git commit: AMBARI-22486 - Conditionally Rebuild MapReduce and Tez Tarballs with LZO if Enabled (jonathanhurley)
Date Wed, 22 Nov 2017 01:32:25 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-feature-AMBARI-22457 519f52706 -> 58c7f784a


AMBARI-22486 - Conditionally Rebuild MapReduce and Tez Tarballs with LZO if Enabled (jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/58c7f784
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/58c7f784
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/58c7f784

Branch: refs/heads/branch-feature-AMBARI-22457
Commit: 58c7f784abe9b6f05a1358c14a8f94afc6725f79
Parents: 519f527
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Mon Nov 20 14:23:20 2017 -0500
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Tue Nov 21 20:32:13 2017 -0500

----------------------------------------------------------------------
 .../libraries/functions/copy_tarball.py         | 99 +++++++++++++++++++-
 .../TEZ/0.4.0.2.1/package/scripts/tez.py        |  5 +-
 .../configuration-mapred/mapred-site.xml        |  2 +-
 .../2.2/services/TEZ/configuration/tez-site.xml |  4 +-
 .../YARN/configuration-mapred/mapred-site.xml   |  2 +-
 .../stacks/HDP/2.6/upgrades/config-upgrade.xml  |  6 +-
 6 files changed, 106 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/58c7f784/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
index b05c97c..bf0701c 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py
@@ -28,6 +28,8 @@ from contextlib import closing
 
 from resource_management.libraries.script.script import Script
 from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.functions import component_version
+from resource_management.libraries.functions import lzo_utils
 from resource_management.libraries.functions.default import default
 from resource_management.core import shell
 from resource_management.core import sudo
@@ -45,6 +47,9 @@ STACK_VERSION_PATTERN = "{{ stack_version }}"
 def _prepare_tez_tarball():
   """
   Prepares the Tez tarball by adding the Hadoop native libraries found in the mapreduce tarball.
+  It's very important to use the version of mapreduce which matches tez here.
+  Additionally, this will also copy native LZO to the tez tarball if LZO is enabled and the
+  GPL license has been accepted.
   :return:  the full path of the newly created tez tarball to use
   """
   import tempfile
@@ -79,12 +84,31 @@ def _prepare_tez_tarball():
   if not os.path.exists(tez_lib_dir):
     raise Fail("Unable to seed the Tez tarball with native libraries since the target Tez
lib directory {0} does not exist".format(tez_lib_dir))
 
-  # ensure that the tez/lib directory is readable by non-root (which it typically is not)
-  sudo.chmod(tez_lib_dir, 0755)
-
   # copy native libraries from hadoop to tez
   Execute(("cp", "-a", hadoop_lib_native_dir, tez_lib_dir), sudo = True)
 
+  # if enabled, LZO GPL libraries must be copied as well
+  if lzo_utils.should_install_lzo():
+    stack_root = Script.get_stack_root()
+    tez_version = component_version.get_component_repository_version("TEZ")
+    hadoop_lib_native_lzo_dir = os.path.join(stack_root, tez_version, "hadoop", "lib", "native")
+
+    if not sudo.path_isdir(hadoop_lib_native_lzo_dir):
+      Logger.warning("Unable to located native LZO libraries at {0}, falling back to hadoop
home".format(hadoop_lib_native_lzo_dir))
+      hadoop_lib_native_lzo_dir = os.path.join(stack_root, "current", "hadoop-client", "lib",
"native")
+
+    if not sudo.path_isdir(hadoop_lib_native_lzo_dir):
+      raise Fail("Unable to seed the Tez tarball with native libraries since LZO is enabled
but the native LZO libraries could not be found at {0}".format(hadoop_lib_native_lzo_dir))
+
+    Execute(("cp", "-a", hadoop_lib_native_lzo_dir, tez_lib_dir), sudo = True)
+
+
+  # ensure that the tez/lib directory is readable by non-root (which it typically is not)
+  Directory(tez_lib_dir,
+    mode = 0755,
+    cd_access = 'a',
+    recursive_ownership = True)
+
   # create the staging directory so that non-root agents can write to it
   tez_native_tarball_staging_dir = os.path.join(temp_dir, "tez-native-tarball-staging")
   if not os.path.exists(tez_native_tarball_staging_dir):
@@ -111,6 +135,72 @@ def _prepare_tez_tarball():
   return tez_tarball_with_native_lib
 
 
+def _prepare_mapreduce_tarball():
+  """
+  Prepares the mapreduce tarball by including the native LZO libraries if necessary. If LZO
is
+  not enabled or has not been opted-in, then this will do nothing and return the original
+  tarball to upload to HDFS.
+  :return:  the full path of the newly created mapreduce tarball to use or the original path
+  if no changes were made
+  """
+  # get the mapreduce tarball to crack open and add LZO libraries to
+  _, mapreduce_source_file, _, _ = get_tarball_paths("mapreduce")
+
+  if not lzo_utils.should_install_lzo():
+    return mapreduce_source_file
+
+  Logger.info("Preparing the mapreduce tarball with native LZO libraries...")
+
+  temp_dir = Script.get_tmp_dir()
+
+  # create the temp staging directories ensuring that non-root agents using tarfile can work
with them
+  mapreduce_temp_dir = tempfile.mkdtemp(prefix="mapreduce-tarball-", dir=temp_dir)
+  sudo.chmod(mapreduce_temp_dir, 0777)
+
+  # calculate the source directory for LZO
+  hadoop_lib_native_source_dir = os.path.join(os.path.dirname(mapreduce_source_file), "lib",
"native")
+  if not sudo.path_exists(hadoop_lib_native_source_dir):
+    raise Fail("Unable to seed the mapreduce tarball with native LZO libraries since the
source Hadoop native lib directory {0} does not exist".format(hadoop_lib_native_source_dir))
+
+  Logger.info("Extracting {0} to {1}".format(mapreduce_source_file, mapreduce_temp_dir))
+  tar_archive.extract_archive(mapreduce_source_file, mapreduce_temp_dir)
+
+  mapreduce_lib_dir = os.path.join(mapreduce_temp_dir, "hadoop", "lib")
+
+  # copy native libraries from source hadoop to target
+  Execute(("cp", "-af", hadoop_lib_native_source_dir, mapreduce_lib_dir), sudo = True)
+
+  # ensure that the hadoop/lib/native directory is readable by non-root (which it typically
is not)
+  Directory(mapreduce_lib_dir,
+    mode = 0755,
+    cd_access = 'a',
+    recursive_ownership = True)
+
+  # create the staging directory so that non-root agents can write to it
+  mapreduce_native_tarball_staging_dir = os.path.join(temp_dir, "mapreduce-native-tarball-staging")
+  if not os.path.exists(mapreduce_native_tarball_staging_dir):
+    Directory(mapreduce_native_tarball_staging_dir,
+      mode = 0777,
+      cd_access = 'a',
+      create_parents = True,
+      recursive_ownership = True)
+
+  mapreduce_tarball_with_native_lib = os.path.join(mapreduce_native_tarball_staging_dir,
"mapreduce-native.tar.gz")
+  Logger.info("Creating a new mapreduce tarball at {0}".format(mapreduce_tarball_with_native_lib))
+
+  # tar up mapreduce, making sure to specify nothing for the arcname so that it does not
include an absolute path
+  with closing(tarfile.open(mapreduce_tarball_with_native_lib, "w:gz")) as new_tarball:
+    new_tarball.add(mapreduce_temp_dir, arcname = os.path.sep)
+
+  # ensure that the tarball can be read and uploaded
+  sudo.chmod(mapreduce_tarball_with_native_lib, 0744)
+
+  # cleanup
+  sudo.rmtree(mapreduce_temp_dir)
+
+  return mapreduce_tarball_with_native_lib
+
+
 # TODO, in the future, each stack can define its own mapping of tarballs
 # inside the stack definition directory in some sort of xml file.
 # PLEASE DO NOT put this in cluster-env since it becomes much harder to change,
@@ -163,7 +253,8 @@ TARBALL_MAP = {
   "mapreduce": {
     "dirs": ("{0}/{1}/hadoop/mapreduce.tar.gz".format(STACK_ROOT_PATTERN, STACK_VERSION_PATTERN),
                 "/{0}/apps/{1}/mapreduce/mapreduce.tar.gz".format(STACK_NAME_PATTERN, STACK_VERSION_PATTERN)),
-    "service": "MAPREDUCE2"
+    "service": "MAPREDUCE2",
+    "prepare_function": _prepare_mapreduce_tarball
   },
 
   "spark": {

http://git-wip-us.apache.org/repos/asf/ambari/blob/58c7f784/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py
b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py
index dfa6501..35647e4 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py
@@ -23,8 +23,8 @@ import os
 
 # Local Imports
 from resource_management.core.resources.system import Directory, File
+from resource_management.libraries.functions import lzo_utils
 from resource_management.libraries.resources.xml_config import XmlConfig
-from resource_management.libraries.functions.format import format
 from resource_management.core.source import InlineTemplate
 from ambari_commons import OSConst
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
@@ -37,6 +37,9 @@ def tez(config_dir):
   """
   import params
 
+  # ensure that matching LZO libraries are installed for Tez
+  lzo_utils.install_lzo_if_needed()
+
   Directory(params.tez_etc_dir, mode=0755)
 
   Directory(config_dir,

http://git-wip-us.apache.org/repos/asf/ambari/blob/58c7f784/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
index 3438c45..398c9d7 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-site.xml
@@ -438,7 +438,7 @@
   </property>
   <property>
     <name>mapreduce.admin.user.env</name>
-    <value>LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:{{hadoop_lib_home}}/native/Linux-{{architecture}}-64</value>
+    <value>LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:./mr-framework/hadoop/lib/native/Linux-{{architecture}}-64:{{hadoop_lib_home}}/native/Linux-{{architecture}}-64</value>
     <description>
       Additional execution environment entries for map and reduce task processes.
       This is not an additive property. You must preserve the original value if

http://git-wip-us.apache.org/repos/asf/ambari/blob/58c7f784/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
index 4ffb7a4..5513ab1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/TEZ/configuration/tez-site.xml
@@ -78,7 +78,7 @@
   </property>
   <property>
     <name>tez.am.launch.env</name>
-    <value>LD_LIBRARY_PATH=./tezlib/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value>
+    <value>LD_LIBRARY_PATH=./tezlib/lib/native:./tezlib/lib/native/Linux-{{architecture}}-64:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value>
     <description>
         Additional execution environment entries for tez. This is not an additive property.
You must preserve the original value if
         you want to have access to native libraries.
@@ -124,7 +124,7 @@
   </property>
   <property>
     <name>tez.task.launch.env</name>
-    <value>LD_LIBRARY_PATH=./tezlib/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value>
+    <value>LD_LIBRARY_PATH=./tezlib/lib/native:./tezlib/lib/native/Linux-{{architecture}}-64:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value>
     <description>
       Additional execution environment entries for tez. This is not an additive property.
You must preserve the original value if
       you want to have access to native libraries.

http://git-wip-us.apache.org/repos/asf/ambari/blob/58c7f784/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml
index 084e912..099e388 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/YARN/configuration-mapred/mapred-site.xml
@@ -20,7 +20,7 @@
 <configuration xmlns:xi="http://www.w3.org/2001/XInclude" supports_final="true">
   <property>
     <name>mapreduce.admin.user.env</name>
-    <value>LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value>
+    <value>LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:./mr-framework/hadoop/lib/native/Linux-{{architecture}}-64:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64</value>
     <description>
       Additional execution environment entries for map and reduce task processes.
       This is not an additive property. You must preserve the original value if

http://git-wip-us.apache.org/repos/asf/ambari/blob/58c7f784/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
index 4538072..5c672ba 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.6/upgrades/config-upgrade.xml
@@ -269,8 +269,8 @@
         <changes>
           <definition xsi:type="configure" id="hdp_2_6_tez_tarball_ld_library">
             <type>tez-site</type>
-            <set key="tez.am.launch.env" value="LD_LIBRARY_PATH=./tezlib/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64"/>
-            <set key="tez.task.launch.env" value="LD_LIBRARY_PATH=./tezlib/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64"/>
+            <set key="tez.am.launch.env" value="LD_LIBRARY_PATH=./tezlib/lib/native:./tezlib/lib/native/Linux-{{architecture}}-64:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64"/>
+            <set key="tez.task.launch.env" value="LD_LIBRARY_PATH=./tezlib/lib/native:./tezlib/lib/native/Linux-{{architecture}}-64:/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-{{architecture}}-64"/>
           </definition>
         </changes>
       </component>
@@ -281,7 +281,7 @@
         <changes>
           <definition xsi:type="configure" id="hdp_2_6_mapreduce_tarball_ld_library">
             <type>mapred-site</type>
-            <set key="mapreduce.admin.user.env" value="LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:{{hadoop_lib_home}}/native/Linux-{{architecture}}-64"/>
+            <set key="mapreduce.admin.user.env" value="LD_LIBRARY_PATH=./mr-framework/hadoop/lib/native:./mr-framework/hadoop/lib/native/Linux-{{architecture}}-64:{{hadoop_lib_home}}/native/Linux-{{architecture}}-64"/>
           </definition>
         </changes>
       </component>


Mime
View raw message