ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [08/24] ambari git commit: AMBARI-22467. YARN, MapReduce2, Hive, and Oozie Should Conditionally Install LZO (aonishuk)
Date Tue, 28 Nov 2017 21:20:27 GMT
AMBARI-22467. YARN, MapReduce2, Hive, and Oozie Should Conditionally Install LZO (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/096fedbd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/096fedbd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/096fedbd

Branch: refs/heads/branch-2.6
Commit: 096fedbd38ef241446ccee5020a62b62fc3f94bf
Parents: f35fd4e
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Mon Nov 20 12:37:22 2017 +0200
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Mon Nov 20 12:37:22 2017 +0200

----------------------------------------------------------------------
 .../libraries/functions/__init__.py             |  2 +-
 .../libraries/functions/get_lzo_packages.py     | 50 -----------
 .../libraries/functions/lzo_utils.py            | 93 ++++++++++++++++++++
 .../libraries/functions/package_conditions.py   |  8 +-
 .../common-services/HDFS/2.1.0.2.0/metainfo.xml | 30 -------
 .../HDFS/2.1.0.2.0/package/scripts/hdfs.py      |  6 +-
 .../2.1.0.2.0/package/scripts/install_params.py | 10 ---
 .../2.1.0.2.0/package/scripts/params_linux.py   |  7 --
 .../HIVE/0.12.0.2.0/package/scripts/hive.py     |  4 +
 .../OOZIE/4.0.0.2.0/package/scripts/oozie.py    | 48 +++++-----
 .../4.0.0.2.0/package/scripts/params_linux.py   |  8 +-
 .../YARN/2.1.0.2.0/package/scripts/yarn.py      |  3 +
 .../stacks/HDP/2.2/services/HDFS/metainfo.xml   | 35 --------
 .../stacks/HDP/2.3/services/HDFS/metainfo.xml   | 30 -------
 .../stacks/2.2/configs/oozie-upgrade.json       |  3 +-
 15 files changed, 131 insertions(+), 206 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
index f144b2d..b907844 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
@@ -39,7 +39,7 @@ from resource_management.libraries.functions.version import *
 from resource_management.libraries.functions.format_jvm_option import *
 from resource_management.libraries.functions.constants import *
 from resource_management.libraries.functions.get_stack_version import *
-from resource_management.libraries.functions.get_lzo_packages import *
+from resource_management.libraries.functions.lzo_utils import *
 from resource_management.libraries.functions.setup_ranger_plugin import *
 from resource_management.libraries.functions.curl_krb_request import *
 from resource_management.libraries.functions.get_bare_principal import *

http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py
b/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py
deleted file mode 100644
index cfbb7d8..0000000
--- a/ambari-common/src/main/python/resource_management/libraries/functions/get_lzo_packages.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-__all__ = ["get_lzo_packages"]
-
-from ambari_commons.os_check import OSCheck
-from resource_management.libraries.functions.stack_features import check_stack_feature
-from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.script.script import Script
-
-# TODO: Make list of lzo packages stack driven
-def get_lzo_packages(stack_version_unformatted):
-  lzo_packages = []
-  script_instance = Script.get_instance()
-  if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12:
-    lzo_packages += ["liblzo2-2", "hadoop-lzo-native"]
-  elif OSCheck.is_redhat_family() or OSCheck.is_suse_family():
-    lzo_packages += ["lzo", "hadoop-lzo-native"]
-  elif OSCheck.is_ubuntu_family():
-    lzo_packages += ["liblzo2-2"]
-
-  if stack_version_unformatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_unformatted):
-    if OSCheck.is_ubuntu_family():
-      lzo_packages += [script_instance.format_package_name("hadooplzo-${stack_version}")
,
-                       script_instance.format_package_name("hadooplzo-${stack_version}-native")]
-    else:
-      lzo_packages += [script_instance.format_package_name("hadooplzo_${stack_version}"),
-                       script_instance.format_package_name("hadooplzo_${stack_version}-native")]
-  else:
-    lzo_packages += ["hadoop-lzo"]
-
-  return lzo_packages

http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
b/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
new file mode 100644
index 0000000..d6d987f
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/lzo_utils.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+__all__ = ["should_install_lzo", "get_lzo_packages", "install_lzo_if_needed"]
+
+from ambari_commons.os_check import OSCheck
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import StackFeature, stack_features
+from resource_management.libraries.script.script import Script
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions.expect import expect
+from resource_management.core.resources.packaging import Package
+
+INSTALLING_LZO_WITHOUT_GPL = "Cannot install LZO.  The GPL license must be explicitly enabled
using 'ambari-server setup' on the Ambari host, then restart the server and try again."
+
+def get_lzo_packages():
+  lzo_packages = []
+  script_instance = Script.get_instance()
+  if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12:
+    lzo_packages += ["liblzo2-2", "hadoop-lzo-native"]
+  elif OSCheck.is_redhat_family() or OSCheck.is_suse_family():
+    lzo_packages += ["lzo", "hadoop-lzo-native"]
+  elif OSCheck.is_ubuntu_family():
+    lzo_packages += ["liblzo2-2"]
+
+
+  stack_version_unformatted = stack_features.get_stack_feature_version(Script.get_config())
# only used to check stack_feature, NOT as package version!
+  if stack_version_unformatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_unformatted):
+    if OSCheck.is_ubuntu_family():
+      lzo_packages += [script_instance.format_package_name("hadooplzo-${stack_version}")
,
+                       script_instance.format_package_name("hadooplzo-${stack_version}-native")]
+    else:
+      lzo_packages += [script_instance.format_package_name("hadooplzo_${stack_version}"),
+                       script_instance.format_package_name("hadooplzo_${stack_version}-native")]
+  else:
+    lzo_packages += ["hadoop-lzo"]
+
+  return lzo_packages
+
+def should_install_lzo():
+  """
+  Return true if lzo is enabled via core-site.xml and GPL license (required for lzo) is accepted.
+  """
+  config = Script.get_config()
+  io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
+  lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
+
+  if not lzo_enabled:
+    return False
+
+  is_gpl_license_accepted = default("/hostLevelParams/gpl_license_accepted", False)
+  if not is_gpl_license_accepted:
+    Logger.warning(INSTALLING_LZO_WITHOUT_GPL)
+    return False
+
+  return True
+
+def install_lzo_if_needed():
+  """
+  Install lzo package if {#should_install_lzo} is true
+  """
+  if not should_install_lzo():
+    return
+
+  lzo_packages = get_lzo_packages()
+
+  config = Script.get_config()
+  agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability']
+  agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int)
+
+  Package(lzo_packages,
+          retry_on_repo_unavailability=agent_stack_retry_on_unavailability,
+          retry_count=agent_stack_retry_count
+  )

http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
index 31e78b9..ded63cf 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/package_conditions.py
@@ -19,7 +19,7 @@ limitations under the License.
 Ambari Agent
 
 """
-__all__ = ["is_lzo_enabled", "should_install_phoenix", "should_install_ams_collector", "should_install_ams_grafana",
+__all__ = ["should_install_phoenix", "should_install_ams_collector", "should_install_ams_grafana",
            "should_install_mysql", "should_install_ranger_tagsync"]
 
 import os
@@ -44,12 +44,6 @@ def _has_local_components(config, components, indicator_function = any):
 def _has_applicable_local_component(config, components):
   return _has_local_components(config, components, any)
 
-def should_install_lzo():
-  config = Script.get_config()
-  io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
-  lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
-  return lzo_enabled
-
 def should_install_phoenix():
   phoenix_hosts = default('/clusterHostInfo/phoenix_query_server_hosts', [])
   phoenix_enabled = default('/configurations/hbase-env/phoenix_sql_enabled', False)

http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
index 2df1b9f..2ab5ef8 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
@@ -203,11 +203,6 @@
             <package>
               <name>hadoop</name>
             </package>
-            <package>
-              <name>hadoop-lzo</name>
-              <skipUpgrade>true</skipUpgrade>
-              <condition>should_install_lzo</condition>
-            </package>
           </packages>
         </osSpecific>
         
@@ -224,16 +219,6 @@
               <name>snappy-devel</name>
             </package>
             <package>
-              <name>lzo</name>
-              <skipUpgrade>true</skipUpgrade>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
-              <name>hadoop-lzo-native</name>
-              <skipUpgrade>true</skipUpgrade>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
               <name>hadoop-libhdfs</name>
             </package>
           </packages>
@@ -252,16 +237,6 @@
               <name>snappy-devel</name>
             </package>
             <package>
-              <name>liblzo2-2</name>
-              <skipUpgrade>true</skipUpgrade>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
-              <name>hadoop-lzo-native</name>
-              <skipUpgrade>true</skipUpgrade>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
               <name>hadoop-libhdfs</name>
             </package>
           </packages>
@@ -280,11 +255,6 @@
               <name>libsnappy-dev</name>
             </package>
             <package>
-              <name>liblzo2-2</name>
-              <skipUpgrade>true</skipUpgrade>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
               <name>hadoop-hdfs</name>
             </package>
             <package>

http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
index 6f702d3..23aa08c 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs.py
@@ -23,6 +23,7 @@ from resource_management import *
 import os
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 from ambari_commons import OSConst
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
 
 @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
 def hdfs(name=None):
@@ -133,10 +134,7 @@ def hdfs(name=None):
        content=Template("slaves.j2")
   )
   
-  if params.lzo_enabled and len(params.lzo_packages) > 0:
-      Package(params.lzo_packages,
-              retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
-              retry_count=params.agent_stack_retry_count)
+  install_lzo_if_needed()
       
 def install_snappy():
   import params

http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
index fe488c3..dc3279f 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/install_params.py
@@ -23,17 +23,7 @@ if OSCheck.is_windows_family():
   exclude_packages = []
 else:
   from resource_management.libraries.functions.default import default
-  from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
   from resource_management.libraries.script.script import Script
 
   _config = Script.get_config()
   stack_version_unformatted = str(_config['hostLevelParams']['stack_version'])
-
-  # The logic for LZO also exists in OOZIE's params.py
-  io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
-  lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
-  lzo_packages = get_lzo_packages(stack_version_unformatted)
-
-  exclude_packages = []
-  if not lzo_enabled:
-    exclude_packages += lzo_packages

http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
index 5ca012b..5e1f4ac 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py
@@ -40,7 +40,6 @@ from resource_management.libraries.functions.get_not_managed_resources import
ge
 from resource_management.libraries.script.script import Script
 from resource_management.libraries.resources.hdfs_resource import HdfsResource
 from resource_management.libraries.functions.format_jvm_option import format_jvm_option
-from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
 from resource_management.libraries.functions.hdfs_utils import is_https_enabled_in_hdfs
 from resource_management.libraries.functions import is_empty
 from resource_management.libraries.functions.get_architecture import get_architecture
@@ -383,12 +382,6 @@ HdfsResource = functools.partial(
   immutable_paths = get_not_managed_resources(),
   dfs_type = dfs_type
 )
-
-
-# The logic for LZO also exists in OOZIE's params.py
-io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
-lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
-lzo_packages = get_lzo_packages(stack_version_unformatted)
   
 name_node_params = default("/commandParams/namenode", None)
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
index abbe59e..fbff099 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive.py
@@ -41,6 +41,7 @@ from resource_management.core.logger import Logger
 from resource_management.core import utils
 from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster,
setup_atlas_hook
 from resource_management.libraries.functions.security_commons import update_credential_provider_path
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
 from ambari_commons.constants import SERVICE
 
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
@@ -105,6 +106,9 @@ def hive(name=None):
 @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
 def hive(name=None):
   import params
+
+  install_lzo_if_needed()
+
   hive_client_conf_path = format("{stack_root}/current/{component_directory}/conf")
   # Permissions 644 for conf dir (client) files, and 600 for conf.server
   mode_identified = 0644 if params.hive_config_dir == hive_client_conf_path else 0600

http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
index 7d22603..2316f79 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie.py
@@ -35,6 +35,7 @@ from resource_management.libraries.functions.stack_features import check_stack_f
 from resource_management.libraries.functions.oozie_prepare_war import prepare_war
 from resource_management.libraries.functions.copy_tarball import get_current_version
 from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
 from resource_management.libraries.script.script import Script
 from resource_management.libraries.functions.security_commons import update_credential_provider_path
 from resource_management.core.resources.packaging import Package
@@ -185,16 +186,21 @@ def oozie(is_server=False, upgrade_type=None):
     File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
       content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
     )
-  pass
 
   oozie_ownership()
-  
-  if is_server:      
+
+  if params.lzo_enabled:
+    install_lzo_if_needed()
+    Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
+      not_if  = no_op_test,
+    )
+
+  if is_server:
     oozie_server_specific(upgrade_type)
-  
+
 def oozie_ownership():
   import params
-  
+
   File ( format("{conf_dir}/hadoop-config.xml"),
     owner = params.oozie_user,
     group = params.user_group
@@ -237,14 +243,14 @@ def get_oozie_ext_zip_source_paths(upgrade_type, params):
 
 def oozie_server_specific(upgrade_type):
   import params
-  
+
   no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat
{pid_file}` >/dev/null 2>&1"), user=params.oozie_user)
-  
+
   File(params.pid_file,
     action="delete",
     not_if=no_op_test
   )
-  
+
   oozie_server_directories = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir,
params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir,
params.oozie_webapps_conf_dir, params.oozie_server_dir]
   Directory( oozie_server_directories,
     owner = params.oozie_user,
@@ -253,25 +259,25 @@ def oozie_server_specific(upgrade_type):
     create_parents = True,
     cd_access="a",
   )
-  
+
   Directory(params.oozie_libext_dir,
             create_parents = True,
   )
-  
+
   hashcode_file = format("{oozie_home}/.hashcode")
   skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share")
 
   untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home)
 
   Execute( untar_sharelib,    # time-expensive
-    not_if  = format("{no_op_test} || {skip_recreate_sharelib}"), 
+    not_if  = format("{no_op_test} || {skip_recreate_sharelib}"),
     sudo = True,
   )
 
   configure_cmds = []
   # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path
   source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params)
-  
+
   # Copy the first oozie ext-2.2.zip file that is found.
   # This uses a list to handle the cases when migrating from some versions of BigInsights
to HDP.
   if source_ext_zip_paths is not None:
@@ -285,8 +291,8 @@ def oozie_server_specific(upgrade_type):
                 sudo=True,
                 )
         break
-  
-  
+
+
   Directory(params.oozie_webapps_conf_dir,
             owner = params.oozie_user,
             group = params.user_group,
@@ -305,14 +311,6 @@ def oozie_server_specific(upgrade_type):
     Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'),
       not_if  = no_op_test)
 
-  if params.lzo_enabled and len(params.all_lzo_packages) > 0:
-    Package(params.all_lzo_packages,
-            retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
-            retry_count=params.agent_stack_retry_count)
-    Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
-      not_if  = no_op_test,
-    )
-
   prepare_war(params)
 
   File(hashcode_file,
@@ -363,7 +361,7 @@ def oozie_server_specific(upgrade_type):
   Directory(params.oozie_server_dir,
     owner = params.oozie_user,
     group = params.user_group,
-    recursive_ownership = True,  
+    recursive_ownership = True,
   )
   if params.security_enabled:
     File(os.path.join(params.conf_dir, 'zkmigrator_jaas.conf'),
@@ -408,7 +406,7 @@ def copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=N
   effective_version = params.stack_version_formatted if upgrade_type is None else format_stack_version(params.version)
   if not check_stack_feature(StackFeature.ATLAS_HOOK_SUPPORT, effective_version):
     return
-    
+
   # Important that oozie_server_hostnames is sorted by name so that this only runs on a single
Oozie server.
   if not (len(params.oozie_server_hostnames) > 0 and params.hostname == params.oozie_server_hostnames[0]):
     Logger.debug("Will not attempt to copy Atlas Hive hook to DFS since this is not the first
Oozie Server "
@@ -463,7 +461,7 @@ def copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type=None, upgrade_direction=N
                           recursive_chmod=True,
                           replace_existing_files=True
                           )
-                          
+
       Logger.info("Copying Atlas Hive hook properties file to Oozie Sharelib in DFS.")
       atlas_hook_filepath_source = os.path.join(params.hive_conf_dir, params.atlas_hook_filename)
       atlas_hook_file_path_dest_in_dfs = os.path.join(hive_sharelib_dir, params.atlas_hook_filename)

http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
index cd1f7ad..8f38843 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/params_linux.py
@@ -31,7 +31,7 @@ from resource_management.libraries.functions import get_port_from_url
 from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
 from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster
 from resource_management.libraries.script.script import Script
-from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
+from resource_management.libraries.functions.lzo_utils import should_install_lzo
 from resource_management.libraries.functions.expect import expect
 from resource_management.libraries.functions.get_architecture import get_architecture
 from resource_management.libraries.functions.stack_features import get_stack_feature_version
@@ -387,8 +387,4 @@ HdfsResource = functools.partial(
 
 is_webhdfs_enabled = config['configurations']['hdfs-site']['dfs.webhdfs.enabled']
 
-# The logic for LZO also exists in HDFS' params.py
-io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
-lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
-
-all_lzo_packages = get_lzo_packages(stack_version_unformatted)
+lzo_enabled = should_install_lzo()

http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
index f6e8391..197382c 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
@@ -27,6 +27,7 @@ from resource_management.libraries.script.script import Script
 from resource_management.core.resources.service import ServiceConfig
 from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.is_empty import is_empty
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
 from resource_management.core.resources.system import Directory
 from resource_management.core.resources.system import File
 from resource_management.libraries.resources.xml_config import XmlConfig
@@ -110,6 +111,8 @@ def yarn(name=None, config_dir=None):
   """
   import params
 
+  install_lzo_if_needed()
+
   if config_dir is None:
     config_dir = params.hadoop_conf_dir
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml
index 07a9ef8..8aa69a9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HDFS/metainfo.xml
@@ -40,19 +40,6 @@
               <name>snappy-devel</name>
             </package>
             <package>
-              <name>lzo</name>
-              <skipUpgrade>true</skipUpgrade>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
-              <name>hadooplzo_${stack_version}</name>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
-              <name>hadooplzo_${stack_version}-native</name>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
               <name>hadoop_${stack_version}-libhdfs</name>
             </package>
           </packages>
@@ -74,19 +61,6 @@
               <name>snappy-devel</name>
             </package>
             <package>
-              <name>liblzo2-2</name>
-              <skipUpgrade>true</skipUpgrade>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
-              <name>hadooplzo_${stack_version}</name>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
-              <name>hadooplzo_${stack_version}-native</name>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
               <name>hadoop_${stack_version}-libhdfs</name>
             </package>
           </packages>
@@ -120,15 +94,6 @@
               <name>libsnappy-dev</name>
             </package>
             <package>
-              <name>hadooplzo-${stack_version}</name>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
-              <name>liblzo2-2</name>
-              <skipUpgrade>true</skipUpgrade>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
               <name>libhdfs0-${stack_version}</name>
             </package>
           </packages>

http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml
index 63e85e3..e02bf6b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HDFS/metainfo.xml
@@ -72,19 +72,6 @@
               <name>snappy-devel</name>
             </package>
             <package>
-              <name>lzo</name>
-              <skipUpgrade>true</skipUpgrade>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
-              <name>hadooplzo_${stack_version}</name>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
-              <name>hadooplzo_${stack_version}-native</name>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
               <name>hadoop_${stack_version}-libhdfs</name>
             </package>
           </packages>
@@ -106,19 +93,6 @@
               <name>snappy-devel</name>
             </package>
             <package>
-              <name>liblzo2-2</name>
-              <skipUpgrade>true</skipUpgrade>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
-              <name>hadooplzo_${stack_version}</name>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
-              <name>hadooplzo_${stack_version}-native</name>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
               <name>hadoop_${stack_version}-libhdfs</name>
             </package>
           </packages>
@@ -152,10 +126,6 @@
               <name>libsnappy-dev</name>
             </package>
             <package>
-              <name>hadooplzo-${stack_version}</name>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
               <name>libhdfs0-${stack_version}</name>
             </package>
           </packages>

http://git-wip-us.apache.org/repos/asf/ambari/blob/096fedbd/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
index 86ca03a..29cbddc 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
@@ -55,7 +55,8 @@
         "mysql_jdbc_url": "http://c6401.ambari.apache.org:8080/resources//mysql-connector-java.jar",
         "custom_mysql_jdbc_name" : "mysql-connector-java.jar",
         "custom_oracle_jdbc_name" : "oracle-jdbc-driver.jar",
-        "custom_postgres_jdbc_name" : "test-postgres-jdbc.jar"
+        "custom_postgres_jdbc_name" : "test-postgres-jdbc.jar",
+        "gpl_license_accepted": "true"
     }, 
     "commandType": "EXECUTION_COMMAND", 
     "roleParams": {


Mime
View raw message