Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 3D54B200D41 for ; Wed, 22 Nov 2017 11:32:13 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 3BADE160BFD; Wed, 22 Nov 2017 10:32:13 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 34DFE160BDA for ; Wed, 22 Nov 2017 11:32:12 +0100 (CET) Received: (qmail 76550 invoked by uid 500); 22 Nov 2017 10:32:11 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 76541 invoked by uid 99); 22 Nov 2017 10:32:11 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 22 Nov 2017 10:32:11 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 46BA0DFFB5; Wed, 22 Nov 2017 10:32:11 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aonishuk@apache.org To: commits@ambari.apache.org Message-Id: <1dd80e15d727425d83b5bef172828cb0@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-22489. Port AMBARI-22467 to BigInsight (aonishuk) Date: Wed, 22 Nov 2017 10:32:11 +0000 (UTC) archived-at: Wed, 22 Nov 2017 10:32:13 -0000 Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-22457 c7457a820 -> aea1a3c48 AMBARI-22489. Port AMBARI-22467 to BigInsight (aonishuk) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/aea1a3c4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/aea1a3c4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/aea1a3c4 Branch: refs/heads/branch-feature-AMBARI-22457 Commit: aea1a3c48fdf58be5fad04d7b983c65d28a248f9 Parents: c7457a8 Author: Andrew Onishuk Authored: Wed Nov 22 12:32:04 2017 +0200 Committer: Andrew Onishuk Committed: Wed Nov 22 12:32:04 2017 +0200 ---------------------------------------------------------------------- .../BigInsights/4.0/services/HDFS/metainfo.xml | 17 ------------ .../BigInsights/4.2/services/HDFS/metainfo.xml | 20 -------------- .../4.2/services/HDFS/package/scripts/hdfs.py | 5 ++-- .../4.2/services/HIVE/package/scripts/hive.py | 3 +++ .../4.2/services/OOZIE/package/scripts/oozie.py | 12 +++++---- .../services/OOZIE/package/scripts/params.py | 28 ++------------------ .../4.2/services/YARN/package/scripts/yarn.py | 3 +++ 7 files changed, 17 insertions(+), 71 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml index 918cdb3..82c82d0 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml @@ -152,10 +152,6 @@ hadoop - - hadoop-lzo - should_install_lzo - @@ -165,15 +161,6 @@ snappy - - lzo - should_install_lzo - - - hadoop-lzo-native - should_install_lzo - - hadoop-libhdfs @@ -192,10 +179,6 @@ libsnappy-dev - liblzo2-2 - should_install_lzo - - hadoop-hdfs http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml index 72ce2be..4253eb1 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml @@ -182,11 +182,6 @@ hadoop_4_2_* - - hadoop-lzo - true - should_install_lzo - @@ -196,16 +191,6 @@ snappy - lzo - true - should_install_lzo - - - hadoop-lzo-native - true - should_install_lzo - - hadoop_4_2_*-libhdfs @@ -229,11 +214,6 @@ libsnappy-dev - liblzo2-2 - true - should_install_lzo - - hadoop-hdfs http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py index 2b158bd..9334de0 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py @@ -20,10 +20,10 @@ Ambari Agent """ from resource_management import * +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed import sys import os - def hdfs(name=None): import params @@ -115,8 +115,7 @@ def hdfs(name=None): content=Template("slaves.j2") ) - if params.lzo_enabled and len(params.lzo_packages) > 0: - Package(params.lzo_packages) + install_lzo_if_needed() def install_snappy(): import params http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py index 0446e1c..f451a65 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py @@ -34,6 +34,7 @@ from resource_management.core.shell import as_user from resource_management.libraries.functions.is_empty import is_empty from resource_management.libraries.resources.xml_config import XmlConfig from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed from resource_management.core.exceptions import Fail from resource_management.core.shell import as_sudo from resource_management.core.shell import quote_bash_args @@ -48,6 +49,8 @@ from ambari_commons import OSConst def hive(name=None): import params + install_lzo_if_needed() + XmlConfig("hive-site.xml", conf_dir = params.hive_conf_dir, configurations = params.config['configurations']['hive-site'], http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py index 64a83c0..0459432 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py @@ -26,6 +26,7 @@ from resource_management.core.resources.system import Execute from resource_management.core.source import DownloadSource from resource_management.core.source import InlineTemplate from resource_management.core.source import Template +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed from resource_management.libraries.functions import format from resource_management.libraries.functions import compare_versions from resource_management.libraries.resources.xml_config import XmlConfig @@ -99,6 +100,12 @@ def oozie(is_server=False): pass oozie_ownership() + + if params.lzo_enabled: + install_lzo_if_needed() + Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), + not_if = no_op_test, + ) if is_server: oozie_server_specific() @@ -227,11 +234,6 @@ def oozie_server_specific(): Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension.jar'), not_if = no_op_test, ) - #if params.lzo_enabled and len(params.lzo_packages_for_current_host) > 0: - # Package(params.lzo_packages_for_current_host) - # Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), - # not_if = no_op_test, - # ) prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd") prepare_war_cmd = format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure}") http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py index d1a4a66..ee9b88a 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py @@ -27,6 +27,7 @@ from resource_management.libraries.functions import get_port_from_url from resource_management.libraries.functions import format from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.version import compare_versions +from resource_management.libraries.functions.lzo_utils import should_install_lzo from resource_management.libraries.resources import HdfsResource from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_select @@ -234,29 +235,4 @@ HdfsResource = functools.partial( default_fs = default_fs ) -#LZO support - -#-----LZO is not suppported in IOP distribution since it is GPL license-------- - -''' -io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None) -lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower() - -# stack_is_iop40_or_further -underscored_version = stack_version_unformatted.replace('.', '_') -dashed_version = stack_version_unformatted.replace('.', '-') -lzo_packages_to_family = { - "any": ["hadoop-lzo", ], - "redhat": ["lzo", "hadoop-lzo-native"], - "suse": ["lzo", "hadoop-lzo-native"], - "ubuntu": ["liblzo2-2", ] -} - - -lzo_packages_to_family["redhat"] += [format("hadooplzo_{underscorred_version}_*")] -lzo_packages_to_family["suse"] += [format("hadooplzo_{underscorred_version}_*")] -lzo_packages_to_family["ubuntu"] += [format("hadooplzo_{dashed_version}_*")] - -lzo_packages_for_current_host = lzo_packages_to_family['any'] + lzo_packages_to_family[System.get_instance().os_family] -all_lzo_packages = set(itertools.chain(*lzo_packages_to_family.values())) -''' +lzo_enabled = should_install_lzo() http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py index a284183..9006d2e 100755 --- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py +++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py @@ -20,6 +20,7 @@ Ambari Agent """ from resource_management import * +from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed import sys import os @@ -50,6 +51,8 @@ def create_local_dir(dir_name): def yarn(name = None): import params + + install_lzo_if_needed() if name == "historyserver": if params.yarn_log_aggregation_enabled: