Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id AFA2A200D51 for ; Thu, 7 Dec 2017 17:39:58 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id AE79D160C21; Thu, 7 Dec 2017 16:39:58 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id BB023160C23 for ; Thu, 7 Dec 2017 17:39:56 +0100 (CET) Received: (qmail 97600 invoked by uid 500); 7 Dec 2017 16:39:54 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 97411 invoked by uid 99); 7 Dec 2017 16:39:54 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 07 Dec 2017 16:39:54 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 35115F6142; Thu, 7 Dec 2017 16:39:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ncole@apache.org To: commits@ambari.apache.org Date: Thu, 07 Dec 2017 16:40:28 -0000 Message-Id: <288de16e39064a499a0fa235e0f50eff@git.apache.org> In-Reply-To: <39245bff0c1946dd9f6da424048b1821@git.apache.org> References: <39245bff0c1946dd9f6da424048b1821@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [37/50] [abbrv] ambari git commit: AMBARI-22598 - Pig service check failed after PU with LzoCodec CNF (jonathanhurley) archived-at: Thu, 07 Dec 2017 16:39:58 -0000 AMBARI-22598 - Pig service check failed after PU with LzoCodec CNF (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/508eba0c Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/508eba0c Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/508eba0c Branch: refs/heads/branch-feature-AMBARI-21674 Commit: 508eba0c62aa42cea5bcc660a4661ff373a5af91 Parents: 97ceed0 Author: Jonathan Hurley Authored: Tue Dec 5 13:30:55 2017 -0500 Committer: Jonathan Hurley Committed: Wed Dec 6 08:59:14 2017 -0500 ---------------------------------------------------------------------- .../MAHOUT/1.0.0.2.3/package/scripts/mahout.py | 12 +++++++--- .../PIG/0.12.0.2.0/package/scripts/pig.py | 16 +++++++++---- .../PIG/0.16.1.3.0/package/scripts/pig.py | 16 +++++++++---- .../SPARK/1.2.1/package/scripts/setup_spark.py | 20 ++++++++-------- .../SPARK/2.2.0/package/scripts/setup_spark.py | 25 ++++++++++---------- .../SPARK2/2.0.0/package/scripts/setup_spark.py | 22 ++++++++--------- 6 files changed, 65 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/508eba0c/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/mahout.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/mahout.py b/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/mahout.py index 68d6844..d2df928 100644 --- a/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/mahout.py +++ b/ambari-server/src/main/resources/common-services/MAHOUT/1.0.0.2.3/package/scripts/mahout.py @@ -20,14 +20,20 @@ Ambari Agent """ import os -from resource_management.core.resources.system import Directory, File -from resource_management.libraries.resources.xml_config import XmlConfig -from resource_management.libraries.functions.format import format +from resource_management.core.resources import Directory +from resource_management.core.resources import File +from resource_management.libraries.functions import format from resource_management.libraries.functions import is_empty +from resource_management.libraries.functions import lzo_utils +from resource_management.libraries.resources import XmlConfig + def mahout(): import params + # ensure that matching LZO libraries are installed for Mahout + lzo_utils.install_lzo_if_needed() + Directory( params.mahout_conf_dir, create_parents = True, owner = params.mahout_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/508eba0c/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py index b6825d5..1378fdc 100644 --- a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py +++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py @@ -19,16 +19,22 @@ Ambari Agent """ import os -from resource_management.core.resources.system import Directory, File -from resource_management.core.source import InlineTemplate -from resource_management.libraries.functions.format import format from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from resource_management.core.resources import Directory +from resource_management.core.resources import File +from resource_management.core.source import InlineTemplate +from resource_management.libraries.functions import format +from resource_management.libraries.functions import lzo_utils + @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def pig(): import params + # ensure that matching LZO libraries are installed for Pig + lzo_utils.install_lzo_if_needed() + Directory( params.pig_conf_dir, create_parents = True, owner = params.hdfs_user, @@ -49,7 +55,7 @@ def pig(): content=params.pig_properties ) - if (params.log4j_props != None): + if (params.log4j_props is not None): File(format("{params.pig_conf_dir}/log4j.properties"), mode=0644, group=params.user_group, @@ -72,7 +78,7 @@ def pig(): content=params.pig_properties ) - if (params.log4j_props != None): + if (params.log4j_props is not None): File(os.path.join(params.pig_conf_dir, "log4j.properties"), mode='f', owner=params.pig_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/508eba0c/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py index b6825d5..1378fdc 100644 --- a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py +++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py @@ -19,16 +19,22 @@ Ambari Agent """ import os -from resource_management.core.resources.system import Directory, File -from resource_management.core.source import InlineTemplate -from resource_management.libraries.functions.format import format from ambari_commons import OSConst from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl +from resource_management.core.resources import Directory +from resource_management.core.resources import File +from resource_management.core.source import InlineTemplate +from resource_management.libraries.functions import format +from resource_management.libraries.functions import lzo_utils + @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT) def pig(): import params + # ensure that matching LZO libraries are installed for Pig + lzo_utils.install_lzo_if_needed() + Directory( params.pig_conf_dir, create_parents = True, owner = params.hdfs_user, @@ -49,7 +55,7 @@ def pig(): content=params.pig_properties ) - if (params.log4j_props != None): + if (params.log4j_props is not None): File(format("{params.pig_conf_dir}/log4j.properties"), mode=0644, group=params.user_group, @@ -72,7 +78,7 @@ def pig(): content=params.pig_properties ) - if (params.log4j_props != None): + if (params.log4j_props is not None): File(os.path.join(params.pig_conf_dir, "log4j.properties"), mode='f', owner=params.pig_user, http://git-wip-us.apache.org/repos/asf/ambari/blob/508eba0c/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py index 6a29efb..830a7ee 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/package/scripts/setup_spark.py @@ -18,21 +18,18 @@ limitations under the License. """ -import sys -import fileinput -import shutil import os -from resource_management.core.exceptions import ComponentIsNotRunning -from resource_management.core.logger import Logger -from resource_management.core import shell + from resource_management.core.source import InlineTemplate from resource_management.core.resources.system import Directory, File -from resource_management.libraries.resources.properties_file import PropertiesFile from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions.constants import StackFeature -from resource_management.libraries.functions.format import format -from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions import lzo_utils +from resource_management.libraries.resources import PropertiesFile +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.resources import HdfsResource +from resource_management.libraries.resources import XmlConfig + def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None): """ @@ -45,6 +42,9 @@ def setup_spark(env, type, upgrade_type=None, action=None, config_dir=None): import params + # ensure that matching LZO libraries are installed for Spark + lzo_utils.install_lzo_if_needed() + if config_dir is None: config_dir = params.spark_conf http://git-wip-us.apache.org/repos/asf/ambari/blob/508eba0c/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/setup_spark.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/setup_spark.py index 9329ce0..148a967 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/setup_spark.py +++ b/ambari-server/src/main/resources/common-services/SPARK/2.2.0/package/scripts/setup_spark.py @@ -18,31 +18,32 @@ limitations under the License. """ -import sys -import fileinput -import shutil import os -from resource_management.core.exceptions import ComponentIsNotRunning -from resource_management.core.logger import Logger -from resource_management.core import shell +from resource_management.core.resources import Directory +from resource_management.core.resources import File from resource_management.core.source import InlineTemplate -from resource_management.core.resources.system import Directory, File -from resource_management.libraries.resources.properties_file import PropertiesFile from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions.constants import StackFeature -from resource_management.libraries.functions.format import format -from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions import format +from resource_management.libraries.functions import lzo_utils +from resource_management.libraries.resources import PropertiesFile +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.resources import HdfsResource +from resource_management.libraries.resources import XmlConfig def setup_spark(env, type, upgrade_type = None, action = None): import params + # ensure that matching LZO libraries are installed for Spark + lzo_utils.install_lzo_if_needed() + Directory([params.spark_pid_dir, params.spark_log_dir], owner=params.spark_user, group=params.user_group, mode=0775, - create_parents = True + create_parents = True, + cd_access = 'a', ) if type == 'server' and action == 'config': params.HdfsResource(params.spark_hdfs_user_dir, http://git-wip-us.apache.org/repos/asf/ambari/blob/508eba0c/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_spark.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_spark.py index 792b2a1..3638953 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_spark.py +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/setup_spark.py @@ -18,26 +18,26 @@ limitations under the License. """ -import sys -import fileinput -import shutil import os -from resource_management.core.exceptions import ComponentIsNotRunning -from resource_management.core.logger import Logger -from resource_management.core import shell +from resource_management.core.resources import Directory +from resource_management.core.resources import File from resource_management.core.source import InlineTemplate -from resource_management.core.resources.system import Directory, File -from resource_management.libraries.resources.properties_file import PropertiesFile from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.stack_features import check_stack_feature -from resource_management.libraries.functions.constants import StackFeature -from resource_management.libraries.functions.format import format -from resource_management.libraries.resources.xml_config import XmlConfig +from resource_management.libraries.functions import format +from resource_management.libraries.functions import lzo_utils +from resource_management.libraries.resources import PropertiesFile +from resource_management.libraries.functions import StackFeature +from resource_management.libraries.resources import HdfsResource +from resource_management.libraries.resources import XmlConfig def setup_spark(env, type, upgrade_type = None, action = None): import params + # ensure that matching LZO libraries are installed for Spark + lzo_utils.install_lzo_if_needed() + Directory([params.spark_pid_dir, params.spark_log_dir], owner=params.spark_user, group=params.user_group,