ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject ambari git commit: AMBARI-22489. Port AMBARI-22467 to BigInsight (aonishuk)
Date Wed, 22 Nov 2017 10:32:11 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-feature-AMBARI-22457 c7457a820 -> aea1a3c48


AMBARI-22489. Port AMBARI-22467 to BigInsight (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/aea1a3c4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/aea1a3c4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/aea1a3c4

Branch: refs/heads/branch-feature-AMBARI-22457
Commit: aea1a3c48fdf58be5fad04d7b983c65d28a248f9
Parents: c7457a8
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Wed Nov 22 12:32:04 2017 +0200
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Wed Nov 22 12:32:04 2017 +0200

----------------------------------------------------------------------
 .../BigInsights/4.0/services/HDFS/metainfo.xml  | 17 ------------
 .../BigInsights/4.2/services/HDFS/metainfo.xml  | 20 --------------
 .../4.2/services/HDFS/package/scripts/hdfs.py   |  5 ++--
 .../4.2/services/HIVE/package/scripts/hive.py   |  3 +++
 .../4.2/services/OOZIE/package/scripts/oozie.py | 12 +++++----
 .../services/OOZIE/package/scripts/params.py    | 28 ++------------------
 .../4.2/services/YARN/package/scripts/yarn.py   |  3 +++
 7 files changed, 17 insertions(+), 71 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
index 918cdb3..82c82d0 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.0/services/HDFS/metainfo.xml
@@ -152,10 +152,6 @@
             <package>
               <name>hadoop</name>
             </package>
-            <package>
-              <name>hadoop-lzo</name>
-              <condition>should_install_lzo</condition>
-            </package>
           </packages>
         </osSpecific>
 
@@ -165,15 +161,6 @@
             <package>
               <name>snappy</name>
             </package>
-            <package>
-              <name>lzo</name>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
-              <name>hadoop-lzo-native</name>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
               <name>hadoop-libhdfs</name>
             </package>
             <package>
@@ -192,10 +179,6 @@
               <name>libsnappy-dev</name>
             </package>
             <package>
-              <name>liblzo2-2</name>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
               <name>hadoop-hdfs</name>
             </package>
             <package>

http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
index 72ce2be..4253eb1 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/metainfo.xml
@@ -182,11 +182,6 @@
             <package>
               <name>hadoop_4_2_*</name>
             </package>
-            <package>
-              <name>hadoop-lzo</name>
-              <skipUpgrade>true</skipUpgrade>
-              <condition>should_install_lzo</condition>
-            </package>
           </packages>
         </osSpecific>
         <osSpecific>
@@ -196,16 +191,6 @@
               <name>snappy</name>
             </package>
             <package>
-              <name>lzo</name>
-              <skipUpgrade>true</skipUpgrade>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
-              <name>hadoop-lzo-native</name>
-              <skipUpgrade>true</skipUpgrade>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
               <name>hadoop_4_2_*-libhdfs</name>
             </package>
             <package>
@@ -229,11 +214,6 @@
               <name>libsnappy-dev</name>
             </package>
             <package>
-              <name>liblzo2-2</name>
-              <skipUpgrade>true</skipUpgrade>
-              <condition>should_install_lzo</condition>
-            </package>
-            <package>
               <name>hadoop-hdfs</name>
             </package>
             <package>

http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
index 2b158bd..9334de0 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HDFS/package/scripts/hdfs.py
@@ -20,10 +20,10 @@ Ambari Agent
 """
 
 from resource_management import *
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
 import sys
 import os
 
-
 def hdfs(name=None):
   import params
 
@@ -115,8 +115,7 @@ def hdfs(name=None):
        content=Template("slaves.j2")
   )
 
-  if params.lzo_enabled and len(params.lzo_packages) > 0:
-      Package(params.lzo_packages)
+  install_lzo_if_needed()
 
 def install_snappy():
   import params

http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py
index 0446e1c..f451a65 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/HIVE/package/scripts/hive.py
@@ -34,6 +34,7 @@ from resource_management.core.shell import as_user
 from resource_management.libraries.functions.is_empty import is_empty
 from resource_management.libraries.resources.xml_config import XmlConfig
 from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
 from resource_management.core.exceptions import Fail
 from resource_management.core.shell import as_sudo
 from resource_management.core.shell import quote_bash_args
@@ -48,6 +49,8 @@ from ambari_commons import OSConst
 def hive(name=None):
   import params
 
+  install_lzo_if_needed()
+  
   XmlConfig("hive-site.xml",
             conf_dir = params.hive_conf_dir,
             configurations = params.config['configurations']['hive-site'],

http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
index 64a83c0..0459432 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/oozie.py
@@ -26,6 +26,7 @@ from resource_management.core.resources.system import Execute
 from resource_management.core.source import DownloadSource
 from resource_management.core.source import InlineTemplate
 from resource_management.core.source import Template
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
 from resource_management.libraries.functions import format
 from resource_management.libraries.functions import compare_versions
 from resource_management.libraries.resources.xml_config import XmlConfig
@@ -99,6 +100,12 @@ def oozie(is_server=False):
   pass
 
   oozie_ownership()
+  
+  if params.lzo_enabled:
+    install_lzo_if_needed()
+    Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
+      not_if  = no_op_test,
+    )
 
   if is_server:
     oozie_server_specific()
@@ -227,11 +234,6 @@ def oozie_server_specific():
     Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension.jar'),
       not_if  = no_op_test,
     )
-  #if params.lzo_enabled and len(params.lzo_packages_for_current_host) > 0:
-  #  Package(params.lzo_packages_for_current_host)
-  #  Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'),
-  #    not_if  = no_op_test,
-  #  )
 
   prepare_war_cmd_file = format("{oozie_home}/.prepare_war_cmd")
   prepare_war_cmd = format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war {oozie_secure}")

http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
index d1a4a66..ee9b88a 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/OOZIE/package/scripts/params.py
@@ -27,6 +27,7 @@ from resource_management.libraries.functions import get_port_from_url
 from resource_management.libraries.functions import format
 from resource_management.libraries.functions.version import format_stack_version
 from resource_management.libraries.functions.version import compare_versions
+from resource_management.libraries.functions.lzo_utils import should_install_lzo
 from resource_management.libraries.resources import HdfsResource
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import stack_select
@@ -234,29 +235,4 @@ HdfsResource = functools.partial(
   default_fs = default_fs
 )
 
-#LZO support
-
-#-----LZO is not suppported in IOP distribution since it is GPL license--------
-
-'''
-io_compression_codecs = default("/configurations/core-site/io.compression.codecs", None)
-lzo_enabled = io_compression_codecs is not None and "com.hadoop.compression.lzo" in io_compression_codecs.lower()
-
-# stack_is_iop40_or_further
-underscored_version = stack_version_unformatted.replace('.', '_')
-dashed_version = stack_version_unformatted.replace('.', '-')
-lzo_packages_to_family = {
-  "any": ["hadoop-lzo", ],
-  "redhat": ["lzo", "hadoop-lzo-native"],
-  "suse": ["lzo", "hadoop-lzo-native"],
-  "ubuntu": ["liblzo2-2", ]
-}
-
-
-lzo_packages_to_family["redhat"] += [format("hadooplzo_{underscorred_version}_*")]
-lzo_packages_to_family["suse"] += [format("hadooplzo_{underscorred_version}_*")]
-lzo_packages_to_family["ubuntu"] += [format("hadooplzo_{dashed_version}_*")]
-
-lzo_packages_for_current_host = lzo_packages_to_family['any'] + lzo_packages_to_family[System.get_instance().os_family]
-all_lzo_packages = set(itertools.chain(*lzo_packages_to_family.values()))
-'''
+lzo_enabled = should_install_lzo()

http://git-wip-us.apache.org/repos/asf/ambari/blob/aea1a3c4/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py
index a284183..9006d2e 100755
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/services/YARN/package/scripts/yarn.py
@@ -20,6 +20,7 @@ Ambari Agent
 """
 
 from resource_management import *
+from resource_management.libraries.functions.lzo_utils import install_lzo_if_needed
 import sys
 import os
 
@@ -50,6 +51,8 @@ def create_local_dir(dir_name):
 
 def yarn(name = None):
   import params
+  
+  install_lzo_if_needed()
 
   if name == "historyserver":
     if params.yarn_log_aggregation_enabled:


Mime
View raw message