Return-Path: <commits-return-45471-archive-asf-public=cust-asf.ponee.io@ambari.apache.org>
X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io
Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io
Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183])
	by cust-asf2.ponee.io (Postfix) with ESMTP id 87949200CEC
	for <archive-asf-public-internal@cust-asf2.ponee.io>; Mon, 31 Jul 2017 16:10:21 +0200 (CEST)
Received: by cust-asf.ponee.io (Postfix)
	id 86518165434; Mon, 31 Jul 2017 14:10:21 +0000 (UTC)
Delivered-To: archive-asf-public@cust-asf.ponee.io
Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
	by cust-asf.ponee.io (Postfix) with SMTP id E4CE316542A
	for <archive-asf-public@cust-asf.ponee.io>; Mon, 31 Jul 2017 16:10:19 +0200 (CEST)
Received: (qmail 28536 invoked by uid 500); 31 Jul 2017 14:10:19 -0000
Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:commits-help@ambari.apache.org>
List-Unsubscribe: <mailto:commits-unsubscribe@ambari.apache.org>
List-Post: <mailto:commits@ambari.apache.org>
List-Id: <commits.ambari.apache.org>
Reply-To: ambari-dev@ambari.apache.org
Delivered-To: mailing list commits@ambari.apache.org
Received: (qmail 27326 invoked by uid 99); 31 Jul 2017 14:10:18 -0000
Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23)
    by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 31 Jul 2017 14:10:18 +0000
Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33)
	id EDBFDF3337; Mon, 31 Jul 2017 14:10:16 +0000 (UTC)
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: jonathanhurley@apache.org
To: commits@ambari.apache.org
Date: Mon, 31 Jul 2017 14:10:37 -0000
Message-Id: <1ed0359011e649dbaf7965034ba990b4@git.apache.org>
In-Reply-To: <e80c0070d730414982d80beadc2e0f9c@git.apache.org>
References: <e80c0070d730414982d80beadc2e0f9c@git.apache.org>
X-Mailer: ASF-Git Admin Mailer
Subject: [22/25] ambari git commit: AMBARI-21610. EU: Modify configs to enable
 custom extensions functionality. (swagle)
archived-at: Mon, 31 Jul 2017 14:10:21 -0000

AMBARI-21610. EU: Modify configs to enable custom extensions functionality. (swagle)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5aa398c6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5aa398c6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5aa398c6

Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 5aa398c631808c08ab433bc34a2adda1e6cb6666
Parents: 9b309e3
Author: Siddharth Wagle <swagle@hortonworks.com>
Authored: Fri Jul 28 17:04:58 2017 -0700
Committer: Siddharth Wagle <swagle@hortonworks.com>
Committed: Fri Jul 28 17:47:45 2017 -0700

----------------------------------------------------------------------
 .../4.2.5/upgrades/config-upgrade.xml           | 24 ++++++++++++++++++
 .../upgrades/nonrolling-upgrade-to-hdp-2.6.xml  | 22 ++++++++++++++++-
 .../BigInsights/4.2/upgrades/config-upgrade.xml | 26 ++++++++++++++++++++
 .../upgrades/nonrolling-upgrade-to-hdp-2.6.xml  | 22 ++++++++++++++++-
 4 files changed, 92 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/5aa398c6/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
index f71e91a..f4068a2 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
@@ -39,6 +39,7 @@
             <type>hadoop-env</type>
             <replace key="content" find="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/iop/current/hadoop-client/lib/native/Linux-amd64-64" replace-with="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/hdp/current/hadoop-client/lib/native" />
             <replace key="content" find="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}" replace-with="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}&#10;if [ -d &quot;/usr/lib/hadoop-lzo&quot; ]; then&#10;  export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/hadoop-lzo/lib/*&#10;  export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop-lzo/lib/native&#10;fi"/>
+            <insert key="content" value="{% if hadoop_custom_extensions_enabled %} export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:{{stack_root}}/current/ext/hadoop/* {% endif %}" insert-type="append" newline-after="true" newline-before="true"/>
           </definition>
           <definition xsi:type="configure" id="biginsights_4_2_namenode_adjustments">
             <type>hdfs-site</type>
@@ -51,6 +52,10 @@
             <type>hadoop-env</type>
             <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS=&quot;{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS&quot; {% endif %}" insert-type="append" newline-before="true" newline-after="true" />
           </definition>
+          <definition xsi:type="configure" id="biginsights_4_2_core_site_custom_extensions">
+            <type>core-site</type>
+            <set key="hadoop.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hadoop" if-type="core-site" if-key="hadoop.custom-extensions.root" if-key-state="absent"/>
+          </definition>
         </changes>
       </component>
     </service>
@@ -99,6 +104,14 @@
             <type>hbase-env</type>
             <set key="backup_wal_dir" value="true"/>
           </definition>
+          <definition xsi:type="configure" id="biginsights_4_2_hbase_site_custom_extensions" summary="Enable custom extension">
+            <type>hbase-site</type>
+            <set key="hbase.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hbase" if-type="hbase-site" if-key-state="absent" if-key="hbase.custom-extensions.root"/>
+          </definition>
+          <definition xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions" summary="Enable custom extensions">
+            <type>hbase-env</type>
+            <insert key="content" insert-type="append" value="# Extra Java CLASSPATH elements. Optional.&#10;export HBASE_CLASSPATH=${HBASE_CLASSPATH}:{{stack_root}}/current/ext/hbase/*" newline-after="true" newline-before="true"/>
+          </definition>
         </changes>
       </component>
     </service>
@@ -216,6 +229,17 @@
             <type>hive-site</type>
             <set key="custom.hive.warehouse.mode" value="0770" if-type="hive-site" if-key="custom.hive.warehouse.mode" if-key-state="absent"/>
           </definition>
+
+          <definition xsi:type="configure" id="biginsights_4_2_hive_site_custom_extensions" summary="Enable custom extensions in hive-site">
+            <type>hive-site</type>
+            <set key="hive.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hive" if-key-state="absent" if-type="hive-site" if-key="hive.custom-extensions.root"/>
+          </definition>
+
+          <definition xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions" summary="Enable custom extensions in hive-env">
+            <type>hive-env</type>
+            <replace key="content" find="if [ &quot;${HIVE_AUX_JARS_PATH}&quot; != &quot;&quot; ]; then" replace-with="export HIVE_AUX_JARS_PATH={{stack_root}}/current/ext/hive&#10;if [ &quot;${HIVE_AUX_JARS_PATH}&quot; != &quot;&quot; ]; then"/>
+            <replace key="content" find="if [ -f &quot;${HIVE_AUX_JARS_PATH}&quot; ]; then" replace-with="if [ -f &quot;${HIVE_AUX_JARS_PATH}&quot; ] || [ -d &quot;${HIVE_AUX_JARS_PATH}&quot; ] ; then"/>
+          </definition>
         </changes>
       </component>
       

http://git-wip-us.apache.org/repos/asf/ambari/blob/5aa398c6/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
index a7fdeec..00e8b22 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
@@ -222,6 +222,12 @@
         </task>
       </execute-stage>
 
+      <execute-stage service="HDFS" component="NAMENODE" title="Enable custom extensions in core-site">
+        <task xsi:type="configure" id="biginsights_4_2_core_site_custom_extensions">
+          <summary>Enable custom extensions in core-site</summary>
+        </task>
+      </execute-stage>
+
       <!-- core-site -->
       <execute-stage title="Fix auth_to_local mapping">
         <condition xsi:type="security" type="kerberos"/>
@@ -257,6 +263,14 @@
         <task xsi:type="configure" id="biginsights_4_2_hbase_env_config" />
       </execute-stage>
 
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Enable custom extensions for HBASE">
+        <task xsi:type="configure" id="biginsights_4_2_hbase_site_custom_extensions"/>
+      </execute-stage>
+
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Enable custom extensions for HBASE">
+        <task xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions"/>
+      </execute-stage>
+
       <!--RANGER-->
       <execute-stage service="RANGER" component="RANGER_ADMIN" title="Apply config changes for Ranger Admin">
         <task xsi:type="configure" id="hdp_2_6_0_0_remove_bind_anonymous"/>
@@ -304,7 +318,13 @@
       </execute-stage>
       <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
         <task xsi:type="configure" id="biginsights_4_2_hive_server_configure_authentication"/>
-      </execute-stage>      
+      </execute-stage>
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions for Hive Server">
+        <task xsi:type="configure" id="biginsights_4_2_hive_site_custom_extensions"/>
+      </execute-stage>
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions for Hive Server">
+        <task xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions"/>
+      </execute-stage>
       <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for Hive WebHCat server">
         <task xsi:type="configure" id="biginsights_4_2_webhcat_server_update_environment_configurations" />
       </execute-stage>

http://git-wip-us.apache.org/repos/asf/ambari/blob/5aa398c6/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
index e8ccc22..17e80b5 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
@@ -131,6 +131,7 @@
             <type>hadoop-env</type>
             <replace key="content" find="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/iop/current/hadoop-client/lib/native/Linux-amd64-64" replace-with="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/hdp/current/hadoop-client/lib/native" />
             <replace key="content" find="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}" replace-with="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}&#10;if [ -d &quot;/usr/lib/hadoop-lzo&quot; ]; then&#10;  export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/hadoop-lzo/lib/*&#10;  export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop-lzo/lib/native&#10;fi"/>
+            <insert key="content" value="{% if hadoop_custom_extensions_enabled %} export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:{{stack_root}}/current/ext/hadoop/* {% endif %}" insert-type="append" newline-after="true" newline-before="true"/>
           </definition>
           <definition xsi:type="configure" id="biginsights_4_2_namenode_adjustments">
             <type>hdfs-site</type>
@@ -143,6 +144,10 @@
             <type>hadoop-env</type>
             <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS=&quot;{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS&quot; {% endif %}" insert-type="append" newline-before="true" newline-after="true" />
           </definition>
+          <definition xsi:type="configure" id="biginsights_4_2_core_site_custom_extensions">
+            <type>core-site</type>
+            <set key="hadoop.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hadoop" if-type="core-site" if-key="hadoop.custom-extensions.root" if-key-state="absent"/>
+          </definition>
         </changes>
       </component>
     </service>
@@ -153,6 +158,7 @@
           <definition xsi:type="configure" id="biginsights_4_2_yarn_config_update" summary="Update Yarn configurations">
             <type>yarn-site</type>
             <replace key="yarn.nodemanager.aux-services" find=",spark_shuffle" replace-with=""/>
+            <insert key="yarn.application.classpath" insert-type="append" newline-after="false" newline-before="false" value=",/usr/hdp/current/ext/hadoop/*"/>
           </definition>
           <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding YARN Security ACLs">
             <type>yarn-env</type>
@@ -168,6 +174,7 @@
           <definition xsi:type="configure" id="biginsights_4_2_mapreduce_application_framework_patch" summary="Update MapReduce2 configurations">
             <type>mapred-site</type>
             <set key="mapreduce.application.framework.path" value="/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework"/>
+            <insert key="mapreduce.application.classpath" insert-type="append" value=":/usr/hdp/current/ext/hadoop/*" newline-before="false" newline-after="false"/>
           </definition>
         </changes>
       </component>
@@ -191,6 +198,14 @@
             <type>hbase-env</type>
             <set key="backup_wal_dir" value="true"/>
           </definition>
+          <definition xsi:type="configure" id="biginsights_4_2_hbase_site_custom_extensions" summary="Enable custom extensions in hbase-site">
+            <type>hbase-site</type>
+            <set key="hbase.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hbase" if-type="hbase-site" if-key-state="absent" if-key="hbase.custom-extensions.root"/>
+          </definition>
+          <definition xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions" summary="Enable custom extensions in hbase-env">
+            <type>hbase-env</type>
+            <insert key="content" insert-type="append" value="# Extra Java CLASSPATH elements. Optional.&#10;export HBASE_CLASSPATH=${HBASE_CLASSPATH}:{{stack_root}}/current/ext/hbase/*" newline-after="true" newline-before="true"/>
+          </definition>
         </changes>
       </component>
     </service>
@@ -240,6 +255,17 @@
             <type>hive-site</type>
             <set key="custom.hive.warehouse.mode" value="0770" if-type="hive-site" if-key="custom.hive.warehouse.mode" if-key-state="absent"/>
           </definition>
+
+          <definition xsi:type="configure" id="biginsights_4_2_hive_site_custom_extensions" summary="Enable custom extensions in hive-site">
+            <type>hive-site</type>
+            <set key="hive.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hive" if-key-state="absent" if-type="hive-site" if-key="hive.custom-extensions.root"/>
+          </definition>
+
+          <definition xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions" summary="Enable custom extensions in hive-env">
+            <type>hive-env</type>
+            <replace key="content" find="if [ &quot;${HIVE_AUX_JARS_PATH}&quot; != &quot;&quot; ]; then" replace-with="export HIVE_AUX_JARS_PATH={{stack_root}}/current/ext/hive&#10;if [ &quot;${HIVE_AUX_JARS_PATH}&quot; != &quot;&quot; ]; then"/>
+            <replace key="content" find="if [ -f &quot;${HIVE_AUX_JARS_PATH}&quot; ]; then" replace-with="if [ -f &quot;${HIVE_AUX_JARS_PATH}&quot; ] || [ -d &quot;${HIVE_AUX_JARS_PATH}&quot; ] ; then"/>
+          </definition>
         </changes>
       </component>
       

http://git-wip-us.apache.org/repos/asf/ambari/blob/5aa398c6/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
index ed6414d..55736cf 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
@@ -217,6 +217,12 @@
         </task>
       </execute-stage>
 
+      <execute-stage service="HDFS" component="NAMENODE" title="Enable custom extensions in core-site">
+        <task xsi:type="configure" id="biginsights_4_2_core_site_custom_extensions">
+          <summary>Enable custom extensions in core-site</summary>
+        </task>
+      </execute-stage>
+
       <!-- core-site -->
       <execute-stage title="Fix auth_to_local mapping">
         <condition xsi:type="security" type="kerberos"/>
@@ -252,6 +258,14 @@
         <task xsi:type="configure" id="biginsights_4_2_hbase_env_config" />
       </execute-stage>
 
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Enable custom extensions for HBASE">
+        <task xsi:type="configure" id="biginsights_4_2_hbase_site_custom_extensions"/>
+      </execute-stage>
+
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Enable custom extensions for HBASE">
+        <task xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions"/>
+      </execute-stage>
+
       <!-- HIVE -->
       <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
         <task xsi:type="configure" id="biginsights_4_2_hive_env_configure"/>
@@ -261,7 +275,13 @@
       </execute-stage>
       <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes for Hive Server">
         <task xsi:type="configure" id="biginsights_4_2_hive_server_configure_authentication"/>
-      </execute-stage>      
+      </execute-stage>
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions for Hive Server">
+        <task xsi:type="configure" id="biginsights_4_2_hive_site_custom_extensions"/>
+      </execute-stage>
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions for Hive Server">
+        <task xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions"/>
+      </execute-stage>
       <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes for Hive WebHCat server">
         <task xsi:type="configure" id="biginsights_4_2_webhcat_server_update_environment_configurations" />
       </execute-stage>