ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [22/25] ambari git commit: AMBARI-21610. EU: Modify configs to enable custom extensions functionality. (swagle)
Date Mon, 31 Jul 2017 14:10:37 GMT
AMBARI-21610. EU: Modify configs to enable custom extensions functionality. (swagle)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5aa398c6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5aa398c6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5aa398c6

Branch: refs/heads/branch-feature-AMBARI-21450
Commit: 5aa398c631808c08ab433bc34a2adda1e6cb6666
Parents: 9b309e3
Author: Siddharth Wagle <swagle@hortonworks.com>
Authored: Fri Jul 28 17:04:58 2017 -0700
Committer: Siddharth Wagle <swagle@hortonworks.com>
Committed: Fri Jul 28 17:47:45 2017 -0700

----------------------------------------------------------------------
 .../4.2.5/upgrades/config-upgrade.xml           | 24 ++++++++++++++++++
 .../upgrades/nonrolling-upgrade-to-hdp-2.6.xml  | 22 ++++++++++++++++-
 .../BigInsights/4.2/upgrades/config-upgrade.xml | 26 ++++++++++++++++++++
 .../upgrades/nonrolling-upgrade-to-hdp-2.6.xml  | 22 ++++++++++++++++-
 4 files changed, 92 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/5aa398c6/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
index f71e91a..f4068a2 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/config-upgrade.xml
@@ -39,6 +39,7 @@
             <type>hadoop-env</type>
             <replace key="content" find="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/iop/current/hadoop-client/lib/native/Linux-amd64-64"
replace-with="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/hdp/current/hadoop-client/lib/native"
/>
             <replace key="content" find="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}"
replace-with="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}&#10;if
[ -d &quot;/usr/lib/hadoop-lzo&quot; ]; then&#10;  export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/hadoop-lzo/lib/*&#10;
 export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop-lzo/lib/native&#10;fi"/>
+            <insert key="content" value="{% if hadoop_custom_extensions_enabled %} export
HADOOP_CLASSPATH=$HADOOP_CLASSPATH:{{stack_root}}/current/ext/hadoop/* {% endif %}" insert-type="append"
newline-after="true" newline-before="true"/>
           </definition>
           <definition xsi:type="configure" id="biginsights_4_2_namenode_adjustments">
             <type>hdfs-site</type>
@@ -51,6 +52,10 @@
             <type>hadoop-env</type>
             <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS=&quot;{{hadoop_zkfc_opts}}
$HADOOP_ZKFC_OPTS&quot; {% endif %}" insert-type="append" newline-before="true" newline-after="true"
/>
           </definition>
+          <definition xsi:type="configure" id="biginsights_4_2_core_site_custom_extensions">
+            <type>core-site</type>
+            <set key="hadoop.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hadoop"
if-type="core-site" if-key="hadoop.custom-extensions.root" if-key-state="absent"/>
+          </definition>
         </changes>
       </component>
     </service>
@@ -99,6 +104,14 @@
             <type>hbase-env</type>
             <set key="backup_wal_dir" value="true"/>
           </definition>
+          <definition xsi:type="configure" id="biginsights_4_2_hbase_site_custom_extensions"
summary="Enable custom extension">
+            <type>hbase-site</type>
+            <set key="hbase.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hbase"
if-type="hbase-site" if-key-state="absent" if-key="hbase.custom-extensions.root"/>
+          </definition>
+          <definition xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions"
summary="Enable custom extensions">
+            <type>hbase-env</type>
+            <insert key="content" insert-type="append" value="# Extra Java CLASSPATH elements.
Optional.&#10;export HBASE_CLASSPATH=${HBASE_CLASSPATH}:{{stack_root}}/current/ext/hbase/*"
newline-after="true" newline-before="true"/>
+          </definition>
         </changes>
       </component>
     </service>
@@ -216,6 +229,17 @@
             <type>hive-site</type>
             <set key="custom.hive.warehouse.mode" value="0770" if-type="hive-site" if-key="custom.hive.warehouse.mode"
if-key-state="absent"/>
           </definition>
+
+          <definition xsi:type="configure" id="biginsights_4_2_hive_site_custom_extensions"
summary="Enable custom extensions in hive-site">
+            <type>hive-site</type>
+            <set key="hive.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hive"
if-key-state="absent" if-type="hive-site" if-key="hive.custom-extensions.root"/>
+          </definition>
+
+          <definition xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions"
summary="Enable custom extensions in hive-env">
+            <type>hive-env</type>
+            <replace key="content" find="if [ &quot;${HIVE_AUX_JARS_PATH}&quot;
!= &quot;&quot; ]; then" replace-with="export HIVE_AUX_JARS_PATH={{stack_root}}/current/ext/hive&#10;if
[ &quot;${HIVE_AUX_JARS_PATH}&quot; != &quot;&quot; ]; then"/>
+            <replace key="content" find="if [ -f &quot;${HIVE_AUX_JARS_PATH}&quot;
]; then" replace-with="if [ -f &quot;${HIVE_AUX_JARS_PATH}&quot; ] || [ -d &quot;${HIVE_AUX_JARS_PATH}&quot;
] ; then"/>
+          </definition>
         </changes>
       </component>
       

http://git-wip-us.apache.org/repos/asf/ambari/blob/5aa398c6/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
index a7fdeec..00e8b22 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
@@ -222,6 +222,12 @@
         </task>
       </execute-stage>
 
+      <execute-stage service="HDFS" component="NAMENODE" title="Enable custom extensions
in core-site">
+        <task xsi:type="configure" id="biginsights_4_2_core_site_custom_extensions">
+          <summary>Enable custom extensions in core-site</summary>
+        </task>
+      </execute-stage>
+
       <!-- core-site -->
       <execute-stage title="Fix auth_to_local mapping">
         <condition xsi:type="security" type="kerberos"/>
@@ -257,6 +263,14 @@
         <task xsi:type="configure" id="biginsights_4_2_hbase_env_config" />
       </execute-stage>
 
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Enable custom extensions
for HBASE">
+        <task xsi:type="configure" id="biginsights_4_2_hbase_site_custom_extensions"/>
+      </execute-stage>
+
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Enable custom extensions
for HBASE">
+        <task xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions"/>
+      </execute-stage>
+
       <!--RANGER-->
       <execute-stage service="RANGER" component="RANGER_ADMIN" title="Apply config changes
for Ranger Admin">
         <task xsi:type="configure" id="hdp_2_6_0_0_remove_bind_anonymous"/>
@@ -304,7 +318,13 @@
       </execute-stage>
       <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes
for Hive Server">
         <task xsi:type="configure" id="biginsights_4_2_hive_server_configure_authentication"/>
-      </execute-stage>      
+      </execute-stage>
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions
for Hive Server">
+        <task xsi:type="configure" id="biginsights_4_2_hive_site_custom_extensions"/>
+      </execute-stage>
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions
for Hive Server">
+        <task xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions"/>
+      </execute-stage>
       <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes
for Hive WebHCat server">
         <task xsi:type="configure" id="biginsights_4_2_webhcat_server_update_environment_configurations"
/>
       </execute-stage>

http://git-wip-us.apache.org/repos/asf/ambari/blob/5aa398c6/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
index e8ccc22..17e80b5 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/config-upgrade.xml
@@ -131,6 +131,7 @@
             <type>hadoop-env</type>
             <replace key="content" find="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/iop/current/hadoop-client/lib/native/Linux-amd64-64"
replace-with="export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/hdp/current/hadoop-client/lib/native"
/>
             <replace key="content" find="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}"
replace-with="export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}&#10;if
[ -d &quot;/usr/lib/hadoop-lzo&quot; ]; then&#10;  export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/usr/lib/hadoop-lzo/lib/*&#10;
 export JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}:/usr/lib/hadoop-lzo/lib/native&#10;fi"/>
+            <insert key="content" value="{% if hadoop_custom_extensions_enabled %} export
HADOOP_CLASSPATH=$HADOOP_CLASSPATH:{{stack_root}}/current/ext/hadoop/* {% endif %}" insert-type="append"
newline-after="true" newline-before="true"/>
           </definition>
           <definition xsi:type="configure" id="biginsights_4_2_namenode_adjustments">
             <type>hdfs-site</type>
@@ -143,6 +144,10 @@
             <type>hadoop-env</type>
             <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS=&quot;{{hadoop_zkfc_opts}}
$HADOOP_ZKFC_OPTS&quot; {% endif %}" insert-type="append" newline-before="true" newline-after="true"
/>
           </definition>
+          <definition xsi:type="configure" id="biginsights_4_2_core_site_custom_extensions">
+            <type>core-site</type>
+            <set key="hadoop.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hadoop"
if-type="core-site" if-key="hadoop.custom-extensions.root" if-key-state="absent"/>
+          </definition>
         </changes>
       </component>
     </service>
@@ -153,6 +158,7 @@
           <definition xsi:type="configure" id="biginsights_4_2_yarn_config_update" summary="Update
Yarn configurations">
             <type>yarn-site</type>
             <replace key="yarn.nodemanager.aux-services" find=",spark_shuffle" replace-with=""/>
+            <insert key="yarn.application.classpath" insert-type="append" newline-after="false"
newline-before="false" value=",/usr/hdp/current/ext/hadoop/*"/>
           </definition>
           <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding
YARN Security ACLs">
             <type>yarn-env</type>
@@ -168,6 +174,7 @@
           <definition xsi:type="configure" id="biginsights_4_2_mapreduce_application_framework_patch"
summary="Update MapReduce2 configurations">
             <type>mapred-site</type>
             <set key="mapreduce.application.framework.path" value="/hdp/apps/${hdp.version}/mapreduce/mapreduce.tar.gz#mr-framework"/>
+            <insert key="mapreduce.application.classpath" insert-type="append" value=":/usr/hdp/current/ext/hadoop/*"
newline-before="false" newline-after="false"/>
           </definition>
         </changes>
       </component>
@@ -191,6 +198,14 @@
             <type>hbase-env</type>
             <set key="backup_wal_dir" value="true"/>
           </definition>
+          <definition xsi:type="configure" id="biginsights_4_2_hbase_site_custom_extensions"
summary="Enable custom extensions in hbase-site">
+            <type>hbase-site</type>
+            <set key="hbase.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hbase"
if-type="hbase-site" if-key-state="absent" if-key="hbase.custom-extensions.root"/>
+          </definition>
+          <definition xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions"
summary="Enable custom extensions in hbase-env">
+            <type>hbase-env</type>
+            <insert key="content" insert-type="append" value="# Extra Java CLASSPATH elements.
Optional.&#10;export HBASE_CLASSPATH=${HBASE_CLASSPATH}:{{stack_root}}/current/ext/hbase/*"
newline-after="true" newline-before="true"/>
+          </definition>
         </changes>
       </component>
     </service>
@@ -240,6 +255,17 @@
             <type>hive-site</type>
             <set key="custom.hive.warehouse.mode" value="0770" if-type="hive-site" if-key="custom.hive.warehouse.mode"
if-key-state="absent"/>
           </definition>
+
+          <definition xsi:type="configure" id="biginsights_4_2_hive_site_custom_extensions"
summary="Enable custom extensions in hive-site">
+            <type>hive-site</type>
+            <set key="hive.custom-extensions.root" value="/hdp/ext/{{major_stack_version}}/hive"
if-key-state="absent" if-type="hive-site" if-key="hive.custom-extensions.root"/>
+          </definition>
+
+          <definition xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions"
summary="Enable custom extensions in hive-env">
+            <type>hive-env</type>
+            <replace key="content" find="if [ &quot;${HIVE_AUX_JARS_PATH}&quot;
!= &quot;&quot; ]; then" replace-with="export HIVE_AUX_JARS_PATH={{stack_root}}/current/ext/hive&#10;if
[ &quot;${HIVE_AUX_JARS_PATH}&quot; != &quot;&quot; ]; then"/>
+            <replace key="content" find="if [ -f &quot;${HIVE_AUX_JARS_PATH}&quot;
]; then" replace-with="if [ -f &quot;${HIVE_AUX_JARS_PATH}&quot; ] || [ -d &quot;${HIVE_AUX_JARS_PATH}&quot;
] ; then"/>
+          </definition>
         </changes>
       </component>
       

http://git-wip-us.apache.org/repos/asf/ambari/blob/5aa398c6/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
index ed6414d..55736cf 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
@@ -217,6 +217,12 @@
         </task>
       </execute-stage>
 
+      <execute-stage service="HDFS" component="NAMENODE" title="Enable custom extensions
in core-site">
+        <task xsi:type="configure" id="biginsights_4_2_core_site_custom_extensions">
+          <summary>Enable custom extensions in core-site</summary>
+        </task>
+      </execute-stage>
+
       <!-- core-site -->
       <execute-stage title="Fix auth_to_local mapping">
         <condition xsi:type="security" type="kerberos"/>
@@ -252,6 +258,14 @@
         <task xsi:type="configure" id="biginsights_4_2_hbase_env_config" />
       </execute-stage>
 
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Enable custom extensions
for HBASE">
+        <task xsi:type="configure" id="biginsights_4_2_hbase_site_custom_extensions"/>
+      </execute-stage>
+
+      <execute-stage service="HBASE" component="HBASE_MASTER" title="Enable custom extensions
for HBASE">
+        <task xsi:type="configure" id="biginsights_4_2_hbase_env_custom_extensions"/>
+      </execute-stage>
+
       <!-- HIVE -->
       <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes
for Hive Server">
         <task xsi:type="configure" id="biginsights_4_2_hive_env_configure"/>
@@ -261,7 +275,13 @@
       </execute-stage>
       <execute-stage service="HIVE" component="HIVE_SERVER" title="Apply config changes
for Hive Server">
         <task xsi:type="configure" id="biginsights_4_2_hive_server_configure_authentication"/>
-      </execute-stage>      
+      </execute-stage>
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions
for Hive Server">
+        <task xsi:type="configure" id="biginsights_4_2_hive_site_custom_extensions"/>
+      </execute-stage>
+      <execute-stage service="HIVE" component="HIVE_SERVER" title="Enable custom extensions
for Hive Server">
+        <task xsi:type="configure" id="biginsights_4_2_hive_env_custom_extensions"/>
+      </execute-stage>
       <execute-stage service="HIVE" component="WEBHCAT_SERVER" title="Apply config changes
for Hive WebHCat server">
         <task xsi:type="configure" id="biginsights_4_2_webhcat_server_update_environment_configurations"
/>
       </execute-stage>


Mime
View raw message