ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rz...@apache.org
Subject [46/51] [abbrv] ambari git commit: AMBARI-19879. Updating yarn-env and hadoop-env templates with ZK secure options on stack upgrade (Attila Magyar via magyari_sandor)
Date Fri, 17 Feb 2017 22:57:24 GMT
AMBARI-19879. Updating yarn-env and hadoop-env templates with ZK secure options on stack upgrade
(Attila Magyar via magyari_sandor)

(cherry picked from commit 8ef31458a989dec5fb2b7f35223dd689fdeaba7b)

Change-Id: I4f77c3631df581e1d9255bdce16f6ebcaeed029f


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/47d94bf5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/47d94bf5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/47d94bf5

Branch: refs/heads/branch-feature-BUG-74026
Commit: 47d94bf5ed3d8025b5041f6d2cbc3444ed529d3b
Parents: 664c3ba
Author: Attila Magyar <amagyar@hortonworks.com>
Authored: Tue Feb 7 14:42:03 2017 +0100
Committer: Zuul <releng@hortonworks.com>
Committed: Tue Feb 7 10:29:04 2017 -0800

----------------------------------------------------------------------
 .../HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py      |  3 +--
 .../stacks/HDP/2.3/upgrades/config-upgrade.xml        |  9 +++++++++
 .../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml       | 12 ++++++++++++
 .../resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml |  2 ++
 .../stacks/HDP/2.4/upgrades/config-upgrade.xml        |  8 ++++++++
 .../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml       | 14 ++++++++++++++
 .../resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml |  2 ++
 .../stacks/HDP/2.5/upgrades/config-upgrade.xml        |  8 ++++++++
 .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml       | 13 +++++++++++++
 .../resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml |  2 ++
 10 files changed, 71 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
index 275d349..74b72c7 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py
@@ -39,8 +39,6 @@ from resource_management.libraries.functions.stack_features import check_stack_f
 from resource_management.libraries.script import Script
 from resource_management.core.resources.zkmigrator import ZkMigrator
 
-
-
 class ZkfcSlave(Script):
   def get_component_name(self):
     import params
@@ -62,6 +60,7 @@ class ZkfcSlave(Script):
     import params
     env.set_params(params)
     hdfs("zkfc_slave")
+    utils.set_up_zkfc_security(params)
     pass
 
 @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)

http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
index f3e8ddd..f86b03d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml
@@ -375,6 +375,11 @@
             <regex-replace  key="content" find="hadoop.security.log.maxfilesize=([0-9]+)MB"
replace-with="hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB"/>
             <regex-replace  key="content" find="hadoop.security.log.maxbackupindex=([0-9]+)"
replace-with="hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}"/>
           </definition>
+
+          <definition xsi:type="configure" id="hadoop_env_zkfc_security_opts" summary="Adding
HDFS ZKFC Security ACLs">
+            <type>hadoop-env</type>
+            <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS=&quot;{{hadoop_zkfc_opts}}
$HADOOP_ZKFC_OPTS&quot; {% endif %}" insert-type="append" newline-before="true" newline-after="true"
/>
+          </definition>
         </changes>
       </component>
     </service>
@@ -504,6 +509,10 @@
             <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxFileSize=([0-9]+)MB"
replace-with="log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB"/>
             <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxBackupIndex=([0-9]+)"
replace-with="log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}"/>
           </definition>
+          <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding
YARN Security ACLs">
+            <type>yarn-env</type>
+            <insert key="content" value="{% if rm_security_opts is defined %} YARN_OPTS=&quot;{{rm_security_opts}}
$YARN_OPTS&quot; {% endif %}" insert-type="append" newline-before="true" newline-after="true"
/>
+          </definition>
         </changes>
       </component>
     </service>

http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
index dc53c67..8b7451e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml
@@ -286,6 +286,12 @@
         </task>
       </execute-stage>
 
+      <execute-stage service="HDFS" component="NAMENODE" title="Adding HDFS ZKFC Security
ACLs">
+        <task xsi:type="configure" id="hadoop_env_zkfc_security_opts">
+          <summary>Adding HDFS ZKFC Security ACLs</summary>
+        </task>
+      </execute-stage>
+
       <!-- YARN -->
       <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn
Properties for Spark">
         <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig">
@@ -293,6 +299,12 @@
         </task>
       </execute-stage>
 
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Adding YARN Security
ACLs">
+        <task xsi:type="configure" id="yarn_env_security_opts">
+          <summary>Adding YARN Security ACLs</summary>
+        </task>
+      </execute-stage>
+
       <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes
for Yarn Resourcemanager">
         <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"/>
       </execute-stage>

http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
index 290d3c5..01fc102 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml
@@ -680,6 +680,7 @@
           <task xsi:type="configure" id="hdp_2_4_0_0_namenode_ha_adjustments"/>
           <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db" />
           <task xsi:type="configure" id="hdfs_log4j_parameterize" />
+          <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" />
         </pre-upgrade>
 
         <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
@@ -760,6 +761,7 @@
           </task>
           <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db" />
           <task xsi:type="configure" id="yarn_log4j_parameterize" />
+          <task xsi:type="configure" id="yarn_env_security_opts" />
         </pre-upgrade>
         <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
index 6a462ec..57227f6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml
@@ -264,6 +264,10 @@
             <regex-replace  key="content" find="hadoop.security.log.maxfilesize=([0-9]+)MB"
replace-with="hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB"/>
             <regex-replace  key="content" find="hadoop.security.log.maxbackupindex=([0-9]+)"
replace-with="hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}"/>
           </definition>
+          <definition xsi:type="configure" id="hadoop_env_zkfc_security_opts" summary="Adding
HDFS ZKFC Security ACLs">
+            <type>hadoop-env</type>
+            <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS=&quot;{{hadoop_zkfc_opts}}
$HADOOP_ZKFC_OPTS&quot; {% endif %}" insert-type="append" newline-before="true" newline-after="true"
/>
+          </definition>
         </changes>
       </component>
     </service>
@@ -289,6 +293,10 @@
             <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxFileSize=([0-9]+)MB"
replace-with="log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB"/>
             <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxBackupIndex=([0-9]+)"
replace-with="log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}"/>
           </definition>
+          <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding
YARN Security ACLs">
+            <type>yarn-env</type>
+            <insert key="content" value="{% if rm_security_opts is defined %} YARN_OPTS=&quot;{{rm_security_opts}}
$YARN_OPTS&quot; {% endif %}" insert-type="append" newline-before="true" newline-after="true"
/>
+          </definition>
         </changes>
       </component>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
index 8a988cc..5661641 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml
@@ -281,6 +281,13 @@
         </task>
       </execute-stage>
 
+      <!--Yarn-->
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Adding YARN Security
ACLs">
+        <task xsi:type="configure" id="yarn_env_security_opts">
+          <summary>Adding YARN Security ACLs</summary>
+        </task>
+      </execute-stage>
+
       <!-- YARN -->
       <execute-stage service="YARN" component="NODEMANAGER" title="Add Spark2 shuffle">
         <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/>
@@ -400,6 +407,13 @@
         </task>
       </execute-stage>
 
+      <!--HDFS-->
+      <execute-stage service="HDFS" component="NAMENODE" title="Adding HDFS ZKFC Security
ACLs">
+        <task xsi:type="configure" id="hadoop_env_zkfc_security_opts">
+          <summary>Adding HDFS ZKFC Security ACLs</summary>
+        </task>
+      </execute-stage>
+
       <!-- SQOOP -->
       <execute-stage service="SQOOP" component="SQOOP" title="Apply config changes for
Sqoop to remove Atlas Configs">
         <!-- Remove Atlas configs that were incorrectly added to sqoop-site instead of
Atlas' application.properties. -->

http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
index cadef73..7b12af5 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml
@@ -685,6 +685,7 @@
           <task xsi:type="configure" id="hdp_2_5_0_0_namenode_ha_adjustments"/>
           <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db" />
           <task xsi:type="configure" id="hdfs_log4j_parameterize" />
+          <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" />
         </pre-upgrade>
 
         <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
@@ -765,6 +766,7 @@
           </task>
           <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db" />
           <task xsi:type="configure" id="yarn_log4j_parameterize" />
+          <task xsi:type="configure" id="yarn_env_security_opts" />
         </pre-upgrade>
         <pre-downgrade /> <!--  no-op to prevent config changes on downgrade -->
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
index 188e6f4..54a824d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml
@@ -128,6 +128,10 @@
           <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxFileSize=([0-9]+)MB"
replace-with="log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB"/>
           <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxBackupIndex=([0-9]+)"
replace-with="log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}"/>
         </definition>
+        <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding YARN
Security ACLs">
+          <type>yarn-env</type>
+          <insert key="content" value="{% if rm_security_opts is defined %} YARN_OPTS=&quot;{{rm_security_opts}}
$YARN_OPTS&quot; {% endif %}" insert-type="append" newline-before="true" newline-after="true"
/>
+        </definition>
       </changes>
     </component>
   </service>
@@ -146,6 +150,10 @@
           <regex-replace  key="content" find="hadoop.security.log.maxfilesize=([0-9]+)MB"
replace-with="hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB"/>
           <regex-replace  key="content" find="hadoop.security.log.maxbackupindex=([0-9]+)"
replace-with="hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}"/>
         </definition>
+        <definition xsi:type="configure" id="hadoop_env_zkfc_security_opts" summary="Adding
HDFS ZKFC Security ACLs">
+          <type>hadoop-env</type>
+          <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS=&quot;{{hadoop_zkfc_opts}}
$HADOOP_ZKFC_OPTS&quot; {% endif %}" insert-type="append" newline-before="true" newline-after="true"
/>
+        </definition>
       </changes>
     </component>
   </service>

http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
index 1fad885..7bb679e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml
@@ -291,6 +291,13 @@
         </task>
       </execute-stage>
 
+      <!--Yarn-->
+      <execute-stage service="YARN" component="RESOURCEMANAGER" title="Adding YARN Security
ACLs">
+        <task xsi:type="configure" id="yarn_env_security_opts">
+          <summary>Adding YARN Security ACLs</summary>
+        </task>
+      </execute-stage>
+
       <!--TEZ-->
       <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path
for Tez">
         <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath">
@@ -317,6 +324,12 @@
         </task>
       </execute-stage>
 
+      <execute-stage service="HDFS" component="NAMENODE" title="Adding HDFS ZKFC Security
ACLs">
+        <task xsi:type="configure" id="hadoop_env_zkfc_security_opts">
+          <summary>Adding HDFS ZKFC Security ACLs</summary>
+        </task>
+      </execute-stage>
+
       <!--HBASE-->
       <execute-stage service="HBASE" component="HBASE_MASTER" title="Parameterizing HBase
Log4J Properties">
         <task xsi:type="configure" id="hbase_log4j_parameterize">

http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
index 291397a..2f07c97 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml
@@ -589,6 +589,7 @@
       <component name="NAMENODE">
         <pre-upgrade>
           <task xsi:type="configure" id="hdfs_log4j_parameterize" />
+          <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" />
         </pre-upgrade>
         <pre-downgrade />
         <upgrade>
@@ -667,6 +668,7 @@
       <component name="RESOURCEMANAGER">
         <pre-upgrade>
           <task xsi:type="configure" id="yarn_log4j_parameterize" />
+          <task xsi:type="configure" id="yarn_env_security_opts" />
         </pre-upgrade>
         <pre-downgrade />
         <upgrade>


Mime
View raw message