ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject ambari git commit: AMBARI-11289. Ranger HDFS/HBase Plugin Upgrade Pack For HDP-2.2 To HDP-2.3 (ncole)
Date Thu, 21 May 2015 17:29:44 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 9371bb760 -> 8e3433c82


AMBARI-11289. Ranger HDFS/HBase Plugin Upgrade Pack For HDP-2.2 To HDP-2.3 (ncole)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8e3433c8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8e3433c8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8e3433c8

Branch: refs/heads/trunk
Commit: 8e3433c82063a0308596db37a940a6e264ca45de
Parents: 9371bb7
Author: Nate Cole <ncole@hortonworks.com>
Authored: Wed May 20 20:12:36 2015 -0400
Committer: Nate Cole <ncole@hortonworks.com>
Committed: Thu May 21 13:29:30 2015 -0400

----------------------------------------------------------------------
 .../serveraction/upgrades/ConfigureAction.java  |  14 ++-
 .../state/stack/upgrade/ConfigureTask.java      |  17 ++-
 .../stacks/HDP/2.2/upgrades/upgrade-2.3.xml     | 112 ++++++++++++++++++-
 .../upgrades/ConfigureActionTest.java           |   4 +
 4 files changed, 140 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/8e3433c8/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
index c5f33c4..69a03f5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
@@ -275,7 +275,7 @@ public class ConfigureAction extends AbstractServerAction {
 
             // append standard output
             outputBuffer.append(MessageFormat.format("Created {0}/{1} = {2}\n", configType,
-                transfer.toKey, valueToCopy));
+                transfer.toKey, mask(transfer, valueToCopy)));
           }
           break;
         case MOVE:
@@ -295,7 +295,7 @@ public class ConfigureAction extends AbstractServerAction {
 
             // append standard output
             outputBuffer.append(MessageFormat.format("Created {0}/{1} with default value
{2}\n",
-                configType, transfer.toKey, transfer.defaultValue));
+                configType, transfer.toKey, mask(transfer, transfer.defaultValue)));
           }
 
           break;
@@ -370,7 +370,7 @@ public class ConfigureAction extends AbstractServerAction {
           // without a key/value to set
           newValues.put(key, value);
           outputBuffer.append(MessageFormat.format("{0}/{1} changed to {2}\n", configType,
key,
-              value));
+              mask(keyValuePair, value)));
         }
       }
     }
@@ -506,4 +506,12 @@ public class ConfigureAction extends AbstractServerAction {
 
     return result;
   }
+
+  private static String mask(ConfigureTask.Masked mask, String value) {
+    if (mask.mask) {
+      return StringUtils.repeat("*", value.length());
+    }
+    return value;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8e3433c8/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
index 99d6058..f5a77c5 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ConfigureTask.java
@@ -154,11 +154,22 @@ public class ConfigureTask extends ServerSideActionTask {
   }
 
   /**
+   * Used for configuration updates that should mask their values from being
+   * printed in plain text.
+   */
+  @XmlAccessorType(XmlAccessType.FIELD)
+  public static class Masked {
+    @XmlAttribute(name = "mask")
+    public boolean mask = false;
+  }
+
+
+  /**
    * A key/value pair to set in the type specified by {@link ConfigureTask#type}
    */
   @XmlAccessorType(XmlAccessType.FIELD)
   @XmlType(name = "set")
-  public static class ConfigurationKeyValue {
+  public static class ConfigurationKeyValue extends Masked {
     @XmlAttribute(name = "key")
     public String key;
 
@@ -197,7 +208,7 @@ public class ConfigureTask extends ServerSideActionTask {
    */
   @XmlAccessorType(XmlAccessType.FIELD)
   @XmlType(name = "transfer")
-  public static class Transfer {
+  public static class Transfer extends Masked {
     /**
      * The type of operation, such as COPY or DELETE.
      */
@@ -291,7 +302,7 @@ public class ConfigureTask extends ServerSideActionTask {
    */
   @XmlAccessorType(XmlAccessType.FIELD)
   @XmlType(name = "replace")
-  public static class Replace {
+  public static class Replace extends Masked {
     /**
      * The key name
      */

http://git-wip-us.apache.org/repos/asf/ambari/blob/8e3433c8/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
index 7505a07..ddaee86 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
@@ -90,7 +90,6 @@
       <service name="HBASE">
         <component>HBASE_MASTER</component>
       </service>
-
     </group>
 
     <group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
@@ -332,6 +331,60 @@
               <value>org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer</value>
             </condition>
           </task>
+          <task xsi:type="configure" summary="Transitioning Ranger HDFS Policy">
+            <type>ranger-hdfs-policymgr-ssl</type>
+            <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="SSL_KEYSTORE_FILE_PATH"
to-key="xasecure.policymgr.clientssl.keystore" default-value="/usr/hdp/current/hadoop-client/conf/ranger-plugin-keystore.jks"
/>
+            <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="SSL_KEYSTORE_PASSWORD"
to-key="xasecure.policymgr.clientssl.keystore.password" mask="true" default-value="myKeyFilePassword"
/>
+            <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="SSL_TRUSTSTORE_FILE_PATH"
to-key="xasecure.policymgr.clientssl.truststore" default-value="/usr/hdp/current/hadoop-client/conf/ranger-plugin-truststore.jks"
/>
+            <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="SSL_TRUSTSTORE_PASSWORD"
to-key="xasecure.policymgr.clientssl.truststore.password" mask="true" default-value="changeit"
/>
+          </task>
+          <task xsi:type="configure" summary="Transitioning Ranger HDFS Audit">
+            <type>ranger-hdfs-audit</type>
+            <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.DB.IS_ENABLED"
to-key="xasecure.audit.destination.db" default-value="false"/>
+            <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY"
to-key="xasecure.audit.destination.hdfs.dir" default-value="hdfs://{{namenode_hostname}}:8020/ranger/audit"
/>
+            <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.HDFS.IS_ENABLED"
to-key="xasecure.audit.destination.hdfs" default-value="true" />
+            <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY"
to-key="xasecure.audit.destination.hdfs.batch.filespool.dir" default-value="/var/log/hadoop/hdfs/audit/hdfs/spool"
/>
+            <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.DB.USER_NAME"
to-key="xasecure.audit.destination.db.user" default-value="" />
+            <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="XAAUDIT.DB.PASSWORD"
to-key="xasecure.audit.destination.db.password" mask="true" default-value="" />
+            <set key="xasecure.audit.destination.solr" value="true" />
+            <set key="xasecure.audit.destination.solr.urls" value="{{ranger_audit_solr_urls}}"
/>
+            <set key="xasecure.audit.destination.solr.zookeepers" value="" />
+            <set key="xasecure.audit.destination.solr.batch.filespool.dir" value="/var/logs/hdfs/audit/solr/spool"
/>
+            <set key="xasecure.audit.destination.db.jdbc.driver" value="{{jdbc_driver}}"
/>
+            <set key="xasecure.audit.destination.db.jdbc.url" value="{{audit_jdbc_url}}"
/>
+            <set key="xasecure.audit.provider.summary.enabled" value="true" />
+          </task>
+          
+          <task xsi:type="configure" summary="Transitioning Ranger HDFS Security">
+            <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="REPOSITORY_NAME"
to-key="ranger.plugin.hdfs.service.name" default-value="{{repo_name}}" />
+            <transfer operation="copy" from-type="ranger-hdfs-plugin-properties" from-key="POLICY_MGR_URL"
to-key="ranger.plugin.hdfs.policy.rest.url" default-value="{{policymgr_mgr_url}}" />
+          </task>
+          
+          <task xsi:type="configure">
+            <type>ranger-hdfs-plugin-properties</type>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_FILE" />
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_FILE" />
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS"
/>
+            <transfer operation="delete" delete-key="SSL_KEYSTORE_FILE_PATH" />
+            <transfer operation="delete" delete-key="SSL_KEYSTORE_PASSWORD" />
+            <transfer operation="delete" delete-key="SSL_TRUSTSTORE_FILE_PATH" />
+            <transfer operation="delete" delete-key="SSL_TRUSTSTORE_PASSWORD" />
+            <transfer operation="delete" delete-key="XAAUDIT.DB.IS_ENABLED" />
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.IS_ENABLED" />
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.DB.USER_NAME" />
+            <transfer operation="delete" delete-key="XAAUDIT.DB.PASSWORD" />
+            <transfer operation="delete" delete-key="REPOSITORY_NAME" />
+            <transfer operation="delete" delete-key="POLICY_MGR_URL" />
+          </task>
         </pre-upgrade>
         <upgrade>
           <task xsi:type="restart" />
@@ -427,6 +480,63 @@
 
     <service name="HBASE">
       <component name="HBASE_MASTER">
+        <pre-upgrade>
+          <task xsi:type="configure" summary="Transitioning Ranger HBase Policy">
+            <type>ranger-hbase-policymgr-ssl</type>
+            <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="SSL_KEYSTORE_FILE_PATH"
to-key="xasecure.policymgr.clientssl.keystore" default-value="/usr/hdp/current/hbase-client/conf/ranger-plugin-keystore.jks"
/>
+            <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="SSL_KEYSTORE_PASSWORD"
to-key="xasecure.policymgr.clientssl.keystore.password" mask="true" default-value="myKeyFilePassword"
/>
+            <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="SSL_TRUSTSTORE_FILE_PATH"
to-key="xasecure.policymgr.clientssl.truststore" default-value="/usr/hdp/current/hbase-client/conf/ranger-plugin-truststore.jks"
/>
+            <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="SSL_TRUSTSTORE_PASSWORD"
to-key="xasecure.policymgr.clientssl.truststore.password" mask="true" default-value="changeit"
/>
+          </task>
+          <task xsi:type="configure" summary="Transitioning Ranger HBase Audit">
+            <type>ranger-hbase-audit</type>
+            <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.DB.IS_ENABLED"
to-key="xasecure.audit.destination.db" default-value="false" />
+            <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY"
to-key="xasecure.audit.destination.hdfs.dir" default-value="hdfs://{{namenode_hostname}}:8020/ranger/audit"
/>
+            <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.HDFS.IS_ENABLED"
to-key="xasecure.audit.destination.hdfs" default-value="true" />
+            <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY"
to-key="xasecure.audit.destination.hdfs.batch.filespool.dir" default-value="/var/logs/hadoop/hdfs/audit/hdfs/spool"
/>
+            <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.DB.USER_NAME"
to-key="xasecure.audit.destination.db.user" default-value="" />
+            <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="XAAUDIT.DB.PASSWORD"
to-key="xasecure.audit.destination.db.password" mask="true" default-value="" />
+            <set key="xasecure.audit.destination.solr" value="true" />
+            <set key="xasecure.audit.destination.solr.urls" value="{{ranger_audit_solr_urls}}"
/>
+            <set key="xasecure.audit.destination.solr.zookeepers" value="" />
+            <set key="xasecure.audit.destination.solr.batch.filespool.dir" value="/var/logs/hbase/audit/solr/spool"
/>
+            <set key="xasecure.audit.destination.db.jdbc.driver" value="{{jdbc_driver}}"
/>
+            <set key="xasecure.audit.destination.db.jdbc.url" value="{{audit_jdbc_url}}"
/>
+            <set key="xasecure.audit.provider.summary.enabled" value="true" />
+          </task>
+          <task xsi:type="configure">
+            <type>ranger-hbase-security</type>
+            <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="UPDATE_XAPOLICIES_ON_GRANT_REVOKE"
to-key="xasecure.hbase.update.xapolicies.on.grant.revoke" default-value="true" />
+            <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="POLICY_MGR_URL"
to-key="ranger.plugin.hbase.policy.rest.url" default-value="{{policymgr_mgr_url}}" />
+            <transfer operation="copy" from-type="ranger-hbase-plugin-properties" from-key="REPOSITORY_NAME"
to-key="ranger.plugin.hbase.service.name" default-value="{{repo_name}}" />
+          </task>
+          <task xsi:type="configure">
+            <type>ranger-hbase-plugin-properties</type>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_FILE" />
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_FILE" />
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS"
/>
+            <transfer operation="delete" delete-key="SQL_CONNECTOR_JAR" />
+            <transfer operation="delete" delete-key="REPOSITORY_NAME" />
+            <transfer operation="delete" delete-key="POLICY_MGR_URL" />
+            <transfer operation="delete" delete-key="UPDATE_XAPOLICIES_ON_GRANT_REVOKE"
/>
+            <transfer operation="delete" delete-key="SSL_KEYSTORE_FILE_PATH" />
+            <transfer operation="delete" delete-key="SSL_KEYSTORE_PASSWORD" />
+            <transfer operation="delete" delete-key="SSL_TRUSTSTORE_FILE_PATH" />
+            <transfer operation="delete" delete-key="SSL_TRUSTSTORE_PASSWORD" />
+            <transfer operation="delete" delete-key="XAAUDIT.DB.IS_ENABLED" />
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.DESTINATION_DIRECTORY"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.IS_ENABLED" />
+            <transfer operation="delete" delete-key="XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY"
/>
+            <transfer operation="delete" delete-key="XAAUDIT.DB.USER_NAME" />
+            <transfer operation="delete" delete-key="XAAUDIT.DB.PASSWORD" />
+          </task>
+        </pre-upgrade>
         <upgrade>
           <task xsi:type="restart" />
         </upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/8e3433c8/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java
index c2ea948..08a8c35 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/ConfigureActionTest.java
@@ -306,6 +306,7 @@ public class ConfigureActionTest {
     transfer.fromKey = "movedFromKeyMissingWithDefault";
     transfer.toKey = "movedToMissingWithDefault";
     transfer.defaultValue = "defaultValue2";
+    transfer.mask = true;
     transfers.add(transfer);
 
     transfer = new ConfigureTask.Transfer();
@@ -544,6 +545,7 @@ public class ConfigureActionTest {
     configurations.add(fooKey3);
     fooKey3.key = "fooKey3";
     fooKey3.value = "barValue3";
+    fooKey3.mask = true;
 
     Map<String, String> commandParams = new HashMap<String, String>();
     commandParams.put("upgrade_direction", "upgrade");
@@ -573,6 +575,8 @@ public class ConfigureActionTest {
     assertEquals("barValue", config.getProperties().get("fooKey"));
     assertEquals("barValue2", config.getProperties().get("fooKey2"));
     assertEquals("barValue3", config.getProperties().get("fooKey3"));
+    assertTrue(report.getStdOut().contains("******"));
+
   }
 
   private void makeUpgradeCluster() throws Exception {


Mime
View raw message