ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject ambari git commit: AMBARI-21577. Hive-Service check failing in post EU validation (BI-HDP). (swagle)
Date Fri, 28 Jul 2017 15:45:12 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.5 b1438eea9 -> fb744009e


AMBARI-21577. Hive-Service check failing in post EU validation (BI-HDP). (swagle)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fb744009
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fb744009
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fb744009

Branch: refs/heads/branch-2.5
Commit: fb744009e8d6e269e78943750ec09c73f133470d
Parents: b1438ee
Author: Siddharth Wagle <swagle@hortonworks.com>
Authored: Fri Jul 28 08:44:58 2017 -0700
Committer: Siddharth Wagle <swagle@hortonworks.com>
Committed: Fri Jul 28 08:44:58 2017 -0700

----------------------------------------------------------------------
 .../upgrades/FixAuthToLocalMappingAction.java   | 149 +++++++++++++++++++
 .../upgrades/nonrolling-upgrade-to-hdp-2.6.xml  |   8 +
 .../upgrades/nonrolling-upgrade-to-hdp-2.6.xml  |   8 +
 .../FixAuthToLocalMappingActionTest.java        | 126 ++++++++++++++++
 4 files changed, 291 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/fb744009/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingAction.java
b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingAction.java
new file mode 100644
index 0000000..e62f2db
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingAction.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.serveraction.upgrades;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.ambari.server.AmbariException;
+import org.apache.ambari.server.actionmanager.HostRoleStatus;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.controller.KerberosHelper;
+import org.apache.ambari.server.serveraction.AbstractServerAction;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
+
+import com.google.inject.Inject;
+
+/**
+ * Fixes auth_to_local rules during upgrade from IOP to HDP. An example of
+ * invalid rule introduced by HBASE_REST_SERVER in the IOP stack, set auth to
+ * local mapping for HTTP spnego principal to local hbase user, which needs to
+ * be deleted for HIVE service check to pass.
+ */
+public class FixAuthToLocalMappingAction  extends AbstractServerAction {
+
+  private static final String SPNEGO_PRINC_PATTERN = "RULE:\\[2:\\$1@\\$0\\]\\(HTTP@.*\\)s/\\.\\*/.*/\\n";
+  private static final String AMS_HBASE_PATTERN = "RULE:\\[2:\\$1@\\$0\\]\\(amshbase@.*\\)s/\\.\\*/%s/\\n";
+  private static final String ZK_AMS_PATTERN = "RULE:\\[2:\\$1@\\$0\\]\\(zookeeper@.*\\)s/\\.\\*/%s/\\n";
+
+  @Inject
+  private Clusters clusters;
+
+  @Inject
+  private KerberosHelper kerberosHelper;
+
+  @Override
+  public CommandReport execute(ConcurrentMap<String, Object> requestSharedDataContext)
+      throws AmbariException, InterruptedException {
+
+    String clusterName = getExecutionCommand().getClusterName();
+    Cluster cluster = clusters.getCluster(clusterName);
+
+    KerberosDescriptor kd = kerberosHelper.getKerberosDescriptor(cluster);
+    if (kd == null) {
+      return null;
+    }
+
+    Map<String, Set<String>> configProperties = new HashMap<>();
+    for (String property : kd.getAllAuthToLocalProperties()) {
+      if (!StringUtils.isEmpty(property) && property.contains("/")) {
+        String[] propertyParts = property.split("/");
+        if (configProperties.containsKey(propertyParts[0])) {
+          configProperties.get(propertyParts[0]).add(propertyParts[1]);
+        } else {
+          Set<String> properties = new HashSet<>();
+          properties.add(propertyParts[1]);
+          configProperties.put(propertyParts[0], properties);
+        }
+      }
+    }
+
+    String hbaseUser = null;
+    Config hbaseEnv = cluster.getDesiredConfigByType("hbase-env");
+    if (hbaseEnv != null) {
+      Map<String, String> properties = hbaseEnv.getProperties();
+      if (!MapUtils.isEmpty(properties)) {
+        hbaseUser = properties.get("hbase_user");
+      }
+    }
+
+    String amsUser = null;
+    Config amsEnv = cluster.getDesiredConfigByType("ams-env");
+    if (amsEnv != null) {
+      Map<String, String> properties = amsEnv.getProperties();
+      if (!MapUtils.isEmpty(properties)) {
+        amsUser = properties.get("ambari_metrics_user");
+      }
+    }
+
+    boolean replaced = false;
+    StringBuilder message = new StringBuilder("Replaced offending auto_to_local mappings");
+
+    for (Map.Entry<String, Set<String>> configProperty : configProperties.entrySet())
{
+      String configType = configProperty.getKey();
+      Config config = cluster.getDesiredConfigByType(configType);
+
+      if (config == null) {
+        continue;
+      }
+
+      for (String property : configProperty.getValue()) {
+        Map<String, String> properties = config.getProperties();
+        if (!MapUtils.isEmpty(properties) && properties.containsKey(property)) {
+          String authToLocalRules = properties.get(property);
+          if (!StringUtils.isEmpty(authToLocalRules)) {
+            authToLocalRules = authToLocalRules.replaceAll(SPNEGO_PRINC_PATTERN, "");
+            if (hbaseUser != null) {
+              authToLocalRules = authToLocalRules.replaceAll(String.format(AMS_HBASE_PATTERN,
hbaseUser), "");
+            }
+            if (amsUser != null) {
+              authToLocalRules = authToLocalRules.replaceAll(String.format(ZK_AMS_PATTERN,
amsUser), "");
+            }
+            // Only if something was replaced
+            if (!properties.get(property).equals(authToLocalRules)) {
+              properties.put(property, authToLocalRules);
+              message.append(" , property => ");
+              message.append(property);
+              message.append(" , config => ");
+              message.append(configType);
+
+              config.setProperties(properties);
+              config.save();
+              replaced = true;
+            }
+          }
+        }
+      }
+    }
+
+    String finalMessage = message.toString();
+    if (!replaced) {
+      finalMessage = "No offending auto_to_local mappings found";
+    }
+
+    return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", finalMessage, "");
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/fb744009/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
index 9d3cc1e..a7fdeec 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2.5/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
@@ -222,6 +222,14 @@
         </task>
       </execute-stage>
 
+      <!-- core-site -->
+      <execute-stage title="Fix auth_to_local mapping">
+        <condition xsi:type="security" type="kerberos"/>
+        <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixAuthToLocalMappingAction">
+          <summary>Fix auth to local mapping rules</summary>
+        </task>
+      </execute-stage>
+
       <!-- YARN -->
       <execute-stage service="YARN" component="RESOURCEMANAGER" title="Update Yarn configurations">
         <task xsi:type="configure" id="biginsights_4_2_yarn_config_update" />

http://git-wip-us.apache.org/repos/asf/ambari/blob/fb744009/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
index 3637927..5532248 100644
--- a/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
+++ b/ambari-server/src/main/resources/stacks/BigInsights/4.2/upgrades/nonrolling-upgrade-to-hdp-2.6.xml
@@ -217,6 +217,14 @@
         </task>
       </execute-stage>
 
+      <!-- core-site -->
+      <execute-stage title="Fix auth_to_local mapping">
+        <condition xsi:type="security" type="kerberos"/>
+        <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixAuthToLocalMappingAction">
+          <summary>Fix auth to local mapping rules</summary>
+        </task>
+      </execute-stage>
+
       <!-- YARN -->
       <execute-stage service="YARN" component="RESOURCEMANAGER" title="Update Yarn configurations">
         <task xsi:type="configure" id="biginsights_4_2_yarn_config_update" />

http://git-wip-us.apache.org/repos/asf/ambari/blob/fb744009/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingActionTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingActionTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingActionTest.java
new file mode 100644
index 0000000..ab102bc
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/FixAuthToLocalMappingActionTest.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.serveraction.upgrades;
+
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.powermock.api.easymock.PowerMock.verifyAll;
+
+import java.lang.reflect.Field;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
+import org.apache.ambari.server.actionmanager.HostRoleCommand;
+import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.controller.KerberosHelper;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
+import org.easymock.Capture;
+import org.junit.Test;
+
+import com.google.common.collect.Maps;
+import com.google.inject.Injector;
+
+import junit.framework.Assert;
+
+public class FixAuthToLocalMappingActionTest {
+
+  String authToLocalRulesOriginal = "RULE:[1:$1@$0](ambari-qa-c1@EXAMPLE.COM)s/.*/ambari-qa/\nRULE:[1:$1@$0](hbase-c1@EXAMPLE.COM)s/.*/hbase/\nRULE:[1:$1@$0](hdfs-c1@EXAMPLE.COM)s/.*/hdfs/\nRULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nRULE:[2:$1@$0](HTTP@EXAMPLE.COM)s/.*/hbase/\nRULE:[2:$1@$0](amshbase@EXAMPLE.COM)s/.*/ams/\nRULE:[2:$1@$0](dn@EXAMPLE.COM)s/.*/hdfs/\nRULE:[2:$1@$0](hbase@EXAMPLE.COM)s/.*/hbase/\nRULE:[2:$1@$0](hive@EXAMPLE.COM)s/.*/hive/\nRULE:[2:$1@$0](jhs@EXAMPLE.COM)s/.*/mapred/\nRULE:[2:$1@$0](nm@EXAMPLE.COM)s/.*/yarn/\nRULE:[2:$1@$0](nn@EXAMPLE.COM)s/.*/hdfs/\nRULE:[2:$1@$0](rm@EXAMPLE.COM)s/.*/yarn/\nRULE:[2:$1@$0](yarn@EXAMPLE.COM)s/.*/yarn/\nRULE:[2:$1@$0](zookeeper@EXAMPLE.COM)s/.*/ams/\nRULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/\nRULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/\nRULE:[2:$1@$0](hm@.*)s/.*/hbase/\nRULE:[2:$1@$0](jhs@.*)s/.*/mapred/\nRULE:[2:$1@$0](rs@.*)s/.*/hbase/\nDEFAULT";
+  String authToLocalRulesUpdated = "RULE:[1:$1@$0](ambari-qa-c1@EXAMPLE.COM)s/.*/ambari-qa/\nRULE:[1:$1@$0](hbase-c1@EXAMPLE.COM)s/.*/hbase/\nRULE:[1:$1@$0](hdfs-c1@EXAMPLE.COM)s/.*/hdfs/\nRULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nRULE:[2:$1@$0](amshbase@EXAMPLE.COM)s/.*/ams/\nRULE:[2:$1@$0](dn@EXAMPLE.COM)s/.*/hdfs/\nRULE:[2:$1@$0](hbase@EXAMPLE.COM)s/.*/hbase/\nRULE:[2:$1@$0](hive@EXAMPLE.COM)s/.*/hive/\nRULE:[2:$1@$0](jhs@EXAMPLE.COM)s/.*/mapred/\nRULE:[2:$1@$0](nm@EXAMPLE.COM)s/.*/yarn/\nRULE:[2:$1@$0](nn@EXAMPLE.COM)s/.*/hdfs/\nRULE:[2:$1@$0](rm@EXAMPLE.COM)s/.*/yarn/\nRULE:[2:$1@$0](yarn@EXAMPLE.COM)s/.*/yarn/\nRULE:[2:$1@$0]([nd]n@.*)s/.*/hdfs/\nRULE:[2:$1@$0]([rn]m@.*)s/.*/yarn/\nRULE:[2:$1@$0](hm@.*)s/.*/hbase/\nRULE:[2:$1@$0](jhs@.*)s/.*/mapred/\nRULE:[2:$1@$0](rs@.*)s/.*/hbase/\nDEFAULT";
+
+  @Test
+  public void testExecute() throws Exception {
+    String clusterName = "c1";
+
+    Injector injector = createNiceMock(Injector.class);
+    Clusters clusters = createNiceMock(Clusters.class);
+    Cluster cluster = createNiceMock(Cluster.class);
+    KerberosHelper kerberosHelper = createNiceMock(KerberosHelper.class);
+    KerberosDescriptor descriptor = createNiceMock(KerberosDescriptor.class);
+
+    expect(kerberosHelper.getKerberosDescriptor(cluster)).andReturn(descriptor).anyTimes();
+    Set<String> mappings = new HashSet<>();
+    mappings.add("core-site/hadoop.security.auth_to_local");
+
+    expect(descriptor.getAllAuthToLocalProperties()).andReturn(mappings);
+
+    Map<String, String> commandParams = Maps.newHashMap();
+    commandParams.put("clusterName", clusterName);
+
+    ExecutionCommand executionCommand = new ExecutionCommand();
+    executionCommand.setCommandParams(commandParams);
+    executionCommand.setClusterName(clusterName);
+
+    HostRoleCommand hrc = createNiceMock(HostRoleCommand.class);
+    expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand));
+
+    Config hbaseEnv = createNiceMock(Config.class);
+    expect(cluster.getDesiredConfigByType("hbase-env")).andReturn(hbaseEnv);
+    expect(hbaseEnv.getProperties()).andReturn(Collections.singletonMap("hbase_user", "hbase"));
+
+    Config amsEnv = createNiceMock(Config.class);
+    expect(cluster.getDesiredConfigByType("ams-env")).andReturn(amsEnv);
+    expect(amsEnv.getProperties()).andReturn(Collections.singletonMap("ambari_metrics_user",
"ams"));
+
+    Config coreSite = createNiceMock(Config.class);
+    expect(cluster.getDesiredConfigByType("core-site")).andReturn(coreSite);
+
+    Map<String, String> original = Maps.newHashMap();
+    original.put("hadoop.security.auth_to_local", authToLocalRulesOriginal);
+    expect(coreSite.getProperties()).andReturn(original);
+
+    Capture<Map<String, String>> updated = Capture.newInstance();
+    coreSite.setProperties(capture(updated));
+    expectLastCall();
+    coreSite.save();
+    expectLastCall();
+
+    expect(clusters.getCluster(clusterName)).andReturn(cluster).anyTimes();
+    expect(injector.getInstance(Clusters.class)).andReturn(clusters).atLeastOnce();
+
+    FixAuthToLocalMappingAction action = new FixAuthToLocalMappingAction();
+    action.setExecutionCommand(executionCommand);
+    action.setHostRoleCommand(hrc);
+
+    Field clustersField = FixAuthToLocalMappingAction.class.getDeclaredField("clusters");
+    clustersField.setAccessible(true);
+    clustersField.set(action, clusters);
+
+    Field kerberosHelperField = FixAuthToLocalMappingAction.class.getDeclaredField("kerberosHelper");
+    kerberosHelperField.setAccessible(true);
+    kerberosHelperField.set(action, kerberosHelper);
+
+    replay(kerberosHelper, descriptor, injector, clusters, cluster, hrc, hbaseEnv, amsEnv,
coreSite);
+
+    ConcurrentMap<String, Object> emptyMap = Maps.newConcurrentMap();
+    action.execute(emptyMap);
+
+    verifyAll();
+
+    Assert.assertEquals(authToLocalRulesUpdated, updated.getValue().get("hadoop.security.auth_to_local"));
+  }
+}


Mime
View raw message