ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rle...@apache.org
Subject ambari git commit: AMBARI-21798. Oozie server crashes post migration after regen kerberos keytabs and restart all services (rlevas)
Date Thu, 24 Aug 2017 13:14:49 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.6 9de2499a8 -> 42e1b1007


AMBARI-21798. Oozie server crashes post migration after regen kerberos keytabs and restart
all services (rlevas)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/42e1b100
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/42e1b100
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/42e1b100

Branch: refs/heads/branch-2.6
Commit: 42e1b1007f88aeeab9bc77e81896648bfe9a24b3
Parents: 9de2499
Author: Robert Levas <rlevas@hortonworks.com>
Authored: Thu Aug 24 09:14:43 2017 -0400
Committer: Robert Levas <rlevas@hortonworks.com>
Committed: Thu Aug 24 09:14:43 2017 -0400

----------------------------------------------------------------------
 .../server/upgrade/UpgradeCatalog252.java       | 86 ++++++++++++++++----
 .../server/upgrade/UpgradeCatalog252Test.java   | 57 +++++++++++++
 2 files changed, 128 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/42e1b100/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
index 2780b13..e6a7829 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog252.java
@@ -44,9 +44,12 @@ import org.apache.ambari.server.state.ConfigHelper;
 import org.apache.ambari.server.state.PropertyInfo;
 import org.apache.ambari.server.state.StackId;
 import org.apache.ambari.server.state.kerberos.AbstractKerberosDescriptorContainer;
+import org.apache.ambari.server.state.kerberos.KerberosComponentDescriptor;
 import org.apache.ambari.server.state.kerberos.KerberosConfigurationDescriptor;
 import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
 import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory;
+import org.apache.ambari.server.state.kerberos.KerberosServiceDescriptor;
+import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -352,15 +355,52 @@ public class UpgradeCatalog252 extends AbstractUpgradeCatalog {
         final KerberosDescriptor kerberosDescriptor = new KerberosDescriptorFactory().createInstance(data);
 
         if (kerberosDescriptor != null) {
+          boolean updated = false;
+
           // Find and remove configuration specifications for <code>livy-conf/livy.superusers</code>
           // in SPARK since this logic has been moved to the relevant stack/service advisors
-          boolean updatedSpark = removeConfigurationSpecification(kerberosDescriptor.getService("SPARK"),
"livy-conf", "livy.superusers");
+          if(removeConfigurationSpecifications(kerberosDescriptor.getService("SPARK"),
+              Collections.<String, Collection<String>>singletonMap("livy-conf",
Collections.singleton("livy.superusers")))) {
+            updated = true;
+          }
 
           // Find and remove configuration specifications for <code>livy-conf2/livy.superusers</code>
           // in SPARK2 since this logic has been moved to the relevant stack/service advisors
-          boolean updatedSpark2 = removeConfigurationSpecification(kerberosDescriptor.getService("SPARK2"),
"livy2-conf", "livy.superusers");
+          if(removeConfigurationSpecifications(kerberosDescriptor.getService("SPARK2"),
+              Collections.<String, Collection<String>>singletonMap("livy2-conf",
Collections.singleton("livy.superusers")))) {
+            updated = true;
+          }
 
-          if (updatedSpark || updatedSpark2) {
+          // Find and remove configuration specifications for the following configurations
in KNOX/KNOX_GATEWAY
+          // since they are invalid due to static "knox" embedded in the property name:
+          // * oozie-site/oozie.service.ProxyUserService.proxyuser.knox.groups
+          // * oozie-site/oozie.service.ProxyUserService.proxyuser.knox.hosts
+          // * webhcat-site/webhcat.proxyuser.knox.groups
+          // * webhcat-site/webhcat.proxyuser.knox.hosts
+          // * core-site/hadoop.proxyuser.knox.groups
+          // * core-site/hadoop.proxyuser.knox.hosts
+          // * falcon-runtime.properties/*.falcon.service.ProxyUserService.proxyuser.knox.groups
+          // * falcon-runtime.properties/*.falcon.service.ProxyUserService.proxyuser.knox.hosts
+          KerberosServiceDescriptor knoxKerberosDescriptor = kerberosDescriptor.getService("KNOX");
+          if(knoxKerberosDescriptor != null) {
+            KerberosComponentDescriptor knoxGatewayKerberosDescriptor = knoxKerberosDescriptor.getComponent("KNOX_GATEWAY");
+            if (knoxGatewayKerberosDescriptor != null) {
+              Map<String, Collection<String>> configsToRemove = new HashMap<>();
+              configsToRemove.put("oozie-site",
+                  Arrays.asList("oozie.service.ProxyUserService.proxyuser.knox.groups", "oozie.service.ProxyUserService.proxyuser.knox.hosts"));
+              configsToRemove.put("webhcat-site",
+                  Arrays.asList("webhcat.proxyuser.knox.groups", "webhcat.proxyuser.knox.hosts"));
+              configsToRemove.put("core-site",
+                  Arrays.asList("hadoop.proxyuser.knox.groups", "hadoop.proxyuser.knox.hosts"));
+              configsToRemove.put("falcon-runtime.properties",
+                  Arrays.asList("*.falcon.service.ProxyUserService.proxyuser.knox.groups",
"*.falcon.service.ProxyUserService.proxyuser.knox.hosts"));
+              if (removeConfigurationSpecifications(knoxGatewayKerberosDescriptor, configsToRemove))
{
+                updated = true;
+              }
+            }
+          }
+
+          if (updated) {
             artifactEntity.setArtifactData(kerberosDescriptor.toMap());
             artifactDAO.merge(artifactEntity);
           }
@@ -470,24 +510,40 @@ public class UpgradeCatalog252 extends AbstractUpgradeCatalog {
   }
 
   /**
-   * Given an {@link AbstractKerberosDescriptorContainer}, attempts to remove the specified
property
-   * (<code>configType/propertyName</code> from it.
+   * Given an {@link AbstractKerberosDescriptorContainer}, attempts to remove the specified
+   * configurations (<code>configType/propertyName</code> from it.
    *
    * @param kerberosDescriptorContainer the container to update
-   * @param configType                  the configuration type
-   * @param propertyName                the property name
+   * @param configurations              a map of configuration types to sets of property
names.
    * @return true if changes where made to the container; false otherwise
    */
-  private boolean removeConfigurationSpecification(AbstractKerberosDescriptorContainer kerberosDescriptorContainer,
String configType, String propertyName) {
+  private boolean removeConfigurationSpecifications(AbstractKerberosDescriptorContainer kerberosDescriptorContainer,
Map<String, Collection<String>> configurations) {
     boolean updated = false;
     if (kerberosDescriptorContainer != null) {
-      KerberosConfigurationDescriptor configurationDescriptor = kerberosDescriptorContainer.getConfiguration(configType);
-      if (configurationDescriptor != null) {
-        Map<String, String> properties = configurationDescriptor.getProperties();
-        if ((properties != null) && properties.containsKey(propertyName)) {
-          properties.remove(propertyName);
-          LOG.info("Removed {}/{} from the descriptor named {}", configType, propertyName,
kerberosDescriptorContainer.getName());
-          updated = true;
+      if (!MapUtils.isEmpty(configurations)) {
+        for (Map.Entry<String, Collection<String>> entry : configurations.entrySet())
{
+          String configType = entry.getKey();
+
+          for (String propertyName : entry.getValue()) {
+            Map<String, KerberosConfigurationDescriptor> configurationDescriptors =
kerberosDescriptorContainer.getConfigurations(false);
+            KerberosConfigurationDescriptor configurationDescriptor = (configurationDescriptors
== null)
+                ? null
+                : configurationDescriptors.get(configType);
+            if (configurationDescriptor != null) {
+              Map<String, String> properties = configurationDescriptor.getProperties();
+              if ((properties != null) && properties.containsKey(propertyName)) {
+                properties.remove(propertyName);
+                LOG.info("Removed {}/{} from the descriptor named {}", configType, propertyName,
kerberosDescriptorContainer.getName());
+                updated = true;
+
+                // If there are no more properties in the configurationDescriptor, remove
it from the container.
+                if(properties.isEmpty()) {
+                  configurationDescriptors.remove(configType);
+                  kerberosDescriptorContainer.setConfigurations(configurationDescriptors);
+                }
+              }
+            }
+          }
         }
       }
     }

http://git-wip-us.apache.org/repos/asf/ambari/blob/42e1b100/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog252Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog252Test.java
b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog252Test.java
index 72956eb..e6dbb7c 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog252Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog252Test.java
@@ -54,6 +54,7 @@ import org.apache.ambari.server.state.Cluster;
 import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Config;
 import org.apache.ambari.server.state.Service;
+import org.apache.ambari.server.state.kerberos.KerberosComponentDescriptor;
 import org.apache.ambari.server.state.kerberos.KerberosDescriptor;
 import org.apache.ambari.server.state.kerberos.KerberosDescriptorFactory;
 import org.apache.ambari.server.state.stack.OsFamily;
@@ -289,6 +290,48 @@ public class UpgradeCatalog252Test {
         "      ]" +
         "    }," +
         "    {" +
+        "      \"name\": \"KNOX\"," +
+        "      \"components\": [" +
+        "        {" +
+        "          \"name\": \"KNOX_GATEWAY\"," +
+        "          \"configurations\": [" +
+        "            {" +
+        "              \"core-site\": {" +
+        "                \"property1\": \"true\"," +
+        "                \"property2\": \"true\"," +
+        "                \"hadoop.proxyuser.knox.groups\": \"somevalue\"," +
+        "                \"hadoop.proxyuser.knox.hosts\": \"somevalue\"" +
+        "              }" +
+        "            }," +
+        "            {" +
+        "              \"webhcat-site\": {" +
+        "                \"webhcat.proxyuser.knox.groups\": \"somevalue\"," +
+        "                \"webhcat.proxyuser.knox.hosts\": \"somevalue\"" +
+        "              }" +
+        "            }," +
+        "            {" +
+        "              \"oozie-site\": {" +
+        "                \"oozie.service.ProxyUserService.proxyuser.knox.groups\": \"somevalue\","
+
+        "                \"oozie.service.ProxyUserService.proxyuser.knox.hosts\": \"somevalue\""
+
+        "              }" +
+        "            }," +
+        "            {" +
+        "              \"falcon-runtime.properties\": {" +
+        "                \"*.falcon.service.ProxyUserService.proxyuser.knox.groups\": \"somevalue\","
+
+        "                \"*.falcon.service.ProxyUserService.proxyuser.knox.hosts\": \"somevalue\""
+
+        "              }" +
+        "            }," +
+        "            {" +
+        "              \"some-env\": {" +
+        "                \"groups\": \"${hadoop-env/proxyuser_group}\"," +
+        "                \"hosts\": \"${clusterHostInfo/existing_service_master_hosts}\""
+
+        "              }" +
+        "            }" +
+        "          ]" +
+        "        }" +
+        "      ]" +
+        "    }," +
+        "    {" +
         "      \"name\": \"NOT_SPARK\"," +
         "      \"configurations\": [" +
         "        {" +
@@ -360,5 +403,19 @@ public class UpgradeCatalog252Test {
     Assert.assertNotNull(result.getService("NOT_SPARK").getConfiguration("not-livy-conf"));
     Assert.assertNotNull(result.getService("NOT_SPARK").getConfiguration("not-livy-conf").getProperties());
     Assert.assertTrue(result.getService("NOT_SPARK").getConfiguration("not-livy-conf").getProperties().containsKey("livy.superusers"));
+
+    Assert.assertNotNull(result.getService("KNOX"));
+
+    KerberosComponentDescriptor knoxGateway = result.getService("KNOX").getComponent("KNOX_GATEWAY");
+    Assert.assertNotNull(knoxGateway);
+    Assert.assertNotNull(knoxGateway.getConfiguration("core-site"));
+    Assert.assertNotNull(knoxGateway.getConfiguration("core-site").getProperties());
+    Assert.assertTrue(knoxGateway.getConfiguration("core-site").getProperties().containsKey("property1"));
+    Assert.assertFalse(knoxGateway.getConfiguration("core-site").getProperties().containsKey("hadoop.proxyuser.knox.groups"));
+    Assert.assertFalse(knoxGateway.getConfiguration("core-site").getProperties().containsKey("hadoop.proxyuser.knox.hosts"));
+    Assert.assertNull(knoxGateway.getConfiguration("oozie-site"));
+    Assert.assertNull(knoxGateway.getConfiguration("webhcat-site"));
+    Assert.assertNull(knoxGateway.getConfiguration("falcon-runtime.properties"));
+    Assert.assertNotNull(knoxGateway.getConfiguration("some-env"));
   }
 }


Mime
View raw message