ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dmitriu...@apache.org
Subject [10/21] ambari git commit: AMBARI-15064. RU/EU can't start if hosts have name in MixedCASE in configs for NameNode, HBASE Master, ResourceManager (alejandro)
Date Fri, 19 Feb 2016 13:26:27 GMT
AMBARI-15064. RU/EU can't start if hosts have name in MixedCASE in configs for NameNode, HBASE
Master, ResourceManager (alejandro)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8f58e5f9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8f58e5f9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8f58e5f9

Branch: refs/heads/branch-dev-patch-upgrade
Commit: 8f58e5f97e0e6c7853956ad4f48d45603cee9d7f
Parents: 7da4706
Author: Alejandro Fernandez <afernandez@hortonworks.com>
Authored: Thu Feb 18 11:21:04 2016 -0800
Committer: Alejandro Fernandez <afernandez@hortonworks.com>
Committed: Thu Feb 18 11:21:16 2016 -0800

----------------------------------------------------------------------
 .../controller/AmbariActionExecutionHelper.java |  13 +-
 .../internal/UpgradeResourceProvider.java       |   1 +
 .../ambari/server/stack/MasterHostResolver.java |  53 +++++--
 .../AmbariManagementControllerTest.java         |   2 +-
 .../ambari/server/state/UpgradeHelperTest.java  | 140 +++++++++++++++++--
 5 files changed, 178 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/8f58e5f9/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
index 795dfa7..88180c0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
@@ -262,6 +262,13 @@ public class AmbariActionExecutionHelper {
     final String serviceName = actionContext.getExpectedServiceName();
     final String componentName = actionContext.getExpectedComponentName();
 
+    LOG.debug("Called addExecutionCommandsToStage() for serviceName: {}, componentName: {}.",
serviceName, componentName);
+    if (resourceFilter.getHostNames().isEmpty()) {
+      LOG.debug("Resource filter has no hostnames.");
+    } else {
+      LOG.debug("Resource filter has hosts: {}", StringUtils.join(resourceFilter.getHostNames(),
", "));
+    }
+
     if (null != cluster) {
       StackId stackId = cluster.getCurrentStackVersion();
       if (serviceName != null && !serviceName.isEmpty()) {
@@ -275,6 +282,7 @@ public class AmbariActionExecutionHelper {
                 stackId.getStackVersion(), serviceName, componentName);
           } catch (ObjectNotFoundException e) {
             // do nothing, componentId is checked for null later
+            LOG.error("Did not find service {} and component {} in stack {}.", serviceName,
componentName, stackId.getStackName());
           }
         } else {
           for (String component : cluster.getService(serviceName).getServiceComponents().keySet())
{
@@ -288,6 +296,7 @@ public class AmbariActionExecutionHelper {
         // All hosts are valid target host
         candidateHosts.addAll(clusters.getHostsForCluster(cluster.getClusterName()).keySet());
       }
+      LOG.debug("Request for service {} and component {} is set to run on candidate hosts:
{}.", serviceName, componentName, StringUtils.join(candidateHosts, ", "));
 
       // Filter hosts that are in MS
       Set<String> ignoredHosts = maintenanceStateHelper.filterHostsInMaintenanceState(
@@ -301,7 +310,9 @@ public class AmbariActionExecutionHelper {
                 }
               }
       );
+
       if (! ignoredHosts.isEmpty()) {
+        LOG.debug("Hosts to ignore: {}.", StringUtils.join(ignoredHosts, ", "));
         LOG.debug("Ignoring action for hosts due to maintenance state." +
             "Ignored hosts =" + ignoredHosts + ", component="
             + componentName + ", service=" + serviceName
@@ -323,7 +334,7 @@ public class AmbariActionExecutionHelper {
       for (String hostname : resourceFilter.getHostNames()) {
         if (!candidateHosts.contains(hostname)) {
           throw new AmbariException("Request specifies host " + hostname +
-            " but its not a valid host based on the " +
+            " but it is not a valid host based on the " +
             "target service=" + serviceName + " and component=" + componentName);
         }
       }

http://git-wip-us.apache.org/repos/asf/ambari/blob/8f58e5f9/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index 70440fc..e5664c2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -1175,6 +1175,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
     RequestResourceFilter filter = new RequestResourceFilter("", "",
         new ArrayList<String>(wrapper.getHosts()));
 
+    LOG.debug("Analyzing upgrade item {} with tasks: {}.", entity.getText(), entity.getTasks());
     Map<String, String> params = getNewParameterMap();
     params.put(COMMAND_PARAM_TASKS, entity.getTasks());
     params.put(COMMAND_PARAM_VERSION, context.getVersion());

http://git-wip-us.apache.org/repos/asf/ambari/blob/8f58e5f9/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java
b/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java
index 561350b..360f2b8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java
@@ -63,7 +63,7 @@ public class MasterHostResolver {
   /**
    * Union of status for several services.
    */
-  enum Status {
+  protected enum Status {
     ACTIVE,
     STANDBY
   }
@@ -247,11 +247,11 @@ public class MasterHostResolver {
     return false;
   }
 
-
   /**
    * Get mapping of the HDFS Namenodes from the state ("active" or "standby") to the hostname.
    * @return Returns a map from the state ("active" or "standby" to the hostname with that
state if exactly
    * one active and one standby host were found, otherwise, return null.
+   * The hostnames are returned in lowercase.
    */
   private Map<Status, String> getNameNodePair() {
     Map<Status, String> stateToHost = new HashMap<Status, String>();
@@ -291,7 +291,9 @@ public class MasterHostResolver {
 
         if (null != state && (state.equalsIgnoreCase(Status.ACTIVE.toString()) ||
state.equalsIgnoreCase(Status.STANDBY.toString()))) {
           Status status = Status.valueOf(state.toUpperCase());
-          stateToHost.put(status, hp.host);
+          stateToHost.put(status, hp.host.toLowerCase());
+        } else {
+          LOG.error(String.format("Could not retrieve state for NameNode %s from property
%s by querying JMX.", hp.host, key));
         }
       } catch (MalformedURLException e) {
         LOG.error(e.getMessage());
@@ -304,6 +306,12 @@ public class MasterHostResolver {
     return null;
   }
 
+  /**
+   * Resolve the name of the Resource Manager master and convert the hostname to lowercase.
+   * @param cluster Cluster
+   * @param hostType RM hosts
+   * @throws MalformedURLException
+   */
   private void resolveResourceManagers(Cluster cluster, HostsType hostType) throws MalformedURLException
{
     LinkedHashSet<String> orderedHosts = new LinkedHashSet<String>(hostType.hosts);
 
@@ -320,18 +328,24 @@ public class MasterHostResolver {
 
       if (null != value) {
         if (null == hostType.master) {
-          hostType.master = hostname;
+          hostType.master = hostname.toLowerCase();
         }
 
         // Quick and dirty to make sure the master is last in the list
-        orderedHosts.remove(hostname);
-        orderedHosts.add(hostname);
+        orderedHosts.remove(hostname.toLowerCase());
+        orderedHosts.add(hostname.toLowerCase());
       }
 
     }
     hostType.hosts = orderedHosts;
   }
 
+  /**
+   * Resolve the HBASE master and convert the hostname to lowercase.
+   * @param cluster Cluster
+   * @param hostsType HBASE master host.
+   * @throws AmbariException
+   */
   private void resolveHBaseMasters(Cluster cluster, HostsType hostsType) throws AmbariException
{
     String hbaseMasterInfoPortProperty = "hbase.master.info.port";
     String hbaseMasterInfoPortValue = m_configHelper.getValueFromDesiredConfigurations(cluster,
ConfigHelper.HBASE_SITE, hbaseMasterInfoPortProperty);
@@ -348,22 +362,31 @@ public class MasterHostResolver {
       if (null != value) {
         Boolean bool = Boolean.valueOf(value);
         if (bool.booleanValue()) {
-          hostsType.master = hostname;
+          hostsType.master = hostname.toLowerCase();
         } else {
-          hostsType.secondary = hostname;
+          hostsType.secondary = hostname.toLowerCase();
         }
       }
-
     }
   }
 
-  private String queryJmxBeanValue(String hostname, int port, String beanName, String attributeName,
-                                   boolean asQuery) {
+  protected String queryJmxBeanValue(String hostname, int port, String beanName, String attributeName,
+                                  boolean asQuery) {
     return queryJmxBeanValue(hostname, port, beanName, attributeName, asQuery, false);
   }
 
-  private String queryJmxBeanValue(String hostname, int port, String beanName, String attributeName,
-      boolean asQuery, boolean encrypted) {
+  /**
+   * Query the JMX attribute at http(s)://$server:$port/jmx?qry=$query or http(s)://$server:$port/jmx?get=$bean::$attribute
+   * @param hostname host name
+   * @param port port number
+   * @param beanName if asQuery is false, then search for this bean name
+   * @param attributeName if asQuery is false, then search for this attribute name
+   * @param asQuery whether to search bean or query
+   * @param encrypted true if using https instead of http.
+   * @return The jmx value.
+   */
+  protected String queryJmxBeanValue(String hostname, int port, String beanName, String attributeName,
+                                  boolean asQuery, boolean encrypted) {
 
     String protocol = encrypted ? "https://" : "http://";
     String endPoint = protocol + (asQuery ?
@@ -385,9 +408,9 @@ public class MasterHostResolver {
       return jmxBeans.get("beans").get(0).get(attributeName);
     } catch (Exception e) {
       if (LOG.isDebugEnabled()) {
-        LOG.info("Could not load JMX from {}/{} from {}", beanName, attributeName, hostname,
e);
+        LOG.debug("Could not load JMX from {}/{} from {}", beanName, attributeName, hostname,
e);
       } else {
-        LOG.info("Could not load JMX from {}/{} from {}", beanName, attributeName, hostname);
+        LOG.debug("Could not load JMX from {}/{} from {}", beanName, attributeName, hostname);
       }
     }
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/8f58e5f9/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index a0ac966..ffee3fa 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -4650,7 +4650,7 @@ public class AmbariManagementControllerTest {
 
     actionRequest = new ExecuteActionRequest("c1", null, "a2", resourceFilters, null, params,
false);
     expectActionCreationErrorWithMessage(actionRequest, requestProperties,
-        "Request specifies host h6 but its not a valid host based on the target service=HDFS
and component=DATANODE");
+        "Request specifies host h6 but it is not a valid host based on the target service=HDFS
and component=DATANODE");
 
     hosts.clear();
     hosts.add("h1");

http://git-wip-us.apache.org/repos/asf/ambari/blob/8f58e5f9/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
index eb5bf62..4ea8f15 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
@@ -95,21 +95,23 @@ public class UpgradeHelperTest {
   private AmbariManagementController m_managementController;
   private Gson m_gson = new Gson();
 
-  @Before
-  public void before() throws Exception {
+  /**
+   * Because test cases need to share config mocks, put common ones in this function.
+   * @throws Exception
+   */
+  private void setConfigMocks() throws Exception {
     // configure the mock to return data given a specific placeholder
     m_configHelper = EasyMock.createNiceMock(ConfigHelper.class);
+    expect(m_configHelper.getPlaceholderValueFromDesiredConfigurations(
+        EasyMock.anyObject(Cluster.class), EasyMock.eq("{{foo/bar}}"))).andReturn("placeholder-rendered-properly").anyTimes();
+    expect(m_configHelper.getEffectiveDesiredTags(
+        EasyMock.anyObject(Cluster.class), EasyMock.anyObject(String.class))).andReturn(new
HashMap<String, Map<String, String>>()).anyTimes();
+  }
 
-    expect(
-      m_configHelper.getPlaceholderValueFromDesiredConfigurations(
-        EasyMock.anyObject(Cluster.class), EasyMock.eq("{{foo/bar}}"))).andReturn(
-        "placeholder-rendered-properly").anyTimes();
-
-    expect(
-        m_configHelper.getEffectiveDesiredTags(
-            EasyMock.anyObject(Cluster.class), EasyMock.anyObject(String.class))).
-        andReturn(new HashMap<String, Map<String, String>>()).anyTimes();
-
+  @Before
+  public void before() throws Exception {
+    setConfigMocks();
+    // Most test cases can replay the common config mocks. If any test case needs custom
ones, it can re-initialize m_configHelper;
     replay(m_configHelper);
 
     final InMemoryDefaultTestModule injectorModule = new InMemoryDefaultTestModule() {
@@ -1244,7 +1246,7 @@ public class UpgradeHelperTest {
     Service s = c.getService("ZOOKEEPER");
     ServiceComponent sc = s.addServiceComponent("ZOOKEEPER_SERVER");
 
-    ServiceComponentHost sch1 =sc.addServiceComponentHost("h1");
+    ServiceComponentHost sch1 = sc.addServiceComponentHost("h1");
     sch1.setVersion("2.1.1.0-1234");
 
     ServiceComponentHost sch2 = sc.addServiceComponentHost("h2");
@@ -1252,7 +1254,6 @@ public class UpgradeHelperTest {
 
     List<ServiceComponentHost> schs = c.getServiceComponentHosts("ZOOKEEPER", "ZOOKEEPER_SERVER");
     assertEquals(2, schs.size());
-
     MasterHostResolver mhr = new MasterHostResolver(null, c, "2.1.1.0-1234");
 
     HostsType ht = mhr.getMasterAndHosts("ZOOKEEPER", "ZOOKEEPER_SERVER");
@@ -1267,6 +1268,117 @@ public class UpgradeHelperTest {
     assertEquals("h2", ht.hosts.iterator().next());
   }
 
+  /**
+   * Test that MasterHostResolver is case-insensitive even if configs have hosts in upper
case for NameNode.
+   * @throws Exception
+   */
+  @Test
+  public void testResolverCaseInsensitive() throws Exception {
+    Clusters clusters = injector.getInstance(Clusters.class);
+    ServiceFactory serviceFactory = injector.getInstance(ServiceFactory.class);
+
+    String clusterName = "c1";
+    String version = "2.1.1.0-1234";
+
+    StackId stackId = new StackId("HDP-2.1.1");
+    clusters.addCluster(clusterName, stackId);
+    Cluster c = clusters.getCluster(clusterName);
+
+    helper.getOrCreateRepositoryVersion(stackId,
+        c.getDesiredStackVersion().getStackVersion());
+
+    c.createClusterVersion(stackId,
+        c.getDesiredStackVersion().getStackVersion(), "admin",
+        RepositoryVersionState.UPGRADING);
+
+    for (int i = 0; i < 2; i++) {
+      String hostName = "h" + (i+1);
+      clusters.addHost(hostName);
+      Host host = clusters.getHost(hostName);
+
+      Map<String, String> hostAttributes = new HashMap<String, String>();
+      hostAttributes.put("os_family", "redhat");
+      hostAttributes.put("os_release_version", "6");
+
+      host.setHostAttributes(hostAttributes);
+
+      host.persist();
+      clusters.mapHostToCluster(hostName, clusterName);
+    }
+
+    // Add services
+    c.addService(serviceFactory.createNew(c, "HDFS"));
+
+    Service s = c.getService("HDFS");
+    ServiceComponent sc = s.addServiceComponent("NAMENODE");
+    sc.addServiceComponentHost("h1");
+    sc.addServiceComponentHost("h2");
+
+    List<ServiceComponentHost> schs = c.getServiceComponentHosts("HDFS", "NAMENODE");
+    assertEquals(2, schs.size());
+
+    setConfigMocks();
+    expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.nameservices")).andReturn("ha").anyTimes();
+    expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.ha.namenodes.ha")).andReturn("nn1,nn2").anyTimes();
+    expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.http.policy")).andReturn("HTTP_ONLY").anyTimes();
+
+    // Notice that these names are all caps.
+    expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.namenode.http-address.ha.nn1")).andReturn("H1:50070").anyTimes();
+    expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.namenode.http-address.ha.nn2")).andReturn("H2:50070").anyTimes();
+    replay(m_configHelper);
+
+    MasterHostResolver mhr = new MockMasterHostResolver(m_configHelper, c, version);
+
+    HostsType ht = mhr.getMasterAndHosts("HDFS", "NAMENODE");
+    assertNotNull(ht.master);
+    assertNotNull(ht.secondary);
+    assertEquals(2, ht.hosts.size());
+
+    // Should be stored in lowercase.
+    assertTrue(ht.hosts.contains("h1"));
+    assertTrue(ht.hosts.contains("h1"));
+  }
+
+  /**
+   * Extend {@link org.apache.ambari.server.stack.MasterHostResolver} in order to overwrite
the JMX methods.
+   */
+  private class MockMasterHostResolver extends MasterHostResolver {
+
+    public MockMasterHostResolver(ConfigHelper configHelper, Cluster cluster) {
+      this(configHelper, cluster, null);
+    }
+
+    public MockMasterHostResolver(ConfigHelper configHelper, Cluster cluster, String version)
{
+      super(configHelper, cluster, version);
+    }
+
+    /**
+     * Mock the call to get JMX Values.
+     * @param hostname host name
+     * @param port port number
+     * @param beanName if asQuery is false, then search for this bean name
+     * @param attributeName if asQuery is false, then search for this attribute name
+     * @param asQuery whether to search bean or query
+     * @param encrypted true if using https instead of http.
+     * @return
+     */
+    @Override
+    public String queryJmxBeanValue(String hostname, int port, String beanName, String attributeName,
+                                    boolean asQuery, boolean encrypted) {
+
+      if (beanName.equalsIgnoreCase("Hadoop:service=NameNode,name=NameNodeStatus") &&
attributeName.equalsIgnoreCase("State") && asQuery) {
+        switch (hostname) {
+          case "H1":
+            return Status.ACTIVE.toString();
+          case "H2":
+            return Status.STANDBY.toString();
+          default:
+            return "UNKNOWN_NAMENODE_STATUS_FOR_THIS_HOST";
+        }
+      }
+      return  "NOT_MOCKED";
+    }
+  }
 
   private class MockModule implements Module {
 


Mime
View raw message