ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [09/12] git commit: AMBARI-6870. BE: Provide configurations validation via /validations endpoint on stack-version
Date Fri, 15 Aug 2014 13:16:35 GMT
AMBARI-6870. BE: Provide configurations validation via /validations endpoint on stack-version


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/60680064
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/60680064
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/60680064

Branch: refs/heads/branch-alerts-dev
Commit: 606800640cd4e737d4d7ae8b9e07e69c325bb55a
Parents: f46d037
Author: Srimanth Gunturi <sgunturi@hortonworks.com>
Authored: Thu Aug 14 18:08:36 2014 -0700
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Fri Aug 15 09:15:57 2014 -0400

----------------------------------------------------------------------
 .../stackadvisor/StackAdvisorHelper.java        |  21 +-
 .../stackadvisor/StackAdvisorRequest.java       |  16 +-
 .../GetConfigurationRecommnedationCommand.java  |   3 +-
 .../GetConfigurationValidationCommand.java      |  61 +++++
 .../commands/StackAdvisorCommand.java           |  21 ++
 .../internal/StackAdvisorResourceProvider.java  |  56 ++++-
 .../stacks/HDP/2.0.6/services/stack_advisor.py  | 143 +++++++++++
 .../stackadvisor/StackAdvisorHelperTest.java    |  14 ++
 .../stacks/2.0.6/common/test_stack_advisor.py   | 248 +++++++++++++++++++
 9 files changed, 554 insertions(+), 29 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/60680064/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
index 156b637..6903c1d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelper.java
@@ -25,6 +25,7 @@ import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.St
 import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutRecommnedationCommand;
 import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutValidationCommand;
 import org.apache.ambari.server.api.services.stackadvisor.commands.GetConfigurationRecommnedationCommand;
+import org.apache.ambari.server.api.services.stackadvisor.commands.GetConfigurationValidationCommand;
 import org.apache.ambari.server.api.services.stackadvisor.commands.StackAdvisorCommand;
 import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
 import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
@@ -74,6 +75,9 @@ public class StackAdvisorHelper {
     if (requestType == StackAdvisorRequestType.HOST_GROUPS) {
       command = new GetComponentLayoutValidationCommand(recommendationsDir, stackAdvisorScript,
           requestId, saRunner);
+    } else if (requestType == StackAdvisorRequestType.CONFIGURATIONS) {
+      command = new GetConfigurationValidationCommand(recommendationsDir, stackAdvisorScript,
+          requestId, saRunner);
     } else {
       throw new StackAdvisorException(String.format("Unsupported request type, type=%s",
           requestType));
@@ -117,21 +121,4 @@ public class StackAdvisorHelper {
     return command;
   }
 
-  /**
-   * Return configurations recommendation based on hosts and services
-   * information.
-   *
-   * @param request the recommendation request
-   * @return {@link RecommendationResponse} instance
-   * @throws StackAdvisorException in case of stack advisor script errors
-   */
-  public synchronized RecommendationResponse getConfigurationRecommnedation(
-      StackAdvisorRequest request) throws StackAdvisorException {
-    requestId += 1;
-
-    GetConfigurationRecommnedationCommand command = new GetConfigurationRecommnedationCommand(
-        recommendationsDir, stackAdvisorScript, requestId, saRunner);
-    return command.invoke(request);
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/60680064/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
index bc1678e..991f198 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorRequest.java
@@ -27,8 +27,6 @@ import java.util.Set;
 
 import org.apache.commons.lang.StringUtils;
 
-import static org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse.*;
-
 /**
  * Stack advisor request.
  */
@@ -42,6 +40,7 @@ public class StackAdvisorRequest {
   private Map<String, Set<String>> componentHostsMap = new HashMap<String,
Set<String>>();
   private Map<String, Set<String>> hostComponents = new HashMap<String, Set<String>>();
   private Map<String, Set<String>> hostGroupBindings = new HashMap<String,
Set<String>>();
+  private Map<String, Map<String, Map<String, String>>> configurations
= new HashMap<String, Map<String, Map<String, String>>>();
 
   public String getStackName() {
     return stackName;
@@ -83,6 +82,10 @@ public class StackAdvisorRequest {
     return hostGroupBindings;
   }
 
+  public Map<String, Map<String, Map<String, String>>> getConfigurations()
{
+    return configurations;
+  }
+
   private StackAdvisorRequest(String stackName, String stackVersion) {
     this.stackName = stackName;
     this.stackVersion = stackVersion;
@@ -125,11 +128,18 @@ public class StackAdvisorRequest {
       return this;
     }
 
-    public StackAdvisorRequestBuilder forHostsGroupBindings(Map<String, Set<String>>
hostGroupBindings) {
+    public StackAdvisorRequestBuilder forHostsGroupBindings(
+        Map<String, Set<String>> hostGroupBindings) {
       this.instance.hostGroupBindings = hostGroupBindings;
       return this;
     }
 
+    public StackAdvisorRequestBuilder withConfigurations(
+        Map<String, Map<String, Map<String, String>>> configurations) {
+      this.instance.configurations = configurations;
+      return this;
+    }
+
     public StackAdvisorRequest build() {
       return this.instance;
     }

http://git-wip-us.apache.org/repos/asf/ambari/blob/60680064/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java
index 48946aa..e65f97f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationRecommnedationCommand.java
@@ -49,7 +49,8 @@ public class GetConfigurationRecommnedationCommand extends
 
   @Override
   protected void validate(StackAdvisorRequest request) throws StackAdvisorException {
-    if (request.getHosts().isEmpty() || request.getServices().isEmpty()) {
+    if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices()
== null
+        || request.getServices().isEmpty()) {
       throw new StackAdvisorException("Hosts and services must not be empty");
     }
   }

http://git-wip-us.apache.org/repos/asf/ambari/blob/60680064/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationValidationCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationValidationCommand.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationValidationCommand.java
new file mode 100644
index 0000000..2cd6481
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/GetConfigurationValidationCommand.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.api.services.stackadvisor.commands;
+
+import java.io.File;
+
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorException;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest;
+import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRunner;
+import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
+
+/**
+ * {@link StackAdvisorCommand} implementation for configuration validation.
+ */
+public class GetConfigurationValidationCommand extends StackAdvisorCommand<ValidationResponse>
{
+
+  public GetConfigurationValidationCommand(File recommendationsDir, String stackAdvisorScript,
+      int requestId, StackAdvisorRunner saRunner) {
+    super(recommendationsDir, stackAdvisorScript, requestId, saRunner);
+  }
+
+  @Override
+  protected StackAdvisorCommandType getCommandType() {
+    return StackAdvisorCommandType.VALIDATE_CONFIGURATIONS;
+  }
+
+  @Override
+  protected void validate(StackAdvisorRequest request) throws StackAdvisorException {
+    if (request.getHosts() == null || request.getHosts().isEmpty() || request.getServices()
== null
+        || request.getServices().isEmpty()) {
+      throw new StackAdvisorException("Hosts, services and configurations must not be empty");
+    }
+  }
+
+  @Override
+  protected ValidationResponse updateResponse(StackAdvisorRequest request,
+      ValidationResponse response) {
+    return response;
+  }
+
+  @Override
+  protected String getResultFileName() {
+    return "configurations-validation.json";
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/60680064/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
index 1a30d38..607e337 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java
@@ -78,6 +78,7 @@ public abstract class StackAdvisorCommand<T> extends BaseService {
   private static final String COMPONENT_INFO_PROPETRY = "StackServiceComponents";
   private static final String COMPONENT_NAME_PROPERTY = "component_name";
   private static final String COMPONENT_HOSTNAMES_PROPETRY = "hostnames";
+  private static final String CONFIGURATIONS_PROPETRY = "configurations";
 
   private File recommendationsDir;
   private String stackAdvisorScript;
@@ -133,6 +134,7 @@ public abstract class StackAdvisorCommand<T> extends BaseService
{
       ObjectNode root = (ObjectNode) this.mapper.readTree(data.servicesJSON);
 
       populateComponentHostsMap(root, request.getComponentHostsMap());
+      populateConfigurations(root, request.getConfigurations());
 
       data.servicesJSON = mapper.writeValueAsString(root);
     } catch (Exception e) {
@@ -145,6 +147,25 @@ public abstract class StackAdvisorCommand<T> extends BaseService
{
     return data;
   }
 
+  private void populateConfigurations(ObjectNode root,
+      Map<String, Map<String, Map<String, String>>> configurations) {
+    ObjectNode configurationsNode = root.putObject(CONFIGURATIONS_PROPETRY);
+    for (String siteName : configurations.keySet()) {
+      ObjectNode siteNode = configurationsNode.putObject(siteName);
+
+      Map<String, Map<String, String>> siteMap = configurations.get(siteName);
+      for (String properties : siteMap.keySet()) {
+        ObjectNode propertiesNode = siteNode.putObject(properties);
+
+        Map<String, String> propertiesMap = siteMap.get(properties);
+        for (String propertyName : propertiesMap.keySet()) {
+          String propertyValue = propertiesMap.get(propertyName);
+          propertiesNode.put(propertyName, propertyValue);
+        }
+      }
+    }
+  }
+
   private void populateComponentHostsMap(ObjectNode root, Map<String, Set<String>>
componentHostsMap) {
     ArrayNode services = (ArrayNode) root.get(SERVICES_PROPETRY);
     Iterator<JsonNode> servicesIter = services.getElements();

http://git-wip-us.apache.org/repos/asf/ambari/blob/60680064/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
index 95024d2..40b423e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/StackAdvisorResourceProvider.java
@@ -95,11 +95,14 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
       List<String> services = (List<String>) getRequestProperty(request, SERVICES_PROPERTY);
       Map<String, Set<String>> hgComponentsMap = calculateHostGroupComponentsMap(request);
       Map<String, Set<String>> hgHostsMap = calculateHostGroupHostsMap(request);
-      Map<String, Set<String>> componentHostsMap = calculateComponentHostsMap(hgComponentsMap,
hgHostsMap);
+      Map<String, Set<String>> componentHostsMap = calculateComponentHostsMap(hgComponentsMap,
+          hgHostsMap);
+      Map<String, Map<String, Map<String, String>>> configurations = calculateConfigurations(request);
 
       StackAdvisorRequest saRequest = StackAdvisorRequestBuilder.forStack(stackName, stackVersion)
-          .ofType(requestType).forHosts(hosts).forServices(services).forHostComponents(hgComponentsMap)
-          .forHostsGroupBindings(hgHostsMap).withComponentHostsMap(componentHostsMap).build();
+          .ofType(requestType).forHosts(hosts).forServices(services)
+          .forHostComponents(hgComponentsMap).forHostsGroupBindings(hgHostsMap)
+          .withComponentHostsMap(componentHostsMap).withConfigurations(configurations).build();
 
       return saRequest;
     } catch (Exception e) {
@@ -120,8 +123,8 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
    */
   @SuppressWarnings("unchecked")
   private Map<String, Set<String>> calculateHostGroupComponentsMap(Request request)
{
-    Set<Map<String, Object>> hostGroups =
-        (Set<Map<String, Object>>) getRequestProperty(request, BLUEPRINT_HOST_GROUPS_PROPERTY);
+    Set<Map<String, Object>> hostGroups = (Set<Map<String, Object>>)
getRequestProperty(request,
+        BLUEPRINT_HOST_GROUPS_PROPERTY);
     Map<String, Set<String>> map = new HashMap<String, Set<String>>();
     if (hostGroups != null) {
       for (Map<String, Object> hostGroup : hostGroups) {
@@ -151,8 +154,8 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
    */
   @SuppressWarnings("unchecked")
   private Map<String, Set<String>> calculateHostGroupHostsMap(Request request)
{
-    Set<Map<String, Object>> bindingHostGroups =
-        (Set<Map<String, Object>>) getRequestProperty(request, BINDING_HOST_GROUPS_PROPERTY);
+    Set<Map<String, Object>> bindingHostGroups = (Set<Map<String, Object>>)
getRequestProperty(
+        request, BINDING_HOST_GROUPS_PROPERTY);
     Map<String, Set<String>> map = new HashMap<String, Set<String>>();
     if (bindingHostGroups != null) {
       for (Map<String, Object> hostGroup : bindingHostGroups) {
@@ -173,9 +176,46 @@ public abstract class StackAdvisorResourceProvider extends ReadOnlyResourceProvi
     return map;
   }
 
+  private static final String CONFIGURATIONS_PROPERTY_ID = "recommendations/blueprint/configurations/";
+
+  private Map<String, Map<String, Map<String, String>>> calculateConfigurations(Request
request) {
+    Map<String, Map<String, Map<String, String>>> configurations = new
HashMap<String, Map<String, Map<String, String>>>();
+    Map<String, Object> properties = request.getProperties().iterator().next();
+    for (String property : properties.keySet()) {
+      if (property.startsWith(CONFIGURATIONS_PROPERTY_ID)) {
+        try {
+          String propertyEnd = property.substring(CONFIGURATIONS_PROPERTY_ID.length()); //
mapred-site/properties/yarn.app.mapreduce.am.resource.mb
+          String[] propertyPath = propertyEnd.split("/"); // length == 3
+          String siteName = propertyPath[0];
+          String propertiesProperty = propertyPath[1];
+          String propertyName = propertyPath[2];
+
+          Map<String, Map<String, String>> siteMap = configurations.get(siteName);
+          if (siteMap == null) {
+            siteMap = new HashMap<String, Map<String, String>>();
+            configurations.put(siteName, siteMap);
+          }
+
+          Map<String, String> propertiesMap = siteMap.get(propertiesProperty);
+          if (propertiesMap == null) {
+            propertiesMap = new HashMap<String, String>();
+            siteMap.put(propertiesProperty, propertiesMap);
+          }
+
+          String value = (String) properties.get(property);
+          propertiesMap.put(propertyName, value);
+        } catch (Exception e) {
+          LOG.debug(String.format("Error handling configuration property, name = %s", property),
e);
+          // do nothing
+        }
+      }
+    }
+    return configurations;
+  }
+
   @SuppressWarnings("unchecked")
   private Map<String, Set<String>> calculateComponentHostsMap(Map<String,
Set<String>> hostGroups,
-                                                              Map<String, Set<String>>
bindingHostGroups) {
+      Map<String, Set<String>> bindingHostGroups) {
     /*
      * ClassCastException may occur in case of body inconsistency: property
      * missed, etc.

http://git-wip-us.apache.org/repos/asf/ambari/blob/60680064/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
index 402d95b..025caa2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py
@@ -17,6 +17,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
+import re
 import socket
 import sys
 
@@ -364,8 +365,150 @@ class HDP206StackAdvisor(StackAdvisor):
 
   def validateConfigurations(self, services, hosts):
     """Returns array of Validation objects about issues with configuration values provided
in services"""
+    stackName = services["Versions"]["stack_name"]
+    stackVersion = services["Versions"]["stack_version"]
+
+    validations = {
+      "Versions": {"stack_name": stackName, "stack_version": stackVersion},
+      "items": [ ]
+    }
+    items = validations["items"]
+
+    recommendations = self.recommendConfigurations(services, hosts)
+    recommendedDefaults = recommendations["recommendations"]["blueprint"]["configurations"]
+
+    configurations = services["configurations"]
+    for service in services["services"]:
+      serviceName = service["StackServices"]["service_name"]
+      if serviceName == "MAPREDUCE2":
+        mapReduceErrors = validateMapReduce2Configurations(getSiteProperties(configurations,
"mapred-site"), recommendedDefaults["mapred-site"]["properties"])
+        items.extend(mapReduceErrors)
+      elif serviceName == "HIVE":
+        hiveErrors = validateHiveConfigurations(getSiteProperties(configurations, "hive-site"),
recommendedDefaults["hive-site"]["properties"])
+        items.extend(hiveErrors)
+      elif serviceName == "STORM":
+        oozieErrors = [] #validateStormConfigurations(getSiteProperties(configurations, "storm-site"),
recommendedDefaults["storm-site"]["properties"])
+        items.extend(oozieErrors)
+      elif serviceName == "TEZ":
+        tezErrors = validateTezConfigurations(getSiteProperties(configurations, "tez-site"),
recommendedDefaults["tez-site"]["properties"])
+        items.extend(tezErrors)
+      elif serviceName == "YARN":
+        yarnErrors = validateYARNConfigurations(getSiteProperties(configurations, "yarn-site"),
recommendedDefaults["yarn-site"]["properties"])
+        items.extend(yarnErrors)
+      else:
+        pass
+    return validations
     pass
 
+def getSiteProperties(configurations, siteName):
+  if configurations[siteName] is None:
+    return {}
+  if configurations[siteName]["properties"] is None:
+    return {}
+  return configurations[siteName]["properties"]
+
+def to_number(s):
+  try:
+    return int(re.sub("\D", "", s))
+  except ValueError:
+    return None
+
+def toConfigurationValidationErrors(items, siteName):
+  result = []
+  for item in items:
+    if item["message"] is not None:
+      error = { "type": 'configuration', "level": 'ERROR', "message": item["message"], "config-type":
siteName, "config-name": item["config-name"] }
+      result.append(error)
+  return result
+
+def validatorLessThenDefaultValue(properties, recommendedDefaults, propertyName):
+  value = to_number(properties[propertyName])
+  if value is None:
+    return "Value should be integer"
+  defaultValue = to_number(recommendedDefaults[propertyName])
+  if defaultValue is None:
+    return None
+  if value < defaultValue:
+    return "Value is less than the recommended default of {0}".format(defaultValue)
+  return None
+
+def validateXmxValue(properties, recommendedDefaults, propertyName):
+  value = properties[propertyName]
+  defaultValue = recommendedDefaults[propertyName]
+  if defaultValue is None:
+    return "Config's default value can't be null or undefined"
+  if not checkXmxValueFormat(value):
+    return 'Invalid value format'
+  valueInt = formatXmxSizeToBytes(getXmxSize(value))
+  defaultValueXmx = getXmxSize(defaultValue)
+  defaultValueInt = formatXmxSizeToBytes(defaultValueXmx)
+  if valueInt < defaultValueInt:
+    return "Value is less than the recommended default of -Xmx" + defaultValueXmx
+  return None
+
+def checkXmxValueFormat(value):
+  p = re.compile('-Xmx(\d+)(b|k|m|g|p|t|B|K|M|G|P|T)?')
+  matches = p.findall(value)
+  return len(matches) == 1
+
+def getXmxSize(value):
+  p = re.compile("-Xmx(\d+)(.?)")
+  result = p.findall(value)[0]
+  if len(result) > 1:
+    # result[1] - is a space or size formatter (b|k|m|g etc)
+    return result[0] + result[1].lower()
+  return result[0]
+
+def formatXmxSizeToBytes(value):	
+  value = value.lower()
+  if len(value) == 0:
+    return 0
+  modifier = value[-1]
+
+  if modifier == ' ' or modifier in "0123456789":
+    modifier = 'b'
+  m = {
+    modifier == 'b': 1,
+    modifier == 'k': 1024,
+    modifier == 'm': 1024 * 1024,
+    modifier == 'g': 1024 * 1024 * 1024,
+    modifier == 't': 1024 * 1024 * 1024 * 1024,
+    modifier == 'p': 1024 * 1024 * 1024 * 1024 * 1024
+    }[1]
+  return to_number(value) * m
+
+def validateMapReduce2Configurations(properties, recommendedDefaults):
+  validationItems = [ {"config-name": 'mapreduce.map.java.opts', "message": validateXmxValue(properties,
recommendedDefaults, 'mapreduce.map.java.opts')},
+                      {"config-name": 'mapreduce.reduce.java.opts', "message": validateXmxValue(properties,
recommendedDefaults, 'mapreduce.reduce.java.opts')},
+                      {"config-name": 'mapreduce.task.io.sort.mb', "message": validatorLessThenDefaultValue(properties,
recommendedDefaults, 'mapreduce.task.io.sort.mb')},
+                      {"config-name": 'mapreduce.map.memory.mb', "message": validatorLessThenDefaultValue(properties,
recommendedDefaults, 'mapreduce.map.memory.mb')},
+                      {"config-name": 'mapreduce.reduce.memory.mb', "message": validatorLessThenDefaultValue(properties,
recommendedDefaults, 'mapreduce.reduce.memory.mb')},
+                      {"config-name": 'yarn.app.mapreduce.am.resource.mb', "message": validatorLessThenDefaultValue(properties,
recommendedDefaults, 'yarn.app.mapreduce.am.resource.mb')},
+                      {"config-name": 'yarn.app.mapreduce.am.command-opts', "message": validateXmxValue(properties,
recommendedDefaults, 'yarn.app.mapreduce.am.command-opts')} ]
+  return toConfigurationValidationErrors(validationItems, "mapred-site")
+
+def validateHiveConfigurations(properties, recommendedDefaults):
+  validationItems = [ {"config-name": 'hive.tez.container.size', "message": validatorLessThenDefaultValue(properties,
recommendedDefaults, 'hive.tez.container.size')},
+                      {"config-name": 'hive.tez.java.opts', "message": validateXmxValue(properties,
recommendedDefaults, 'hive.tez.java.opts')},
+                      {"config-name": 'hive.auto.convert.join.noconditionaltask.size', "message":
validatorLessThenDefaultValue(properties, recommendedDefaults, 'hive.auto.convert.join.noconditionaltask.size')}
]
+  return toConfigurationValidationErrors(validationItems, "hive-site")
+
+def validateStormConfigurations(properties, recommendedDefaults):
+  validationItems = [ {"config-name": 'drpc.childopts', "message": validateXmxValue(properties,
recommendedDefaults, 'drpc.childopts')},
+                      {"config-name": 'ui.childopts', "message": validateXmxValue(properties,
recommendedDefaults, 'ui.childopts')},
+                      {"config-name": 'logviewer.childopts', "message": validateXmxValue(properties,
recommendedDefaults, 'logviewer.childopts')} ]
+  return toConfigurationValidationErrors(validationItems, "storm-site")
+
+def validateTezConfigurations(properties, recommendedDefaults):
+  validationItems = [ {"config-name": 'tez.am.resource.memory.mb', "message": validatorLessThenDefaultValue(properties,
recommendedDefaults, 'tez.am.resource.memory.mb')},
+                      {"config-name": 'tez.am.java.opts', "message": validateXmxValue(properties,
recommendedDefaults, 'tez.am.java.opts')} ]
+  return toConfigurationValidationErrors(validationItems, "tez-site")
+
+def validateYARNConfigurations(properties, recommendedDefaults):
+  validationItems = [ {"config-name": 'yarn.nodemanager.resource.memory-mb', "message": validatorLessThenDefaultValue(properties,
recommendedDefaults, 'yarn.nodemanager.resource.memory-mb')},
+                      {"config-name": 'yarn.scheduler.minimum-allocation-mb', "message":
validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')},
+                      {"config-name": 'yarn.scheduler.maximum-allocation-mb', "message":
validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')}
]
+  return toConfigurationValidationErrors(validationItems, "yarn-site")
 
 # Helper methods
 def getHostForComponent(component, hostsList):

http://git-wip-us.apache.org/repos/asf/ambari/blob/60680064/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelperTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelperTest.java
index 6f8c42b..e4cf97a 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/stackadvisor/StackAdvisorHelperTest.java
@@ -31,6 +31,7 @@ import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.St
 import org.apache.ambari.server.api.services.stackadvisor.StackAdvisorRequest.StackAdvisorRequestType;
 import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutRecommnedationCommand;
 import org.apache.ambari.server.api.services.stackadvisor.commands.GetComponentLayoutValidationCommand;
+import org.apache.ambari.server.api.services.stackadvisor.commands.GetConfigurationValidationCommand;
 import org.apache.ambari.server.api.services.stackadvisor.commands.StackAdvisorCommand;
 import org.apache.ambari.server.api.services.stackadvisor.recommendations.RecommendationResponse;
 import org.apache.ambari.server.api.services.stackadvisor.validations.ValidationResponse;
@@ -149,4 +150,17 @@ public class StackAdvisorHelperTest {
     assertEquals(GetComponentLayoutValidationCommand.class, command.getClass());
   }
 
+  @Test
+  public void testCreateValidationCommand_returnsGetConfigurationValidationCommand()
+      throws IOException, StackAdvisorException {
+    Configuration configuration = mock(Configuration.class);
+    StackAdvisorRunner saRunner = mock(StackAdvisorRunner.class);
+    StackAdvisorHelper helper = new StackAdvisorHelper(configuration, saRunner);
+    StackAdvisorRequestType requestType = StackAdvisorRequestType.CONFIGURATIONS;
+
+    StackAdvisorCommand<ValidationResponse> command = helper.createValidationCommand(requestType);
+
+    assertEquals(GetConfigurationValidationCommand.class, command.getClass());
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/60680064/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
new file mode 100644
index 0000000..3139a3e
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -0,0 +1,248 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import socket
+from unittest import TestCase
+
+class TestHDP206StackAdvisor(TestCase):
+
+  def setUp(self):
+    import imp
+    import os
+
+    testDirectory = os.path.dirname(os.path.abspath(__file__))
+    stackAdvisorPath = os.path.join(testDirectory, '../../../../../main/resources/stacks/HDP/stack_advisor.py')
+    hdp206StackAdvisorPath = os.path.join(testDirectory, '../../../../../main/resources/stacks/HDP/2.0.6/services/stack_advisor.py')
+    hdp206StackAdvisorClassName = 'HDP206StackAdvisor'
+    with open(stackAdvisorPath, 'rb') as fp:
+      stack_advisor = imp.load_module( 'stack_advisor', fp, stackAdvisorPath, ('.py', 'rb',
imp.PY_SOURCE) )
+    with open(hdp206StackAdvisorPath, 'rb') as fp:
+      stack_advisor_impl = imp.load_module('stack_advisor_impl', fp, hdp206StackAdvisorPath,
('.py', 'rb', imp.PY_SOURCE))
+    clazz = getattr(stack_advisor_impl, hdp206StackAdvisorClassName)
+    self.stackAdvisor = clazz()
+
+  def test_recommendationCardinalityALL(self):
+    servicesInfo = [
+      {
+        "name": "GANGLIA",
+        "components": [{"name": "GANGLIA_MONITOR", "cardinality": "ALL", "category": "SLAVE",
"is_master": False}]
+      }
+    ]
+    services = self.prepareServices(servicesInfo)
+    hosts = self.prepareHosts(["host1", "host2"])
+    result = self.stackAdvisor.recommendComponentLayout(services, hosts)
+
+    expectedComponentsHostsMap = {
+      "GANGLIA_MONITOR": ["host1", "host2"]
+    }
+    self.assertHostLayout(expectedComponentsHostsMap, result)
+
+  def test_recommendationAssignmentNotChanged(self):
+    servicesInfo = [
+      {
+        "name": "GANGLIA",
+        "components": [{"name": "GANGLIA_MONITOR", "cardinality": "ALL", "category": "SLAVE",
"is_master": False, "hostnames": ["host1"]}]
+      }
+    ]
+    services = self.prepareServices(servicesInfo)
+    hosts = self.prepareHosts(["host1", "host2"])
+    result = self.stackAdvisor.recommendComponentLayout(services, hosts)
+
+    expectedComponentsHostsMap = {
+      "GANGLIA_MONITOR": ["host1"]
+    }
+    self.assertHostLayout(expectedComponentsHostsMap, result)
+
+  def test_recommendationIsNotPreferableOnAmbariServer(self):
+    servicesInfo = [
+      {
+        "name": "GANGLIA",
+        "components": [{"name": "GANGLIA_SERVER", "cardinality": "ALL", "category": "MASTER",
"is_master": True}]
+      }
+    ]
+    services = self.prepareServices(servicesInfo)
+    localhost = socket.getfqdn()
+    hosts = self.prepareHosts([localhost, "host2"])
+    result = self.stackAdvisor.recommendComponentLayout(services, hosts)
+
+    expectedComponentsHostsMap = {
+      "GANGLIA_SERVER": ["host2"]
+    }
+    self.assertHostLayout(expectedComponentsHostsMap, result)
+
+  def test_validationNamenodeAndSecondaryNamenode2Hosts(self):
+    servicesInfo = [
+      {
+        "name": "HDFS",
+        "components": [
+          {"name": "NAMENODE", "cardinality": "1-2", "category": "MASTER", "is_master": True,
"hostnames": ["host1"]},
+          {"name": "SECONDARY_NAMENODE", "cardinality": "1", "category": "MASTER", "is_master":
True, "hostnames": ["host1"]}]
+      }
+    ]
+    services = self.prepareServices(servicesInfo)
+    hosts = self.prepareHosts(["host1", "host2"])
+    result = self.stackAdvisor.validateComponentLayout(services, hosts)
+
+    expectedMessages = [
+      "NameNode and Secondary NameNode cannot be hosted on same machine",
+      "NameNode and Secondary NameNode cannot be hosted on same machine",
+      "Host is not used"
+    ]
+    self.assertValidationMessages(expectedMessages, result)
+
+  def test_validationCardinalityALL(self):
+    servicesInfo = [
+      {
+        "name": "GANGLIA",
+        "components": [
+          {"name": "GANGLIA_MONITOR", "cardinality": "ALL", "category": "SLAVE", "is_master":
False, "hostnames": ["host1"]},
+          {"name": "GANGLIA_SERVER", "cardinality": "1-2", "category": "MASTER", "is_master":
True, "hostnames": ["host2", "host1"]}
+        ]
+      }
+    ]
+    services = self.prepareServices(servicesInfo)
+    hosts = self.prepareHosts(["host1", "host2"])
+    result = self.stackAdvisor.validateComponentLayout(services, hosts)
+
+    expectedMessages = [
+      "Cardinality violation, cardinality=ALL, hosts count=1"
+    ]
+    self.assertValidationMessages(expectedMessages, result)
+
+  def test_validationHostIsNotUsedForNonValuableComponent(self):
+    servicesInfo = [
+      {
+        "name": "GANGLIA",
+        "components": [
+          {"name": "GANGLIA_MONITOR", "cardinality": "ALL", "category": "SLAVE", "is_master":
False, "hostnames": ["host1", "host2"]},
+          {"name": "GANGLIA_SERVER", "cardinality": "1", "category": "MASTER", "is_master":
True, "hostnames": ["host2"]}
+        ]
+      }
+    ]
+    services = self.prepareServices(servicesInfo)
+    hosts = self.prepareHosts(["host1", "host2"])
+    result = self.stackAdvisor.validateComponentLayout(services, hosts)
+
+    expectedMessages = [
+      "Host is not used"
+    ]
+    self.assertValidationMessages(expectedMessages, result)
+
+  def test_validationCardinality01TwoHostsAssigned(self):
+    servicesInfo = [
+      {
+        "name": "GANGLIA",
+        "components": [
+          {"name": "GANGLIA_SERVER", "cardinality": "0-1", "category": "MASTER", "is_master":
True, "hostnames": ["host1", "host2"]}
+        ]
+      }
+    ]
+    services = self.prepareServices(servicesInfo)
+    hosts = self.prepareHosts(["host1", "host2"])
+    result = self.stackAdvisor.validateComponentLayout(services, hosts)
+
+    expectedMessages = [
+      "Cardinality violation, cardinality=0-1, hosts count=2"
+    ]
+    self.assertValidationMessages(expectedMessages, result)
+
+  def test_validationHostIsNotUsed(self):
+    servicesInfo = [
+      {
+        "name": "GANGLIA",
+        "components": [
+          {"name": "GANGLIA_SERVER", "cardinality": "1", "category": "MASTER", "is_master":
True, "hostnames": ["host1"]}
+        ]
+      }
+    ]
+    services = self.prepareServices(servicesInfo)
+    hosts = self.prepareHosts(["host1", "host2"])
+    result = self.stackAdvisor.validateComponentLayout(services, hosts)
+
+    expectedMessages = [
+      "Host is not used"
+    ]
+    self.assertValidationMessages(expectedMessages, result)
+
+
+  def prepareHosts(self, hostsNames):
+    hosts = { "items": [] }
+    for hostName in hostsNames:
+      nextHost = {"Hosts":{"host_name" : hostName}}
+      hosts["items"].append(nextHost)
+    return hosts
+
+  def prepareServices(self, servicesInfo):
+    services = { "Versions" : { "stack_name" : "HDP", "stack_version" : "2.0.6" } }
+    services["services"] = []
+
+    for serviceInfo in servicesInfo:
+      nextService = {"StackServices":{"service_name" : serviceInfo["name"]}}
+      nextService["components"] = []
+      for component in serviceInfo["components"]:
+        nextComponent = {
+          "StackServiceComponents": {
+            "component_name": component["name"],
+            "cardinality": component["cardinality"],
+            "component_category": component["category"],
+            "is_master": component["is_master"]
+          }
+        }
+        try:
+          nextComponent["StackServiceComponents"]["hostnames"] = component["hostnames"]
+        except KeyError, err:
+          nextComponent["StackServiceComponents"]["hostnames"] = []
+        nextService["components"].append(nextComponent)
+      services["services"].append(nextService)
+
+    return services
+
+  def assertHostLayout(self, componentsHostsMap, recommendation):
+    blueprintMapping = recommendation["recommendations"]["blueprint"]["host_groups"]
+    bindings = recommendation["recommendations"]["blueprint_cluster_binding"]["host_groups"]
+
+    actualComponentHostsMap = {}
+    for hostGroup in blueprintMapping:
+      hostGroupName = hostGroup["name"]
+      hostsInfos = [binding["hosts"] for binding in bindings if binding["name"] == hostGroupName][0]
+      hosts = [info["fqdn"] for info in hostsInfos]
+
+      for component in hostGroup["components"]:
+        componentName = component["name"]
+        try:
+          actualComponentHostsMap[componentName]
+        except KeyError, err:
+          actualComponentHostsMap[componentName] = []
+        for host in hosts:
+          if host not in actualComponentHostsMap[componentName]:
+            actualComponentHostsMap[componentName].append(host)
+
+    for componentName in componentsHostsMap.keys():
+      expectedHosts = componentsHostsMap[componentName]
+      actualHosts = actualComponentHostsMap[componentName]
+      self.checkEqual(expectedHosts, actualHosts)
+
+  def checkEqual(self, l1, l2):
+    if not len(l1) == len(l2) or not sorted(l1) == sorted(l2):
+      raise AssertionError("list1={0}, list2={1}".format(l1, l2))
+
+  def assertValidationMessages(self, expectedMessages, result):
+    realMessages = [item["message"] for item in result["items"]]
+    self.checkEqual(expectedMessages, realMessages)
+
+


Mime
View raw message