ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-18355: Conditional dependencies in stack defition to handle blueprint validation gracefully (Amruta Borkar via dili)
Date Mon, 14 Nov 2016 20:34:39 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk a1906001c -> bc7a29f54


AMBARI-18355: Conditional dependencies in stack defition to handle blueprint validation gracefully
(Amruta Borkar via dili)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/bc7a29f5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/bc7a29f5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/bc7a29f5

Branch: refs/heads/trunk
Commit: bc7a29f54046978b79f659d94a17f6335f693191
Parents: a190600
Author: Di Li <dili@apache.org>
Authored: Mon Nov 14 15:33:36 2016 -0500
Committer: Di Li <dili@apache.org>
Committed: Mon Nov 14 15:33:36 2016 -0500

----------------------------------------------------------------------
 .../server/state/DependencyConditionInfo.java   | 104 +++++++++++++++++++
 .../ambari/server/state/DependencyInfo.java     |  37 ++++++-
 .../server/topology/BlueprintValidatorImpl.java |  18 +++-
 .../common-services/HDFS/2.1.0.2.0/metainfo.xml |  43 +++++++-
 .../server/topology/BlueprintImplTest.java      |  41 --------
 .../topology/BlueprintValidatorImplTest.java    |  75 ++++++++++++-
 6 files changed, 270 insertions(+), 48 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/bc7a29f5/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java
new file mode 100644
index 0000000..66c3e2b
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.state;
+
+import java.util.Map;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlSeeAlso;
+import javax.xml.bind.annotation.XmlTransient;
+
+/**
+ * Represents stack component dependency condition information provided in
+ * metainfo.xml file.
+ */
+@XmlTransient
+@XmlSeeAlso({ PropertyExists.class, PropertyValueEquals.class })
+public interface DependencyConditionInfo {
+  /**
+   * Returns true if the dependency condition is satisfied
+   *
+   * @param properties
+   * @return boolean
+   */
+  boolean isResolved(Map<String, Map<String, String>> properties);
+}
+
+/**
+ * This class represents a dependency condition that is based on existence of a
+ * certain property.
+ */
+class PropertyExists implements DependencyConditionInfo {
+  /**
+   * configType of the conditional dependency
+   */
+  protected String configType;
+
+  /**
+   * property of the conditional dependency
+   */
+  protected String property;
+
+  @XmlElement
+  public String getProperty() {
+    return property;
+  }
+
+  public void setProperty(String property) {
+    this.property = property;
+  }
+
+  @XmlElement
+  public String getConfigType() {
+    return configType;
+  }
+
+  public void setConfigType(String configType) {
+    this.configType = configType;
+  }
+
+  @Override
+  public boolean isResolved(Map<String, Map<String, String>> properties) {
+    return (properties.get(configType).containsKey(property));
+  }
+}
+
+/**
+ * This class represents a dependency condition that is based on value of a
+ * certain property.
+ */
+class PropertyValueEquals extends PropertyExists {
+  /**
+   * propertyValue of the property
+   */
+  protected String propertyValue;
+
+  @XmlElement
+  public String getPropertyValue() {
+    return propertyValue;
+  }
+
+  public void setPropertyValue(String propertyValue) {
+    this.propertyValue = propertyValue;
+  }
+
+  @Override
+  public boolean isResolved(Map<String, Map<String, String>> properties) {
+    return (super.isResolved(properties) && propertyValue.equals(properties.get(configType).get(property)));
+  }
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/bc7a29f5/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
index e3db662..540767c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
@@ -19,7 +19,12 @@
 package org.apache.ambari.server.state;
 
 
+import java.util.ArrayList;
+import java.util.List;
 import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlElementWrapper;
+import javax.xml.bind.annotation.XmlElements;
+import org.apache.commons.collections.CollectionUtils;
 
 /**
  * Represents stack component dependency information.
@@ -54,7 +59,10 @@ public class DependencyInfo {
   @XmlElement(name="auto-deploy")
   private AutoDeployInfo m_autoDeploy;
 
-
+  /**
+   * Conditions for Component dependency to other components.
+   */
+  private List<DependencyConditionInfo> dependencyConditions = new ArrayList<DependencyConditionInfo>();
   /**
    * Setter for name property.
    *
@@ -135,6 +143,33 @@ public class DependencyInfo {
   public String getServiceName() {
     return serviceName;
   }
+  /**
+   * Get the dependencyConditions list
+   *
+   * @return dependencyConditions
+   */
+  @XmlElementWrapper(name="conditions")
+  @XmlElements(@XmlElement(name="condition"))
+  public List<DependencyConditionInfo> getDependencyConditions() {
+    return dependencyConditions;
+  }
+
+  /**
+   * Set dependencyConditions
+   *
+   * @param dependencyConditions
+   */
+  public void setDependencyConditions(List<DependencyConditionInfo> dependencyConditions)
{
+    this.dependencyConditions = dependencyConditions;
+  }
+
+  /**
+   * Confirms if dependency have any condition or not
+   * @return true if dependencies are based on a condition
+   */
+  public boolean hasDependencyConditions(){
+    return !CollectionUtils.isEmpty(dependencyConditions);
+  }
 
   @Override
   public String toString() {

http://git-wip-us.apache.org/repos/asf/ambari/blob/bc7a29f5/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
index a5f33ff..c5647c3 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
@@ -26,10 +26,10 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
-import java.util.regex.Pattern;
 
 import org.apache.ambari.server.controller.internal.Stack;
 import org.apache.ambari.server.state.AutoDeployInfo;
+import org.apache.ambari.server.state.DependencyConditionInfo;
 import org.apache.ambari.server.state.DependencyInfo;
 import org.apache.ambari.server.utils.SecretReference;
 import org.apache.ambari.server.utils.VersionUtils;
@@ -135,9 +135,6 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
           }
         }
         if (ClusterTopologyImpl.isNameNodeHAEnabled(clusterConfigurations) && component.equals("NAMENODE"))
{
-            if(!hostGroup.getComponentNames().contains("ZKFC")){
-              throw new InvalidTopologyException("Compoenent ZKFC is mandatory for hostgroup
" + hostGroup+" when NAMENODE HA is enabled");
-            }
             Map<String, String> hadoopEnvConfig = clusterConfigurations.get("hadoop-env");
             if(hadoopEnvConfig != null && !hadoopEnvConfig.isEmpty() && hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_active")
&& hadoopEnvConfig.containsKey("dfs_ha_initial_namenode_standby")) {
               ArrayList<HostGroup> hostGroupsForComponent = new ArrayList<HostGroup>(
blueprint.getHostGroupsForComponent(component));
@@ -289,6 +286,19 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
         AutoDeployInfo autoDeployInfo  = dependency.getAutoDeploy();
         boolean        resolved        = false;
 
+        //check if conditions are met, if any
+        if(dependency.hasDependencyConditions()) {
+          boolean conditionsSatisfied = true;
+          for (DependencyConditionInfo dependencyCondition : dependency.getDependencyConditions())
{
+            if (!dependencyCondition.isResolved(blueprint.getConfiguration().getFullProperties()))
{
+              conditionsSatisfied = false;
+              break;
+            }
+          }
+          if(!conditionsSatisfied){
+            continue;
+          }
+        }
         if (dependencyScope.equals("cluster")) {
           Collection<String> missingDependencyInfo = verifyComponentCardinalityCount(
               componentName, new Cardinality("1+"), autoDeployInfo);

http://git-wip-us.apache.org/repos/asf/ambari/blob/bc7a29f5/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
index 9c1387d..2b7a684 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
@@ -15,7 +15,7 @@
    See the License for the specific language governing permissions and
    limitations under the License.
 -->
-<metainfo>
+<metainfo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
   <schemaVersion>2.0</schemaVersion>
   <services>
     <service>
@@ -32,6 +32,47 @@
           <cardinality>1-2</cardinality>
           <versionAdvertised>true</versionAdvertised>
           <reassignAllowed>true</reassignAllowed>
+          <dependencies>
+            <dependency>
+              <name>HDFS/ZKFC</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>false</enabled>
+              </auto-deploy>
+              <conditions>
+                <condition xsi:type="propertyExists">
+                  <configType>hdfs-site</configType>
+                  <property>dfs.nameservices</property>
+                </condition>
+              </conditions>
+            </dependency>
+            <dependency>
+              <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>false</enabled>
+              </auto-deploy>
+              <conditions>
+                <condition xsi:type="propertyExists">
+                  <configType>hdfs-site</configType>
+                  <property>dfs.nameservices</property>
+                </condition>
+              </conditions>
+            </dependency>
+            <dependency>
+              <name>HDFS/JOURNALNODE</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>false</enabled>
+              </auto-deploy>
+              <conditions>
+                <condition xsi:type="propertyExists">
+                  <configType>hdfs-site</configType>
+                  <property>dfs.nameservices</property>
+                </condition>
+              </conditions>
+            </dependency>
+          </dependencies>
           <commandScript>
             <script>scripts/namenode.py</script>
             <scriptType>PYTHON</scriptType>

http://git-wip-us.apache.org/repos/asf/ambari/blob/bc7a29f5/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
index ff9af17..6763b7c 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintImplTest.java
@@ -212,47 +212,6 @@ public class BlueprintImplTest {
     assertTrue(entity.getSecurityType() == SecurityType.NONE);
     assertTrue(entity.getSecurityDescriptorReference() == null);
   }
-  @Test(expected = InvalidTopologyException.class)
-  public void testValidateConfigurations__hostGroupConfigForNameNodeHANOZKFC() throws Exception
{
-    Map<String, Map<String, String>> group2Props = new HashMap<>();
-    Map<String, String> group2Category2Props = new HashMap<>();
-    group2Props.put("category2", group2Category2Props);
-    group2Category2Props.put("prop2", "val");
-    // set config for group2 which contains a required property
-    Configuration group2Configuration = new Configuration(group2Props, EMPTY_ATTRIBUTES,
configuration);
-    expect(group2.getConfiguration()).andReturn(group2Configuration).atLeastOnce();
-
-    expect(group1.getCardinality()).andReturn("1").atLeastOnce();
-    expect(group1.getComponents()).andReturn(Arrays.asList(new Component("NAMENODE"),new
Component("ZKFC"))).atLeastOnce();
-    expect(group2.getCardinality()).andReturn("1").atLeastOnce();
-    expect(group2.getComponents()).andReturn(Arrays.asList(new Component("NAMENODE"),new
Component("ZKFC"))).atLeastOnce();
-    Map<String, String> category2Props = new HashMap<>();
-    properties.put("category2", category2Props);
-    category2Props.put("prop2", "val");
-    group1Components.add("NAMENODE");
-    group1Components.add("ZKFC");
-    group2Components.add("NAMENODE");
-    expect(stack.getServiceForComponent("NAMENODE")).andReturn("SERVICE2").atLeastOnce();
-    expect(stack.getServiceForComponent("ZKFC")).andReturn("SERVICE2").atLeastOnce();
-    Map<String, String> hdfsProps = new HashMap<String, String>();
-    properties.put("hdfs-site", hdfsProps);
-    hdfsProps.put("foo", "val");
-    hdfsProps.put("bar", "val");
-    hdfsProps.put("dfs.nameservices", "val");
-    Map<String, String> hadoopProps = new HashMap<String, String>();
-    properties.put("hadoop-env", hadoopProps);
-    hadoopProps.put("dfs_ha_initial_namenode_active", "%HOSTGROUP::group1%");
-    hadoopProps.put("dfs_ha_initial_namenode_standby", "%HOSTGROUP::group2%");
-    replay(stack, group1, group2);
-
-    Blueprint blueprint = new BlueprintImpl("test", hostGroups, stack, configuration, null);
-    blueprint.validateRequiredProperties();
-    BlueprintEntity entity = blueprint.toEntity();
-
-    verify(stack, group1, group2);
-    assertTrue(entity.getSecurityType() == SecurityType.NONE);
-    assertTrue(entity.getSecurityDescriptorReference() == null);
-  }
 
   @Test(expected= IllegalArgumentException.class)
   public void testValidateConfigurations__hostGroupConfigForNameNodeHAInCorrectHostGroups()
throws Exception {

http://git-wip-us.apache.org/repos/asf/ambari/blob/bc7a29f5/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
index b1de8ef..1501c53 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
@@ -34,7 +34,9 @@ import java.util.Map;
 import org.apache.ambari.server.controller.internal.Stack;
 import org.apache.ambari.server.state.AutoDeployInfo;
 import org.apache.ambari.server.state.ComponentInfo;
+import org.apache.ambari.server.state.DependencyConditionInfo;
 import org.apache.ambari.server.state.DependencyInfo;
+import org.easymock.EasyMock;
 import org.easymock.EasyMockRule;
 import org.easymock.Mock;
 import org.easymock.MockType;
@@ -66,15 +68,22 @@ public class BlueprintValidatorImplTest {
 
   @Mock(type = MockType.NICE)
   private DependencyInfo dependency1;
+  @Mock(type = MockType.NICE)
+  private DependencyInfo dependency2;
 
   @Mock(type = MockType.NICE)
   private ComponentInfo dependencyComponentInfo;
+  @Mock(type = MockType.NICE)
+  private DependencyConditionInfo dependencyConditionInfo1;
+  @Mock(type = MockType.NICE)
+  private DependencyConditionInfo dependencyConditionInfo2;
 
   private final Collection<String> group1Components = new ArrayList<String>();
   private final Collection<String> group2Components = new ArrayList<String>();
   private final Collection<String> services = new ArrayList<String>();
 
   private Collection<DependencyInfo> dependencies1 = new ArrayList<DependencyInfo>();
+  private List<DependencyConditionInfo> dependenciesConditionInfos1 = new ArrayList<DependencyConditionInfo>();
   private AutoDeployInfo autoDeploy = new AutoDeployInfo();
   private Map<String, Map<String, String>> configProperties = new HashMap<String,
Map<String, String>>();
   private Configuration configuration = new Configuration(configProperties, Collections.<String,
Map<String, Map<String, String>>>emptyMap());
@@ -105,13 +114,15 @@ public class BlueprintValidatorImplTest {
     expect(stack.getCardinality("component1")).andReturn(new Cardinality("1"));
     expect(stack.getCardinality("component2")).andReturn(new Cardinality("1+"));
     expect(stack.getCardinality("component3")).andReturn(new Cardinality("1+"));
+    dependenciesConditionInfos1.add(dependencyConditionInfo1);
+    dependenciesConditionInfos1.add(dependencyConditionInfo2);
 
     expect(blueprint.getConfiguration()).andReturn(configuration).anyTimes();
   }
 
   @After
   public void tearDown() {
-    reset(blueprint, stack, group1, group2, dependency1);
+    reset(blueprint, stack, group1, group2, dependency1, dependency2, dependencyConditionInfo1,
dependencyConditionInfo2);
   }
 
   @Test
@@ -341,4 +352,66 @@ public class BlueprintValidatorImplTest {
     verify(group1);
 
   }
+  @Test(expected=InvalidTopologyException.class)
+  public void testWhenComponentIsConditionallyDependentAndOnlyOneOfTheConditionsIsSatisfied()
throws Exception {
+    // GIVEN
+    hostGroups.clear();
+    hostGroups.put("group1", group1);
+
+    group1Components.add("component-1");
+    dependencies1.add(dependency1);
+    dependencies1.add(dependency2);
+    services.addAll(Collections.singleton("service-1"));
+
+
+    expect(blueprint.getHostGroupsForComponent("component-1")).andReturn(Arrays.asList(group1)).anyTimes();
+    expect(blueprint.getName()).andReturn("blueprint-1").anyTimes();
+    Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
+    Map<String, String> typeProps = new HashMap<String, String>();
+    typeProps.put("yarn.resourcemanager.hostname", "testhost");
+    properties.put("yarn-site", typeProps);
+
+    Configuration clusterConfig = new Configuration(properties,
+       Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+    Cardinality cardinality = new Cardinality("1");
+
+    expect(stack.getComponents("service-1")).andReturn(Arrays.asList("component-1")).anyTimes();
+    expect(stack.getAutoDeployInfo("component-1")).andReturn(autoDeploy).anyTimes();
+    expect(stack.getDependenciesForComponent("component-1")).andReturn(dependencies1).anyTimes();
+    expect(stack.getCardinality("component-1")).andReturn(cardinality).anyTimes();
+
+    AutoDeployInfo dependencyAutoDeploy = null;
+
+    expect(dependency1.getScope()).andReturn("host").anyTimes();
+    expect(dependency1.getAutoDeploy()).andReturn(dependencyAutoDeploy).anyTimes();
+    expect(dependency1.getComponentName()).andReturn("component-d").anyTimes();
+    expect(dependency1.getServiceName()).andReturn("service-d").anyTimes();
+    expect(dependency1.getName()).andReturn("dependency-1").anyTimes();
+    expect(dependency1.hasDependencyConditions()).andReturn(true).anyTimes();
+    expect(dependency1.getDependencyConditions()).andReturn(dependenciesConditionInfos1).anyTimes();
+    expect(dependency2.getScope()).andReturn("host").anyTimes();
+    expect(dependency2.getAutoDeploy()).andReturn(dependencyAutoDeploy).anyTimes();
+    expect(dependency2.getComponentName()).andReturn("component-d").anyTimes();
+    expect(dependency2.getServiceName()).andReturn("service-d").anyTimes();
+    expect(dependency2.getName()).andReturn("dependency-2").anyTimes();
+    expect(dependency2.hasDependencyConditions()).andReturn(false).anyTimes();
+
+    expect(dependencyConditionInfo1.isResolved(EasyMock.anyObject(Map.class))).andReturn(true).anyTimes();
+    expect(dependencyConditionInfo2.isResolved(EasyMock.anyObject(Map.class))).andReturn(false).anyTimes();
+
+
+    expect(dependencyComponentInfo.isClient()).andReturn(false).anyTimes();
+    expect(stack.getComponentInfo("component-d")).andReturn(dependencyComponentInfo).anyTimes();
+
+    replay(blueprint, stack, group1, group2, dependency1, dependency2, dependencyComponentInfo,dependencyConditionInfo1,dependencyConditionInfo2);
+
+    // WHEN
+    BlueprintValidator validator = new BlueprintValidatorImpl(blueprint);
+    validator.validateTopology();
+
+    // THEN
+    verify(group1);
+
+  }
 }


Mime
View raw message