ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nc...@apache.org
Subject [31/32] ambari git commit: AMBARI-18355: Introduce conditional dependencies in stack defition to handle blueprint validation gracefully (Amruta Borkar via dili)
Date Fri, 07 Oct 2016 19:14:28 GMT
AMBARI-18355: Introduce conditional dependencies in stack defition to handle blueprint validation
gracefully (Amruta Borkar via dili)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f6124a05
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f6124a05
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f6124a05

Branch: refs/heads/branch-dev-patch-upgrade
Commit: f6124a056d2a8ed16bec917775b9d3554ab5d74d
Parents: c153cfb
Author: Di Li <dili@apache.org>
Authored: Fri Oct 7 13:34:54 2016 -0400
Committer: Di Li <dili@apache.org>
Committed: Fri Oct 7 13:34:54 2016 -0400

----------------------------------------------------------------------
 .../server/state/DependencyConditionInfo.java   | 102 +++++++++++++++++++
 .../ambari/server/state/DependencyInfo.java     |  36 ++++++-
 .../server/topology/BlueprintValidatorImpl.java |  13 +++
 .../common-services/HDFS/2.1.0.2.0/metainfo.xml |  44 ++++++++
 .../topology/BlueprintValidatorImplTest.java    |  75 +++++++++++++-
 5 files changed, 268 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/f6124a05/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java
new file mode 100644
index 0000000..84e186f
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyConditionInfo.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.state;
+
+import java.util.Map;
+import java.util.Objects;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.adapters.XmlAdapter;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+
+import org.apache.commons.lang.NotImplementedException;
+
+/**
+ * Represents stack component dependency condition information.
+ */
+@XmlJavaTypeAdapter(DependencyConditionAdapter.class)
+public abstract class DependencyConditionInfo {
+    public abstract boolean isResolved(Map<String, Map<String, String>> properties);
+}
+
+class PropertyExistsDependencyCondition extends DependencyConditionInfo{
+
+    protected final String configType;
+    protected final String property;
+    public PropertyExistsDependencyCondition( String configType, String property) {
+        this.configType = Objects.requireNonNull(configType, "Config Type must not be null.");
+        this.property = Objects.requireNonNull(property, "Property Name must not be null.");
+    }
+
+    @Override
+    public boolean isResolved(Map<String, Map<String, String>> properties) {
+        return (properties.get(configType).containsKey(property));
+    }
+}
+
+class PropertyValueEqualsDependencyCondition extends PropertyExistsDependencyCondition {
+
+    protected final String propertyValue;
+    public PropertyValueEqualsDependencyCondition(String configType, String property, String
propertyValue) {
+        super(configType, property);
+        this.propertyValue = Objects.requireNonNull(propertyValue, "Property value must not
be null.");
+    }
+
+    @Override
+    public boolean isResolved(Map<String, Map<String, String>> properties) {
+            return (super.isResolved(properties) && propertyValue.equals(properties.get(configType).get(property)));
+    }
+}
+
+class DependencyConditionAdapter extends XmlAdapter<DependencyConditionAdapter.AdaptedDependencyCondition,
DependencyConditionInfo> {
+
+    static class AdaptedDependencyCondition{
+            @XmlElement
+            private String configType;
+            @XmlElement
+            private String property;
+            @XmlElement
+            private String propertyValue;
+            @XmlElement(name="condition-type")
+            private String conditionType;
+    }
+
+    @Override
+    public AdaptedDependencyCondition marshal(DependencyConditionInfo arg0) throws Exception
{
+        throw new NotImplementedException();
+    }
+
+    @Override
+    public DependencyConditionInfo unmarshal(AdaptedDependencyCondition adaptedDependencyCondition)
throws Exception {
+        if (null == adaptedDependencyCondition) {
+            return null;
+        }
+        DependencyConditionInfo dependencyConditionInfo = null;
+            switch (adaptedDependencyCondition.conditionType) {
+                case "IF-PROPERTY-EXISTS":
+                    dependencyConditionInfo = new PropertyExistsDependencyCondition(adaptedDependencyCondition.configType,
adaptedDependencyCondition.property);
+                    break;
+                case "PROPERTY-VALUE-EQUALS":
+                    dependencyConditionInfo = new PropertyValueEqualsDependencyCondition(adaptedDependencyCondition.configType,
adaptedDependencyCondition.property, adaptedDependencyCondition.propertyValue);
+                    break;
+                default:
+                    throw new IllegalArgumentException("Specified condition type is not not
supported " + adaptedDependencyCondition.conditionType);
+            }
+        return dependencyConditionInfo;
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/f6124a05/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
index e3db662..3c88401 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/DependencyInfo.java
@@ -20,6 +20,10 @@ package org.apache.ambari.server.state;
 
 
 import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlElementWrapper;
+import javax.xml.bind.annotation.XmlElements;
+import java.util.ArrayList;
+import java.util.List;
 
 /**
  * Represents stack component dependency information.
@@ -54,7 +58,10 @@ public class DependencyInfo {
   @XmlElement(name="auto-deploy")
   private AutoDeployInfo m_autoDeploy;
 
-
+  /**
+   * Conditions for Component dependency to other components.
+   */
+  private List<DependencyConditionInfo> dependencyConditions = new ArrayList<DependencyConditionInfo>();
   /**
    * Setter for name property.
    *
@@ -135,6 +142,33 @@ public class DependencyInfo {
   public String getServiceName() {
     return serviceName;
   }
+  /**
+   * Get the dependencyConditions list
+   *
+   * @return dependencyConditions
+   */
+  @XmlElementWrapper(name="conditions")
+  @XmlElements(@XmlElement(name="condition"))
+  public List<DependencyConditionInfo> getDependencyConditions() {
+    return dependencyConditions;
+  }
+
+  /**
+   * Set dependencyConditions
+   *
+   * @param dependencyConditions
+   */
+  public void setDependencyConditions(List<DependencyConditionInfo> dependencyConditions)
{
+    this.dependencyConditions = dependencyConditions;
+  }
+
+  /**
+   * Confirms if dependency have any condition or not
+   * @return true if dependencies are based on a condition
+   */
+  public boolean hasDependencyConditions(){
+    return !(dependencyConditions == null || dependencyConditions.isEmpty());
+  }
 
   @Override
   public String toString() {

http://git-wip-us.apache.org/repos/asf/ambari/blob/f6124a05/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
index a5f33ff..7bbe0db 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/BlueprintValidatorImpl.java
@@ -30,6 +30,7 @@ import java.util.regex.Pattern;
 
 import org.apache.ambari.server.controller.internal.Stack;
 import org.apache.ambari.server.state.AutoDeployInfo;
+import org.apache.ambari.server.state.DependencyConditionInfo;
 import org.apache.ambari.server.state.DependencyInfo;
 import org.apache.ambari.server.utils.SecretReference;
 import org.apache.ambari.server.utils.VersionUtils;
@@ -289,6 +290,18 @@ public class BlueprintValidatorImpl implements BlueprintValidator {
         AutoDeployInfo autoDeployInfo  = dependency.getAutoDeploy();
         boolean        resolved        = false;
 
+        //check if conditions are met, if any
+        if(dependency.hasDependencyConditions()) {
+          boolean conditionsSatisfied = true;
+          for (DependencyConditionInfo dependencyCondition : dependency.getDependencyConditions())
{
+            if (!dependencyCondition.isResolved(blueprint.getConfiguration().getFullProperties()))
{
+              conditionsSatisfied = false;
+            }
+          }
+          if(!conditionsSatisfied){
+            continue;
+          }
+        }
         if (dependencyScope.equals("cluster")) {
           Collection<String> missingDependencyInfo = verifyComponentCardinalityCount(
               componentName, new Cardinality("1+"), autoDeployInfo);

http://git-wip-us.apache.org/repos/asf/ambari/blob/f6124a05/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
index 65d166a..dfbbd55 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/metainfo.xml
@@ -32,6 +32,50 @@
           <cardinality>1-2</cardinality>
           <versionAdvertised>true</versionAdvertised>
           <reassignAllowed>true</reassignAllowed>
+          <dependencies>
+            <dependency>
+              <name>HDFS/ZKFC</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>false</enabled>
+              </auto-deploy>
+              <conditions>
+                <condition>
+                  <condition-type>IF-PROPERTY-EXISTS</condition-type>
+                  <configType>hdfs-site</configType>
+                  <property>dfs.nameservices</property>
+                </condition>
+              </conditions>
+            </dependency>
+            <dependency>
+              <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>false</enabled>
+              </auto-deploy>
+              <conditions>
+                <condition>
+                  <condition-type>IF-PROPERTY-EXISTS</condition-type>
+                  <configType>hdfs-site</configType>
+                  <property>dfs.nameservices</property>
+                </condition>
+              </conditions>
+            </dependency>
+            <dependency>
+              <name>HDFS/JOURNALNODE</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>false</enabled>
+              </auto-deploy>
+              <conditions>
+                <condition>
+                  <condition-type>IF-PROPERTY-EXISTS</condition-type>
+                  <configType>hdfs-site</configType>
+                  <property>dfs.nameservices</property>
+                </condition>
+              </conditions>
+            </dependency>
+          </dependencies>
           <commandScript>
             <script>scripts/namenode.py</script>
             <scriptType>PYTHON</scriptType>

http://git-wip-us.apache.org/repos/asf/ambari/blob/f6124a05/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
index b1de8ef..1501c53 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintValidatorImplTest.java
@@ -34,7 +34,9 @@ import java.util.Map;
 import org.apache.ambari.server.controller.internal.Stack;
 import org.apache.ambari.server.state.AutoDeployInfo;
 import org.apache.ambari.server.state.ComponentInfo;
+import org.apache.ambari.server.state.DependencyConditionInfo;
 import org.apache.ambari.server.state.DependencyInfo;
+import org.easymock.EasyMock;
 import org.easymock.EasyMockRule;
 import org.easymock.Mock;
 import org.easymock.MockType;
@@ -66,15 +68,22 @@ public class BlueprintValidatorImplTest {
 
   @Mock(type = MockType.NICE)
   private DependencyInfo dependency1;
+  @Mock(type = MockType.NICE)
+  private DependencyInfo dependency2;
 
   @Mock(type = MockType.NICE)
   private ComponentInfo dependencyComponentInfo;
+  @Mock(type = MockType.NICE)
+  private DependencyConditionInfo dependencyConditionInfo1;
+  @Mock(type = MockType.NICE)
+  private DependencyConditionInfo dependencyConditionInfo2;
 
   private final Collection<String> group1Components = new ArrayList<String>();
   private final Collection<String> group2Components = new ArrayList<String>();
   private final Collection<String> services = new ArrayList<String>();
 
   private Collection<DependencyInfo> dependencies1 = new ArrayList<DependencyInfo>();
+  private List<DependencyConditionInfo> dependenciesConditionInfos1 = new ArrayList<DependencyConditionInfo>();
   private AutoDeployInfo autoDeploy = new AutoDeployInfo();
   private Map<String, Map<String, String>> configProperties = new HashMap<String,
Map<String, String>>();
   private Configuration configuration = new Configuration(configProperties, Collections.<String,
Map<String, Map<String, String>>>emptyMap());
@@ -105,13 +114,15 @@ public class BlueprintValidatorImplTest {
     expect(stack.getCardinality("component1")).andReturn(new Cardinality("1"));
     expect(stack.getCardinality("component2")).andReturn(new Cardinality("1+"));
     expect(stack.getCardinality("component3")).andReturn(new Cardinality("1+"));
+    dependenciesConditionInfos1.add(dependencyConditionInfo1);
+    dependenciesConditionInfos1.add(dependencyConditionInfo2);
 
     expect(blueprint.getConfiguration()).andReturn(configuration).anyTimes();
   }
 
   @After
   public void tearDown() {
-    reset(blueprint, stack, group1, group2, dependency1);
+    reset(blueprint, stack, group1, group2, dependency1, dependency2, dependencyConditionInfo1,
dependencyConditionInfo2);
   }
 
   @Test
@@ -341,4 +352,66 @@ public class BlueprintValidatorImplTest {
     verify(group1);
 
   }
+  @Test(expected=InvalidTopologyException.class)
+  public void testWhenComponentIsConditionallyDependentAndOnlyOneOfTheConditionsIsSatisfied()
throws Exception {
+    // GIVEN
+    hostGroups.clear();
+    hostGroups.put("group1", group1);
+
+    group1Components.add("component-1");
+    dependencies1.add(dependency1);
+    dependencies1.add(dependency2);
+    services.addAll(Collections.singleton("service-1"));
+
+
+    expect(blueprint.getHostGroupsForComponent("component-1")).andReturn(Arrays.asList(group1)).anyTimes();
+    expect(blueprint.getName()).andReturn("blueprint-1").anyTimes();
+    Map<String, Map<String, String>> properties = new HashMap<String, Map<String,
String>>();
+    Map<String, String> typeProps = new HashMap<String, String>();
+    typeProps.put("yarn.resourcemanager.hostname", "testhost");
+    properties.put("yarn-site", typeProps);
+
+    Configuration clusterConfig = new Configuration(properties,
+       Collections.<String, Map<String, Map<String, String>>>emptyMap());
+
+    Cardinality cardinality = new Cardinality("1");
+
+    expect(stack.getComponents("service-1")).andReturn(Arrays.asList("component-1")).anyTimes();
+    expect(stack.getAutoDeployInfo("component-1")).andReturn(autoDeploy).anyTimes();
+    expect(stack.getDependenciesForComponent("component-1")).andReturn(dependencies1).anyTimes();
+    expect(stack.getCardinality("component-1")).andReturn(cardinality).anyTimes();
+
+    AutoDeployInfo dependencyAutoDeploy = null;
+
+    expect(dependency1.getScope()).andReturn("host").anyTimes();
+    expect(dependency1.getAutoDeploy()).andReturn(dependencyAutoDeploy).anyTimes();
+    expect(dependency1.getComponentName()).andReturn("component-d").anyTimes();
+    expect(dependency1.getServiceName()).andReturn("service-d").anyTimes();
+    expect(dependency1.getName()).andReturn("dependency-1").anyTimes();
+    expect(dependency1.hasDependencyConditions()).andReturn(true).anyTimes();
+    expect(dependency1.getDependencyConditions()).andReturn(dependenciesConditionInfos1).anyTimes();
+    expect(dependency2.getScope()).andReturn("host").anyTimes();
+    expect(dependency2.getAutoDeploy()).andReturn(dependencyAutoDeploy).anyTimes();
+    expect(dependency2.getComponentName()).andReturn("component-d").anyTimes();
+    expect(dependency2.getServiceName()).andReturn("service-d").anyTimes();
+    expect(dependency2.getName()).andReturn("dependency-2").anyTimes();
+    expect(dependency2.hasDependencyConditions()).andReturn(false).anyTimes();
+
+    expect(dependencyConditionInfo1.isResolved(EasyMock.anyObject(Map.class))).andReturn(true).anyTimes();
+    expect(dependencyConditionInfo2.isResolved(EasyMock.anyObject(Map.class))).andReturn(false).anyTimes();
+
+
+    expect(dependencyComponentInfo.isClient()).andReturn(false).anyTimes();
+    expect(stack.getComponentInfo("component-d")).andReturn(dependencyComponentInfo).anyTimes();
+
+    replay(blueprint, stack, group1, group2, dependency1, dependency2, dependencyComponentInfo,dependencyConditionInfo1,dependencyConditionInfo2);
+
+    // WHEN
+    BlueprintValidator validator = new BlueprintValidatorImpl(blueprint);
+    validator.validateTopology();
+
+    // THEN
+    verify(group1);
+
+  }
 }


Mime
View raw message