ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From adorosz...@apache.org
Subject [35/39] ambari git commit: AMBARI-22325 Cluster template object improvements (benyoka)
Date Fri, 24 Nov 2017 12:40:17 GMT
AMBARI-22325 Cluster template object improvements (benyoka)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3a016f83
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3a016f83
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3a016f83

Branch: refs/heads/branch-feature-AMBARI-14714-blueprintv2
Commit: 3a016f8380627c916726edb5137e4df1336b527f
Parents: c50ce3d
Author: Balazs Bence Sari <benyoka@apache.org>
Authored: Tue Nov 21 10:32:23 2017 +0100
Committer: Doroszlai, Attila <adoroszlai@hortonworks.com>
Committed: Fri Nov 24 13:30:46 2017 +0100

----------------------------------------------------------------------
 .../ambari/server/topology/ComponentV2.java     |  6 +-
 .../topology/ProvisionClusterTemplate.java      | 72 ++++++++++++++++----
 .../ProvisionClusterTemplateFactory.java        |  9 +--
 .../apache/ambari/server/topology/Service.java  |  6 +-
 .../ambari/server/topology/ServiceId.java       | 27 +++-----
 .../server/topology/BlueprintV2FactoryTest.java | 16 ++---
 .../topology/ProvisionClusterTemplateTest.java  | 49 ++++++++++++-
 .../blueprintv2/cluster_template_v2.json        | 15 +++-
 .../cluster_template_v2_invalid_hostgroup.json  | 70 +++++++++++++++++++
 9 files changed, 215 insertions(+), 55 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java
index 0d26ef6..42158f4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java
@@ -30,7 +30,7 @@ public class ComponentV2 implements Configurable {
 
   private String name;
 
-  private ServiceId serviceId = new ServiceId();
+  private ServiceId serviceId = new ServiceId(null, null);
 
   private ProvisionAction provisionAction = ProvisionAction.INSTALL_AND_START;
 
@@ -110,12 +110,12 @@ public class ComponentV2 implements Configurable {
 
   @JsonProperty("service_group")
   public void setServiceGroup(String serviceGroup) {
-    serviceId.setServiceGroup(serviceGroup);
+    this.serviceId = new ServiceId(this.serviceId.getName(), serviceGroup);
   }
 
   @JsonProperty("service_name")
   public void setServiceName(String serviceName) {
-    serviceId.setName(serviceName);
+    this.serviceId = new ServiceId(serviceName, this.serviceId.getServiceGroup());
   }
 
   public String getServiceName() {

http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java
index 5ef6517..90da776 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java
@@ -18,26 +18,39 @@
  */
 package org.apache.ambari.server.topology;
 
+import static java.util.stream.Collectors.toMap;
+
 import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
+import javax.annotation.Nullable;
 
 import org.apache.ambari.server.controller.internal.ProvisionAction;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
 
 public class ProvisionClusterTemplate {
 
-  private String blueprint;
+  private String blueprint = null;
+
   @JsonProperty("default_password")
-  private String defaultPassword;
+  private String defaultPassword = null;
+
   @JsonProperty("config_recommendation_strategy")
   private ConfigRecommendationStrategy configRecommendationStrategy;
+
   @JsonProperty("provision_action")
   private ProvisionAction provisionAction;
-  private Collection<ProvisionClusterTemplate.Service> services;
-  @JsonProperty("host_groups")
-  private Collection<ProvisionClusterTemplate.HostGroup> hostGroups;
+
+  private Map<ServiceId, ProvisionClusterTemplate.Service> servicesById = Collections.emptyMap();
+
+  private Map<String, ProvisionClusterTemplate.HostGroup> hostGroups = Collections.emptyMap();
+
   private Collection<Credential> credentials;
+
   @JsonProperty("security")
   private SecurityConfiguration securityConfiguration;
 
@@ -57,12 +70,21 @@ public class ProvisionClusterTemplate {
     this.defaultPassword = defaultPassword;
   }
 
+  public @Nullable Service getServiceById(ServiceId serviceId) {
+    return servicesById.get(serviceId);
+  }
+
+  @JsonProperty("services")
   public Collection<Service> getServices() {
-    return services;
+    return servicesById.values();
   }
 
+  @JsonProperty("services")
   public void setServices(Collection<Service> services) {
-    this.services = services;
+    this.servicesById = services.stream().collect(toMap(
+      s -> s.getId(),
+      s -> s
+    ));
   }
 
   public Collection<Credential> getCredentials() {
@@ -97,21 +119,34 @@ public class ProvisionClusterTemplate {
     this.provisionAction = provisionAction;
   }
 
+  @JsonProperty("host_groups")
   public Collection<HostGroup> getHostGroups() {
-    return hostGroups;
+    return hostGroups.values();
+  }
+
+  public HostGroup getHostGroupByName(String name) {
+    return hostGroups.get(name);
   }
 
+  @JsonProperty("host_groups")
   public void setHostGroups(Collection<HostGroup> hostGroups) {
-    this.hostGroups = hostGroups;
+    this.hostGroups = hostGroups.stream().collect(toMap(
+      hg -> hg.getName(),
+      hg -> hg
+    ));
+  }
+
+  public void validate() throws IllegalStateException {
+    getHostGroups().forEach(HostGroup::validate);
   }
 
   public static class HostGroup implements Configurable {
     private String name;
     @JsonIgnore
     private Configuration configuration;
-    private Collection<Host> hosts;
+    private Collection<Host> hosts = Collections.emptyList();
     @JsonProperty("host_count")
-    private Integer hostCount;
+    private int hostCount = 0;
     @JsonProperty("host_predicate")
     private String hostPredicate;
 
@@ -141,11 +176,11 @@ public class ProvisionClusterTemplate {
       this.hosts = hosts;
     }
 
-    public Integer getHostCount() {
+    public int getHostCount() {
       return hostCount;
     }
 
-    public void setHostCount(Integer hostCount) {
+    public void setHostCount(int hostCount) {
       this.hostCount = hostCount;
     }
 
@@ -156,6 +191,12 @@ public class ProvisionClusterTemplate {
     public void setHostPredicate(String hostPredicate) {
       this.hostPredicate = hostPredicate;
     }
+
+    void validate() throws IllegalStateException {
+      Preconditions.checkState((hostCount == 0 && null == hostPredicate) || getHosts().isEmpty(),
+        "Invalid custer topology template. Host group %s must have either declatere its hosts
or " +
+          "hostcount (and optionally host predicate)", name);
+    }
   }
 
   public static class Service implements Configurable {
@@ -175,6 +216,11 @@ public class ProvisionClusterTemplate {
       this.configuration = configuration;
     }
 
+    @JsonIgnore
+    public ServiceId getId() {
+      return new ServiceId(name, serviceGroup);
+    }
+
     public String getName() {
       return name;
     }

http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java
index c0c280f..cd99fa1 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java
@@ -51,16 +51,13 @@ public class ProvisionClusterTemplateFactory {
 
   private void createObjectMapper() {
     objectMapper = new ObjectMapper();
-//    SimpleModule module = new SimpleModule("CustomModel", Version.unknownVersion());
-//    SimpleAbstractTypeResolver resolver = new SimpleAbstractTypeResolver();
-//    resolver.addMapping(HostGroupV2.class, HostGroupV2Impl.class);
-//    module.setAbstractTypes(resolver);
-//    objectMapper.registerModule(module);
     objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
   }
 
   public ProvisionClusterTemplate convertFromJson(String clusterTemplateJson) throws IOException
{
-    return objectMapper.readValue(clusterTemplateJson, ProvisionClusterTemplate.class);
+    ProvisionClusterTemplate template = objectMapper.readValue(clusterTemplateJson, ProvisionClusterTemplate.class);
+    template.validate();
+    return template;
   }
 
 }

http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java
index 4a93ecd..a45386f 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java
@@ -36,7 +36,7 @@ public class Service implements Configurable {
 
   private String type;
 
-  private ServiceId id = new ServiceId();
+  private ServiceId id = new ServiceId(null, null);
 
   private String stackId;
 
@@ -115,12 +115,12 @@ public class Service implements Configurable {
   }
 
   public void setName(String name) {
-    this.id.setName(name);
+    this.id = new ServiceId(name, this.id.getServiceGroup());
   }
 
   public void setServiceGroup(ServiceGroup serviceGroup) {
     this.serviceGroup = serviceGroup;
-    this.id.setServiceGroup(serviceGroup.getName());
+    this.id = new ServiceId(this.id.getName(), serviceGroup.getName()) ;
   }
 
   @JsonProperty("stack_id")

http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java
b/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java
index 2d81a07..9de0b3d 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java
@@ -17,38 +17,33 @@
  */
 package org.apache.ambari.server.topology;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 public class ServiceId {
-  private String serviceGroup;
-  private String name;
+  private final String serviceGroup;
+  private final String name;
 
-  public ServiceId() { }
+  @JsonCreator
+  public ServiceId(@JsonProperty("service_name") String name, @JsonProperty("service_group")
String serviceGroup) {
+    this.name = name;
+    this.serviceGroup = serviceGroup;
+  }
 
   public static ServiceId of(String name, String serviceGroup) {
-    ServiceId id = new ServiceId();
-    id.name = name;
-    id.serviceGroup = serviceGroup;
-    return id;
+    return new ServiceId(name, serviceGroup);
   }
 
+  @JsonProperty("service_group")
   public String getServiceGroup() {
     return serviceGroup;
   }
 
-  @JsonProperty("service_group")
-  public void setServiceGroup(String serviceGroup) {
-    this.serviceGroup = serviceGroup;
-  }
-
+  @JsonProperty("service_name")
   public String getName() {
     return name;
   }
 
-  @JsonProperty("service_name")
-  public void setName(String name) {
-    this.name = name;
-  }
 
   @Override
   public boolean equals(Object o) {

http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java
index 78aa98c..ab9adea 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java
@@ -37,6 +37,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Charsets;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.io.Resources;
 
 public class BlueprintV2FactoryTest {
@@ -79,14 +80,11 @@ public class BlueprintV2FactoryTest {
     assertEquals(2, bp.getServiceGroups().size());
 
     Service zk1 = bp.getServiceGroup("CoreSG").getServiceByName("ZK1");
-    Map<String, Map<String, String>> expectedProperties = new HashMap<>();
-    Map<String, String> zooCfg = new HashMap<>();
-    zooCfg.put("dataDir", "/zookeeper1");
-    expectedProperties.put("zoo.cfg", zooCfg);
-    Map<String, String> zookeeperEnv = new HashMap<>();
-    zookeeperEnv.put("zk_user", "zkuser1");
-    zookeeperEnv.put("zk_server_heapsize", "256MB");
-    expectedProperties.put("zookeeper-env", zookeeperEnv);
+    Map<String, Map<String, String>> expectedProperties = ImmutableMap.of(
+      "zoo.cfg", ImmutableMap.of("dataDir", "/zookeeper1"),
+      "zookeeper-env", ImmutableMap.of(
+        "zk_user", "zkuser1",
+        "zk_server_heapsize", "256MB"));
     assertEquals(expectedProperties, zk1.getConfiguration().getProperties());
   }
 
@@ -120,7 +118,7 @@ public class BlueprintV2FactoryTest {
     verifyBlueprintStructure2(blueprintFactory.convertFromJson(BLUEPRINTV2_2_JSON));
   }
 
-  private void verifyBlueprintStructure2(BlueprintV2 bp) {
+  private void verifyBlueprintStructure2(BlueprintV2 bp) { // for "blueprintv2_2.json"
     assertEquals(new StackId("HDP", "3.0.0"),
       bp.getServiceGroups().iterator().next().getServices().iterator().next().getStack().getStackId());
     assertEquals(1, bp.getStackIds().size());

http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java
b/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java
index b02a00a..3bb6ea4 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java
@@ -18,23 +18,68 @@
  */
 package org.apache.ambari.server.topology;
 
+import static java.util.stream.Collectors.toSet;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
 import java.io.IOException;
+import java.util.Map;
 
 import org.junit.Test;
 
 import com.google.common.base.Charsets;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.io.Resources;
 
 public class ProvisionClusterTemplateTest {
 
-  public static final String CLUSTER_TEMPLATE = getResource("blueprintv2/cluster_template_v2.json");
+  public static final String CLUSTER_TEMPLATE =
+    getResource("blueprintv2/cluster_template_v2.json");
+  public static final String CLUSTER_TEMPLATE_INVALID =
+    getResource("blueprintv2/cluster_template_v2_invalid_hostgroup.json");
 
 
   @Test
   public void testProvisionClusterTemplate() throws Exception {
     ProvisionClusterTemplateFactory factory = new ProvisionClusterTemplateFactory();
     ProvisionClusterTemplate template = factory.convertFromJson(CLUSTER_TEMPLATE);
-    System.out.println(template);
+    verifyClusterTemplate(template);
+  }
+
+  @Test(expected = IllegalStateException.class)
+  public void testProvisionClusterTemplateInvalidTemplate() throws Exception {
+    ProvisionClusterTemplateFactory factory = new ProvisionClusterTemplateFactory();
+    ProvisionClusterTemplate template = factory.convertFromJson(CLUSTER_TEMPLATE_INVALID);
+  }
+
+
+  private void verifyClusterTemplate(ProvisionClusterTemplate template) {
+    ProvisionClusterTemplate.Service zk1 = template.getServiceById(ServiceId.of("ZK1", "CORE_SG"));
+    assertNotNull(zk1);
+    Map<String, Map<String, String>> expectedZkProperties = ImmutableMap.of(
+      "zoo.cfg", ImmutableMap.of("dataDir", "/zookeeper1"));
+    assertEquals(expectedZkProperties, zk1.getConfiguration().getProperties());
+
+    ProvisionClusterTemplate.Service hdfs = template.getServiceById(ServiceId.of("HDFS",
"CORE_SG"));
+    Map<String, Map<String, String>> expectedHdfsProperties = ImmutableMap.of(
+      "hdfs-site", ImmutableMap.of("property-name", "property-value"));
+    assertNotNull(hdfs);
+    assertEquals(expectedHdfsProperties, hdfs.getConfiguration().getProperties());
+
+    ProvisionClusterTemplate.HostGroup hostGroup1 = template.getHostGroupByName("host-group-1");
+    assertNotNull(hostGroup1);
+    assertEquals(2, hostGroup1.getHosts().size());
+    assertEquals(0, hostGroup1.getHostCount());
+    assertEquals(ImmutableSet.of("host.domain.com", "host2.domain.com"),
+      hostGroup1.getHosts().stream().map(host -> host.getFqdn()).collect(toSet()));
+    hostGroup1.getHosts().forEach(host -> assertEquals("/dc1/rack1", host.getRackInfo()));
+
+    ProvisionClusterTemplate.HostGroup hostGroup2 = template.getHostGroupByName("host-group-2");
+    assertNotNull(hostGroup2);
+    assertEquals(0, hostGroup2.getHosts().size());
+    assertEquals(2, hostGroup2.getHostCount());
+    assertEquals("Hosts/os_type=centos6&Hosts/cpu_count=2", hostGroup2.getHostPredicate());
   }
 
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json b/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json
index 1fb0b04..d80f2f5 100644
--- a/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json
+++ b/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json
@@ -9,7 +9,7 @@
         {
           "zoo.cfg": {
             "properties": {
-              "dataDir": "/zookeeper2"
+              "dataDir": "/zookeeper1"
             }
           }
         }
@@ -44,13 +44,22 @@
       ],
       "hosts": [
         {
-          "fqdn": "host.domain.com"
+          "fqdn": "host.domain.com",
+          "rack_info": "/dc1/rack1"
         },
         {
-          "fqdn": "host2.domain.com"
+          "fqdn": "host2.domain.com",
+          "rack_info": "/dc1/rack1"
         }
       ]
+    },
+    {
+      "name": "host-group-2",
+      "configurations": [],
+      "host_count": "2",
+      "host_predicate": "Hosts/os_type=centos6&Hosts/cpu_count=2"
     }
+
   ],
   "credentials": [
     {

http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/test/resources/blueprintv2/cluster_template_v2_invalid_hostgroup.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/blueprintv2/cluster_template_v2_invalid_hostgroup.json
b/ambari-server/src/test/resources/blueprintv2/cluster_template_v2_invalid_hostgroup.json
new file mode 100644
index 0000000..abe0aaf
--- /dev/null
+++ b/ambari-server/src/test/resources/blueprintv2/cluster_template_v2_invalid_hostgroup.json
@@ -0,0 +1,70 @@
+{
+  "blueprint": "blueprint-name",
+  "default_password": "super-secret-password",
+  "services": [
+    {
+      "service_group": "CORE_SG",
+      "name": "ZK1",
+      "configurations": [
+        {
+          "zoo.cfg": {
+            "properties": {
+              "dataDir": "/zookeeper1"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "service_group": "CORE_SG",
+      "name": "HDFS",
+      "configurations": [
+        {
+          "hdfs-site": {
+            "properties": {
+              "property-name": "property-value"
+            }
+          }
+        }
+      ]
+    }
+
+  ],
+  "host_groups": [
+    {
+      "name": "host-group-1",
+      "configurations": [
+        {
+          "yarn-site": {
+            "properties": {
+              "property-name": "property-value"
+            }
+          }
+        }
+      ],
+      "hosts": [
+        {
+          "fqdn": "host.domain.com",
+          "rack_info": "/dc1/rack1"
+        },
+        {
+          "fqdn": "host2.domain.com",
+          "rack_info": "/dc1/rack1"
+        }
+      ],
+      "host_count": "2",
+      "host_predicate": "Hosts/os_type=centos6&Hosts/cpu_count=2"
+    }
+  ],
+  "credentials": [
+    {
+      "alias": "kdc.admin.credential",
+      "principal": "principal",
+      "key": "key",
+      "type": "TEMPORARY"
+    }
+  ],
+  "security": {
+    "type": "NONE"
+  }
+}


Mime
View raw message