Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 53086200D49 for ; Fri, 24 Nov 2017 13:39:48 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 51D4D160BF2; Fri, 24 Nov 2017 12:39:48 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 754FB160BEE for ; Fri, 24 Nov 2017 13:39:46 +0100 (CET) Received: (qmail 50772 invoked by uid 500); 24 Nov 2017 12:39:45 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 50465 invoked by uid 99); 24 Nov 2017 12:39:45 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 24 Nov 2017 12:39:45 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 69F38DFD78; Fri, 24 Nov 2017 12:39:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: adoroszlai@apache.org To: commits@ambari.apache.org Date: Fri, 24 Nov 2017 12:40:17 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [35/39] ambari git commit: AMBARI-22325 Cluster template object improvements (benyoka) archived-at: Fri, 24 Nov 2017 12:39:48 -0000 AMBARI-22325 Cluster template object improvements (benyoka) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3a016f83 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3a016f83 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3a016f83 Branch: refs/heads/branch-feature-AMBARI-14714-blueprintv2 Commit: 3a016f8380627c916726edb5137e4df1336b527f Parents: c50ce3d Author: Balazs Bence Sari Authored: Tue Nov 21 10:32:23 2017 +0100 Committer: Doroszlai, Attila Committed: Fri Nov 24 13:30:46 2017 +0100 ---------------------------------------------------------------------- .../ambari/server/topology/ComponentV2.java | 6 +- .../topology/ProvisionClusterTemplate.java | 72 ++++++++++++++++---- .../ProvisionClusterTemplateFactory.java | 9 +-- .../apache/ambari/server/topology/Service.java | 6 +- .../ambari/server/topology/ServiceId.java | 27 +++----- .../server/topology/BlueprintV2FactoryTest.java | 16 ++--- .../topology/ProvisionClusterTemplateTest.java | 49 ++++++++++++- .../blueprintv2/cluster_template_v2.json | 15 +++- .../cluster_template_v2_invalid_hostgroup.json | 70 +++++++++++++++++++ 9 files changed, 215 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java index 0d26ef6..42158f4 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ComponentV2.java @@ -30,7 +30,7 @@ public class ComponentV2 implements Configurable { private String name; - private ServiceId serviceId = new ServiceId(); + private ServiceId serviceId = new ServiceId(null, null); private ProvisionAction provisionAction = ProvisionAction.INSTALL_AND_START; @@ -110,12 +110,12 @@ public class ComponentV2 implements Configurable { @JsonProperty("service_group") public void setServiceGroup(String serviceGroup) { - serviceId.setServiceGroup(serviceGroup); + this.serviceId = new ServiceId(this.serviceId.getName(), serviceGroup); } @JsonProperty("service_name") public void setServiceName(String serviceName) { - serviceId.setName(serviceName); + this.serviceId = new ServiceId(serviceName, this.serviceId.getServiceGroup()); } public String getServiceName() { http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java index 5ef6517..90da776 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplate.java @@ -18,26 +18,39 @@ */ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toMap; + import java.util.Collection; +import java.util.Collections; +import java.util.Map; + +import javax.annotation.Nullable; import org.apache.ambari.server.controller.internal.ProvisionAction; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.Preconditions; public class ProvisionClusterTemplate { - private String blueprint; + private String blueprint = null; + @JsonProperty("default_password") - private String defaultPassword; + private String defaultPassword = null; + @JsonProperty("config_recommendation_strategy") private ConfigRecommendationStrategy configRecommendationStrategy; + @JsonProperty("provision_action") private ProvisionAction provisionAction; - private Collection services; - @JsonProperty("host_groups") - private Collection hostGroups; + + private Map servicesById = Collections.emptyMap(); + + private Map hostGroups = Collections.emptyMap(); + private Collection credentials; + @JsonProperty("security") private SecurityConfiguration securityConfiguration; @@ -57,12 +70,21 @@ public class ProvisionClusterTemplate { this.defaultPassword = defaultPassword; } + public @Nullable Service getServiceById(ServiceId serviceId) { + return servicesById.get(serviceId); + } + + @JsonProperty("services") public Collection getServices() { - return services; + return servicesById.values(); } + @JsonProperty("services") public void setServices(Collection services) { - this.services = services; + this.servicesById = services.stream().collect(toMap( + s -> s.getId(), + s -> s + )); } public Collection getCredentials() { @@ -97,21 +119,34 @@ public class ProvisionClusterTemplate { this.provisionAction = provisionAction; } + @JsonProperty("host_groups") public Collection getHostGroups() { - return hostGroups; + return hostGroups.values(); + } + + public HostGroup getHostGroupByName(String name) { + return hostGroups.get(name); } + @JsonProperty("host_groups") public void setHostGroups(Collection hostGroups) { - this.hostGroups = hostGroups; + this.hostGroups = hostGroups.stream().collect(toMap( + hg -> hg.getName(), + hg -> hg + )); + } + + public void validate() throws IllegalStateException { + getHostGroups().forEach(HostGroup::validate); } public static class HostGroup implements Configurable { private String name; @JsonIgnore private Configuration configuration; - private Collection hosts; + private Collection hosts = Collections.emptyList(); @JsonProperty("host_count") - private Integer hostCount; + private int hostCount = 0; @JsonProperty("host_predicate") private String hostPredicate; @@ -141,11 +176,11 @@ public class ProvisionClusterTemplate { this.hosts = hosts; } - public Integer getHostCount() { + public int getHostCount() { return hostCount; } - public void setHostCount(Integer hostCount) { + public void setHostCount(int hostCount) { this.hostCount = hostCount; } @@ -156,6 +191,12 @@ public class ProvisionClusterTemplate { public void setHostPredicate(String hostPredicate) { this.hostPredicate = hostPredicate; } + + void validate() throws IllegalStateException { + Preconditions.checkState((hostCount == 0 && null == hostPredicate) || getHosts().isEmpty(), + "Invalid custer topology template. Host group %s must have either declatere its hosts or " + + "hostcount (and optionally host predicate)", name); + } } public static class Service implements Configurable { @@ -175,6 +216,11 @@ public class ProvisionClusterTemplate { this.configuration = configuration; } + @JsonIgnore + public ServiceId getId() { + return new ServiceId(name, serviceGroup); + } + public String getName() { return name; } http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java index c0c280f..cd99fa1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ProvisionClusterTemplateFactory.java @@ -51,16 +51,13 @@ public class ProvisionClusterTemplateFactory { private void createObjectMapper() { objectMapper = new ObjectMapper(); -// SimpleModule module = new SimpleModule("CustomModel", Version.unknownVersion()); -// SimpleAbstractTypeResolver resolver = new SimpleAbstractTypeResolver(); -// resolver.addMapping(HostGroupV2.class, HostGroupV2Impl.class); -// module.setAbstractTypes(resolver); -// objectMapper.registerModule(module); objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); } public ProvisionClusterTemplate convertFromJson(String clusterTemplateJson) throws IOException { - return objectMapper.readValue(clusterTemplateJson, ProvisionClusterTemplate.class); + ProvisionClusterTemplate template = objectMapper.readValue(clusterTemplateJson, ProvisionClusterTemplate.class); + template.validate(); + return template; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java index 4a93ecd..a45386f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/Service.java @@ -36,7 +36,7 @@ public class Service implements Configurable { private String type; - private ServiceId id = new ServiceId(); + private ServiceId id = new ServiceId(null, null); private String stackId; @@ -115,12 +115,12 @@ public class Service implements Configurable { } public void setName(String name) { - this.id.setName(name); + this.id = new ServiceId(name, this.id.getServiceGroup()); } public void setServiceGroup(ServiceGroup serviceGroup) { this.serviceGroup = serviceGroup; - this.id.setServiceGroup(serviceGroup.getName()); + this.id = new ServiceId(this.id.getName(), serviceGroup.getName()) ; } @JsonProperty("stack_id") http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java index 2d81a07..9de0b3d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ServiceId.java @@ -17,38 +17,33 @@ */ package org.apache.ambari.server.topology; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; public class ServiceId { - private String serviceGroup; - private String name; + private final String serviceGroup; + private final String name; - public ServiceId() { } + @JsonCreator + public ServiceId(@JsonProperty("service_name") String name, @JsonProperty("service_group") String serviceGroup) { + this.name = name; + this.serviceGroup = serviceGroup; + } public static ServiceId of(String name, String serviceGroup) { - ServiceId id = new ServiceId(); - id.name = name; - id.serviceGroup = serviceGroup; - return id; + return new ServiceId(name, serviceGroup); } + @JsonProperty("service_group") public String getServiceGroup() { return serviceGroup; } - @JsonProperty("service_group") - public void setServiceGroup(String serviceGroup) { - this.serviceGroup = serviceGroup; - } - + @JsonProperty("service_name") public String getName() { return name; } - @JsonProperty("service_name") - public void setName(String name) { - this.name = name; - } @Override public boolean equals(Object o) { http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java index 78aa98c..ab9adea 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/BlueprintV2FactoryTest.java @@ -37,6 +37,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Charsets; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.io.Resources; public class BlueprintV2FactoryTest { @@ -79,14 +80,11 @@ public class BlueprintV2FactoryTest { assertEquals(2, bp.getServiceGroups().size()); Service zk1 = bp.getServiceGroup("CoreSG").getServiceByName("ZK1"); - Map> expectedProperties = new HashMap<>(); - Map zooCfg = new HashMap<>(); - zooCfg.put("dataDir", "/zookeeper1"); - expectedProperties.put("zoo.cfg", zooCfg); - Map zookeeperEnv = new HashMap<>(); - zookeeperEnv.put("zk_user", "zkuser1"); - zookeeperEnv.put("zk_server_heapsize", "256MB"); - expectedProperties.put("zookeeper-env", zookeeperEnv); + Map> expectedProperties = ImmutableMap.of( + "zoo.cfg", ImmutableMap.of("dataDir", "/zookeeper1"), + "zookeeper-env", ImmutableMap.of( + "zk_user", "zkuser1", + "zk_server_heapsize", "256MB")); assertEquals(expectedProperties, zk1.getConfiguration().getProperties()); } @@ -120,7 +118,7 @@ public class BlueprintV2FactoryTest { verifyBlueprintStructure2(blueprintFactory.convertFromJson(BLUEPRINTV2_2_JSON)); } - private void verifyBlueprintStructure2(BlueprintV2 bp) { + private void verifyBlueprintStructure2(BlueprintV2 bp) { // for "blueprintv2_2.json" assertEquals(new StackId("HDP", "3.0.0"), bp.getServiceGroups().iterator().next().getServices().iterator().next().getStack().getStackId()); assertEquals(1, bp.getStackIds().size()); http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java index b02a00a..3bb6ea4 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/ProvisionClusterTemplateTest.java @@ -18,23 +18,68 @@ */ package org.apache.ambari.server.topology; +import static java.util.stream.Collectors.toSet; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + import java.io.IOException; +import java.util.Map; import org.junit.Test; import com.google.common.base.Charsets; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.io.Resources; public class ProvisionClusterTemplateTest { - public static final String CLUSTER_TEMPLATE = getResource("blueprintv2/cluster_template_v2.json"); + public static final String CLUSTER_TEMPLATE = + getResource("blueprintv2/cluster_template_v2.json"); + public static final String CLUSTER_TEMPLATE_INVALID = + getResource("blueprintv2/cluster_template_v2_invalid_hostgroup.json"); @Test public void testProvisionClusterTemplate() throws Exception { ProvisionClusterTemplateFactory factory = new ProvisionClusterTemplateFactory(); ProvisionClusterTemplate template = factory.convertFromJson(CLUSTER_TEMPLATE); - System.out.println(template); + verifyClusterTemplate(template); + } + + @Test(expected = IllegalStateException.class) + public void testProvisionClusterTemplateInvalidTemplate() throws Exception { + ProvisionClusterTemplateFactory factory = new ProvisionClusterTemplateFactory(); + ProvisionClusterTemplate template = factory.convertFromJson(CLUSTER_TEMPLATE_INVALID); + } + + + private void verifyClusterTemplate(ProvisionClusterTemplate template) { + ProvisionClusterTemplate.Service zk1 = template.getServiceById(ServiceId.of("ZK1", "CORE_SG")); + assertNotNull(zk1); + Map> expectedZkProperties = ImmutableMap.of( + "zoo.cfg", ImmutableMap.of("dataDir", "/zookeeper1")); + assertEquals(expectedZkProperties, zk1.getConfiguration().getProperties()); + + ProvisionClusterTemplate.Service hdfs = template.getServiceById(ServiceId.of("HDFS", "CORE_SG")); + Map> expectedHdfsProperties = ImmutableMap.of( + "hdfs-site", ImmutableMap.of("property-name", "property-value")); + assertNotNull(hdfs); + assertEquals(expectedHdfsProperties, hdfs.getConfiguration().getProperties()); + + ProvisionClusterTemplate.HostGroup hostGroup1 = template.getHostGroupByName("host-group-1"); + assertNotNull(hostGroup1); + assertEquals(2, hostGroup1.getHosts().size()); + assertEquals(0, hostGroup1.getHostCount()); + assertEquals(ImmutableSet.of("host.domain.com", "host2.domain.com"), + hostGroup1.getHosts().stream().map(host -> host.getFqdn()).collect(toSet())); + hostGroup1.getHosts().forEach(host -> assertEquals("/dc1/rack1", host.getRackInfo())); + + ProvisionClusterTemplate.HostGroup hostGroup2 = template.getHostGroupByName("host-group-2"); + assertNotNull(hostGroup2); + assertEquals(0, hostGroup2.getHosts().size()); + assertEquals(2, hostGroup2.getHostCount()); + assertEquals("Hosts/os_type=centos6&Hosts/cpu_count=2", hostGroup2.getHostPredicate()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json b/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json index 1fb0b04..d80f2f5 100644 --- a/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json +++ b/ambari-server/src/test/resources/blueprintv2/cluster_template_v2.json @@ -9,7 +9,7 @@ { "zoo.cfg": { "properties": { - "dataDir": "/zookeeper2" + "dataDir": "/zookeeper1" } } } @@ -44,13 +44,22 @@ ], "hosts": [ { - "fqdn": "host.domain.com" + "fqdn": "host.domain.com", + "rack_info": "/dc1/rack1" }, { - "fqdn": "host2.domain.com" + "fqdn": "host2.domain.com", + "rack_info": "/dc1/rack1" } ] + }, + { + "name": "host-group-2", + "configurations": [], + "host_count": "2", + "host_predicate": "Hosts/os_type=centos6&Hosts/cpu_count=2" } + ], "credentials": [ { http://git-wip-us.apache.org/repos/asf/ambari/blob/3a016f83/ambari-server/src/test/resources/blueprintv2/cluster_template_v2_invalid_hostgroup.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/blueprintv2/cluster_template_v2_invalid_hostgroup.json b/ambari-server/src/test/resources/blueprintv2/cluster_template_v2_invalid_hostgroup.json new file mode 100644 index 0000000..abe0aaf --- /dev/null +++ b/ambari-server/src/test/resources/blueprintv2/cluster_template_v2_invalid_hostgroup.json @@ -0,0 +1,70 @@ +{ + "blueprint": "blueprint-name", + "default_password": "super-secret-password", + "services": [ + { + "service_group": "CORE_SG", + "name": "ZK1", + "configurations": [ + { + "zoo.cfg": { + "properties": { + "dataDir": "/zookeeper1" + } + } + } + ] + }, + { + "service_group": "CORE_SG", + "name": "HDFS", + "configurations": [ + { + "hdfs-site": { + "properties": { + "property-name": "property-value" + } + } + } + ] + } + + ], + "host_groups": [ + { + "name": "host-group-1", + "configurations": [ + { + "yarn-site": { + "properties": { + "property-name": "property-value" + } + } + } + ], + "hosts": [ + { + "fqdn": "host.domain.com", + "rack_info": "/dc1/rack1" + }, + { + "fqdn": "host2.domain.com", + "rack_info": "/dc1/rack1" + } + ], + "host_count": "2", + "host_predicate": "Hosts/os_type=centos6&Hosts/cpu_count=2" + } + ], + "credentials": [ + { + "alias": "kdc.admin.credential", + "principal": "principal", + "key": "key", + "type": "TEMPORARY" + } + ], + "security": { + "type": "NONE" + } +}