Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 380DD200CDD for ; Mon, 24 Jul 2017 01:33:40 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 36B90163DAF; Sun, 23 Jul 2017 23:33:40 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 7835E163D9F for ; Mon, 24 Jul 2017 01:33:38 +0200 (CEST) Received: (qmail 39658 invoked by uid 500); 23 Jul 2017 23:33:36 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 39370 invoked by uid 99); 23 Jul 2017 23:33:36 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 23 Jul 2017 23:33:36 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 2CB50F32BB; Sun, 23 Jul 2017 23:33:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jluniya@apache.org To: commits@ambari.apache.org Date: Sun, 23 Jul 2017 23:33:53 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [20/50] [abbrv] ambari git commit: AMBARI-21442. Ambari updates memory settings in blueprint incorrectly (amagyar) archived-at: Sun, 23 Jul 2017 23:33:40 -0000 AMBARI-21442. Ambari updates memory settings in blueprint incorrectly (amagyar) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/93fe8487 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/93fe8487 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/93fe8487 Branch: refs/heads/branch-feature-AMBARI-14714 Commit: 93fe8487a16fd2fe4f0c6a19bdc5d8a7c7b304a7 Parents: f92d121 Author: Attila Magyar Authored: Mon Jul 17 10:19:38 2017 +0200 Committer: Attila Magyar Committed: Mon Jul 17 10:19:38 2017 +0200 ---------------------------------------------------------------------- .../BlueprintConfigurationProcessor.java | 64 +++----- .../server/controller/internal/Stack.java | 2 +- .../server/controller/internal/UnitUpdater.java | 150 +++++++++++++++++++ .../validators/TopologyValidatorFactory.java | 2 +- .../validators/UnitValidatedProperty.java | 95 ++++++++++++ .../topology/validators/UnitValidator.java | 79 ++++++++++ .../controller/internal/UnitUpdaterTest.java | 114 ++++++++++++++ .../topology/validators/UnitValidatorTest.java | 114 ++++++++++++++ 8 files changed, 571 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/93fe8487/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java index 37284be..1daf76f 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/BlueprintConfigurationProcessor.java @@ -19,6 +19,8 @@ package org.apache.ambari.server.controller.internal; +import static java.util.stream.Collectors.groupingBy; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -48,6 +50,7 @@ import org.apache.ambari.server.topology.ConfigRecommendationStrategy; import org.apache.ambari.server.topology.Configuration; import org.apache.ambari.server.topology.HostGroup; import org.apache.ambari.server.topology.HostGroupInfo; +import org.apache.ambari.server.topology.validators.UnitValidatedProperty; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -2041,39 +2044,6 @@ public class BlueprintConfigurationProcessor { } /** - * Updater which appends "m" to the original property value. - * For example, "1024" would be updated to "1024m". - */ - private static class MPropertyUpdater implements PropertyUpdater { - /** - * Append 'm' to the original property value if it doesn't already exist. - * - * @param propertyName property name - * @param origValue original value of property - * @param properties all properties - * @param topology cluster topology - * - * @return property with 'm' appended - */ - @Override - public String updateForClusterCreate(String propertyName, - String origValue, - Map> properties, - ClusterTopology topology) { - - return origValue.endsWith("m") ? origValue : origValue + 'm'; - } - - @Override - public Collection getRequiredHostGroups(String propertyName, - String origValue, - Map> properties, - ClusterTopology topology) { - return Collections.emptySet(); - } - } - - /** * Class to facilitate special formatting needs of property values. */ private abstract static class AbstractPropertyValueDecorator implements PropertyUpdater { @@ -2784,20 +2754,7 @@ public class BlueprintConfigurationProcessor { new MultipleHostTopologyUpdater("ZOOKEEPER_SERVER")); // Required due to AMBARI-4933. These no longer seem to be required as the default values in the stack // are now correct but are left here in case an existing blueprint still contains an old value. - mHadoopEnvMap.put("namenode_heapsize", new MPropertyUpdater()); - mHadoopEnvMap.put("namenode_opt_newsize", new MPropertyUpdater()); - mHadoopEnvMap.put("namenode_opt_maxnewsize", new MPropertyUpdater()); - mHadoopEnvMap.put("namenode_opt_permsize", new MPropertyUpdater()); - mHadoopEnvMap.put("namenode_opt_maxpermsize", new MPropertyUpdater()); - mHadoopEnvMap.put("dtnode_heapsize", new MPropertyUpdater()); - mapredEnvMap.put("jtnode_opt_newsize", new MPropertyUpdater()); - mapredEnvMap.put("jtnode_opt_maxnewsize", new MPropertyUpdater()); - mapredEnvMap.put("jtnode_heapsize", new MPropertyUpdater()); - hbaseEnvMap.put("hbase_master_heapsize", new MPropertyUpdater()); - hbaseEnvMap.put("hbase_regionserver_heapsize", new MPropertyUpdater()); - oozieEnvHeapSizeMap.put("oozie_heapsize", new MPropertyUpdater()); - oozieEnvHeapSizeMap.put("oozie_permsize", new MPropertyUpdater()); - zookeeperEnvMap.put("zk_server_heapsize", new MPropertyUpdater()); + addUnitPropertyUpdaters(); hawqSiteMap.put("hawq_master_address_host", new SingleHostTopologyUpdater("HAWQMASTER")); hawqSiteMap.put("hawq_standby_address_host", new SingleHostTopologyUpdater("HAWQSTANDBY")); @@ -2816,6 +2773,19 @@ public class BlueprintConfigurationProcessor { }); } + private static void addUnitPropertyUpdaters() { + Map> propsPerConfigType = UnitValidatedProperty.ALL + .stream() + .collect(groupingBy(UnitValidatedProperty::getConfigType)); + for (String configType : propsPerConfigType.keySet()) { + Map unitUpdaters = new HashMap<>(); + for (UnitValidatedProperty each : propsPerConfigType.get(configType)) { + unitUpdaters.put(each.getPropertyName(), new UnitUpdater(each.getServiceName(), each.getConfigType())); + } + mPropertyUpdaters.put(configType, unitUpdaters); + } + } + private Collection setupHDFSProxyUsers(Configuration configuration, Set configTypesUpdated) { // AMBARI-5206 final Map userProps = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/93fe8487/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java index e1ea1cd..a28a3b5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/Stack.java @@ -757,7 +757,7 @@ public class Stack { private Set dependsOnProperties = Collections.emptySet(); - ConfigProperty(StackConfigurationResponse config) { + public ConfigProperty(StackConfigurationResponse config) { this.name = config.getPropertyName(); this.value = config.getPropertyValue(); this.attributes = config.getPropertyAttributes(); http://git-wip-us.apache.org/repos/asf/ambari/blob/93fe8487/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java new file mode 100644 index 0000000..8b7cb67 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UnitUpdater.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.controller.internal; + +import static org.apache.commons.lang.StringUtils.isBlank; + +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; + +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.validators.UnitValidatedProperty; + +/** + * I append the stack defined unit to the original property value. + * For example, "1024" would be updated to "1024m" if the stack unit is MB + * Properties with any other unit than the stack defined unit are rejected. + */ +public class UnitUpdater implements BlueprintConfigurationProcessor.PropertyUpdater { + private final String serviceName; + private final String configType; + + public UnitUpdater(String serviceName, String configType) { + this.serviceName = serviceName; + this.configType = configType; + } + + /** + * @return property value with updated unit + */ + @Override + public String updateForClusterCreate(String propertyName, + String origValue, + Map> properties, + ClusterTopology topology) { + PropertyUnit stackUnit = PropertyUnit.of(topology.getBlueprint().getStack(), serviceName, configType, propertyName); + PropertyValue value = PropertyValue.of(propertyName, origValue); + if (value.hasUnit(stackUnit)) { + return value.toString(); + } else if (!value.hasAnyUnit()) { + return value.withUnit(stackUnit); + } else { // should not happen because of prevalidation in UnitValidator + throw new IllegalArgumentException("Property " + propertyName + "=" + origValue + " has an unsupported unit. Stack supported unit is: " + stackUnit + " or no unit"); + } + } + + @Override + public Collection getRequiredHostGroups(String propertyName, String origValue, Map> properties, ClusterTopology topology) { + return Collections.emptySet(); + } + + public static class PropertyUnit { + private static final String DEFAULT_UNIT = "m"; + private final String unit; + + public static PropertyUnit of(Stack stack, UnitValidatedProperty property) { + return PropertyUnit.of(stack, property.getServiceName(), property.getConfigType(), property.getPropertyName()); + } + + public static PropertyUnit of(Stack stack, String serviceName, String configType, String propertyName) { + return new PropertyUnit( + stackUnit(stack, serviceName, configType, propertyName) + .map(PropertyUnit::toJvmUnit) + .orElse(DEFAULT_UNIT)); + } + + private static Optional stackUnit(Stack stack, String serviceName, String configType, String propertyName) { + try { + return Optional.ofNullable( + stack.getConfigurationPropertiesWithMetadata(serviceName, configType) + .get(propertyName) + .getPropertyValueAttributes() + .getUnit()); + } catch (NullPointerException e) { + return Optional.empty(); + } + } + + private static String toJvmUnit(String stackUnit) { + switch (stackUnit.toLowerCase()) { + case "mb" : return "m"; + case "gb" : return "g"; + case "b" : + case "bytes" : return ""; + default: throw new IllegalArgumentException("Unsupported stack unit: " + stackUnit); + } + } + + private PropertyUnit(String unit) { + this.unit = unit; + } + + @Override + public String toString() { + return unit; + } + } + + public static class PropertyValue { + private final String value; + + public static PropertyValue of(String name, String value) { + return new PropertyValue(normalized(name, value)); + } + + private static String normalized(String name, String value) { + if (isBlank(value)) { + throw new IllegalArgumentException("Missing property value " + name); + } + return value.trim().toLowerCase(); + } + + private PropertyValue(String value) { + this.value = value; + } + + public boolean hasUnit(PropertyUnit unit) { + return value.endsWith(unit.toString()); + } + + public boolean hasAnyUnit() { + return !Character.isDigit(value.charAt(value.length() -1)); + } + + public String withUnit(PropertyUnit unit) { + return value + unit; + } + + @Override + public String toString() { + return value; + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/93fe8487/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java index 5a6f64e..bc76bff 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/TopologyValidatorFactory.java @@ -25,7 +25,7 @@ public class TopologyValidatorFactory { public TopologyValidatorFactory() { validators = ImmutableList.of(new RequiredConfigPropertiesValidator(), new RequiredPasswordValidator(), new HiveServiceValidator(), - new StackConfigTypeValidator()); + new StackConfigTypeValidator(), new UnitValidator(UnitValidatedProperty.ALL)); } public TopologyValidator createConfigurationValidatorChain() { http://git-wip-us.apache.org/repos/asf/ambari/blob/93fe8487/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidatedProperty.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidatedProperty.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidatedProperty.java new file mode 100644 index 0000000..61f01db --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidatedProperty.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import java.util.Set; + +import org.apache.commons.lang.builder.EqualsBuilder; +import org.apache.commons.lang.builder.HashCodeBuilder; + +import com.google.common.collect.ImmutableSet; + +/** + * Some configuration values need to have "m" appended to them to be valid values. + * Required due to AMBARI-4933. + */ +public class UnitValidatedProperty { + public static final Set ALL = ImmutableSet.builder() + .add(new UnitValidatedProperty("HDFS", "hadoop-env", "namenode_heapsize")) + .add(new UnitValidatedProperty("HDFS", "hadoop-env", "namenode_opt_newsize")) + .add(new UnitValidatedProperty("HDFS", "hadoop-env", "namenode_opt_maxnewsize")) + .add(new UnitValidatedProperty("HDFS", "hadoop-env", "namenode_opt_permsize")) + .add(new UnitValidatedProperty("HDFS", "hadoop-env", "namenode_opt_maxpermsize")) + .add(new UnitValidatedProperty("HDFS", "hadoop-env", "dtnode_heapsize")) + .add(new UnitValidatedProperty("MAPREDUCE2", "mapred-env","jtnode_opt_newsize")) + .add(new UnitValidatedProperty("MAPREDUCE2", "mapred-env","jtnode_opt_maxnewsize")) + .add(new UnitValidatedProperty("MAPREDUCE2", "mapred-env","jtnode_heapsize")) + .add(new UnitValidatedProperty("HBASE", "hbase-env", "hbase_master_heapsize")) + .add(new UnitValidatedProperty("HBASE", "hbase-env","hbase_regionserver_heapsize")) + .add(new UnitValidatedProperty("OOZIE", "oozie-env","oozie_heapsize")) + .add(new UnitValidatedProperty("OOZIE", "oozie-env", "oozie_permsize")) + .add(new UnitValidatedProperty("ZOOKEEPER", "zookeeper-env", "zk_server_heapsize")) + .build(); + + private final String configType; + private final String serviceName; + private final String propertyName; + + public UnitValidatedProperty(String serviceName, String configType, String propertyName) { + this.configType = configType; + this.serviceName = serviceName; + this.propertyName = propertyName; + } + + public boolean hasTypeAndName(String configType, String propertyName) { + return configType.equals(this.getConfigType()) && propertyName.equals(this.getPropertyName()); + } + + public String getConfigType() { + return configType; + } + + public String getServiceName() { + return serviceName; + } + + public String getPropertyName() { + return propertyName; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + UnitValidatedProperty that = (UnitValidatedProperty) o; + return new EqualsBuilder() + .append(configType, that.configType) + .append(serviceName, that.serviceName) + .append(propertyName, that.propertyName) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .append(configType) + .append(serviceName) + .append(propertyName) + .toHashCode(); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/93fe8487/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java new file mode 100644 index 0000000..e75ffa4 --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/validators/UnitValidator.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import static org.apache.ambari.server.controller.internal.UnitUpdater.PropertyValue; + +import java.util.Map; +import java.util.Set; + +import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.controller.internal.UnitUpdater.PropertyUnit; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.HostGroupInfo; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.apache.ambari.server.topology.TopologyValidator; + +/** + * I validate the unit of properties by checking if it matches to the stack defined unit. + * Properties with different unit than the stack defined unit are rejected. + */ +public class UnitValidator implements TopologyValidator { + private final Set relevantProps; + + public UnitValidator(Set propertiesToBeValidated) { + this.relevantProps = propertiesToBeValidated; + } + + @Override + public void validate(ClusterTopology topology) throws InvalidTopologyException { + Stack stack = topology.getBlueprint().getStack(); + validateConfig(topology.getConfiguration().getFullProperties(), stack); + for (HostGroupInfo hostGroup : topology.getHostGroupInfo().values()) { + validateConfig(hostGroup.getConfiguration().getFullProperties(), stack); + } + } + + private void validateConfig(Map> configuration, Stack stack) { + for (Map.Entry> each : configuration.entrySet()) { + validateConfigType(each.getKey(), each.getValue(), stack); + } + } + + private void validateConfigType(String configType, Map config, Stack stack) { + for (String propertyName : config.keySet()) { + validateProperty(configType, config, propertyName, stack); + } + } + + private void validateProperty(String configType, Map config, String propertyName, Stack stack) { + relevantProps.stream() + .filter(each -> each.hasTypeAndName(configType, propertyName)) + .findFirst() + .ifPresent(relevantProperty -> checkUnit(config, stack, relevantProperty)); + } + + private void checkUnit(Map configToBeValidated, Stack stack, UnitValidatedProperty prop) { + PropertyUnit stackUnit = PropertyUnit.of(stack, prop); + PropertyValue value = PropertyValue.of(prop.getPropertyName(), configToBeValidated.get(prop.getPropertyName())); + if (value.hasAnyUnit() && !value.hasUnit(stackUnit)) { + throw new IllegalArgumentException("Property " + prop.getPropertyName() + "=" + value + " has an unsupported unit. Stack supported unit is: " + stackUnit + " or no unit"); + } + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/93fe8487/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UnitUpdaterTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UnitUpdaterTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UnitUpdaterTest.java new file mode 100644 index 0000000..6de6cd1 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UnitUpdaterTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.controller.internal; + +import static org.easymock.EasyMock.expect; +import static org.junit.Assert.assertEquals; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.ambari.server.controller.StackConfigurationResponse; +import org.apache.ambari.server.state.ValueAttributesInfo; +import org.apache.ambari.server.topology.Blueprint; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.easymock.EasyMockRule; +import org.easymock.EasyMockSupport; +import org.easymock.Mock; +import org.junit.Rule; +import org.junit.Test; + +public class UnitUpdaterTest extends EasyMockSupport { + public static final String HEAPSIZE = "oozie_heapsize"; + @Rule public EasyMockRule mocks = new EasyMockRule(this); + public static final String OOZIE = "OOZIE"; + public static final String OOZIE_ENV = "oozie-env"; + private Map stackConfigWithMetadata = new HashMap<>(); + private UnitUpdater unitUpdater; + private @Mock ClusterTopology clusterTopology; + private @Mock Blueprint blueprint; + private @Mock Stack stack; + + @Test + public void testStackUnitIsAppendedWhereUnitIsNotDefined() throws Exception { + stackUnitIs(HEAPSIZE, "GB"); + assertEquals("1g", updateUnit(OOZIE, OOZIE_ENV, HEAPSIZE, "1")); + } + + @Test + public void testDefaultMbStackUnitIsAppendedWhereUnitIsNotDefined() throws Exception { + assertEquals("4096m", updateUnit(OOZIE, OOZIE_ENV, HEAPSIZE, "4096")); + } + + @Test + public void testNoUnitIsAppendedWhenPropertyAlreadyHasTheStackUnit() throws Exception { + stackUnitIs(HEAPSIZE, "MB"); + assertEquals("128m", updateUnit(OOZIE, OOZIE_ENV, HEAPSIZE, "128m")); + } + + @Test + public void testNoUnitIsAppendedIfStackUnitIsInBytes() throws Exception { + stackUnitIs(HEAPSIZE, "Bytes"); + assertEquals("128", updateUnit(OOZIE, OOZIE_ENV, HEAPSIZE, "128")); + } + + @Test + public void testUnitSuffixIsCaseInsenitiveAndWhiteSpaceTolerant() throws Exception { + stackUnitIs(HEAPSIZE, "GB"); + assertEquals("1g", updateUnit(OOZIE, OOZIE_ENV, HEAPSIZE, " 1G ")); + } + + @Test(expected = IllegalArgumentException.class) + public void testRejectValuesWhereStackUnitDoesNotMatchToGiveUnit() throws Exception { + stackUnitIs(HEAPSIZE, "MB"); + updateUnit(OOZIE, OOZIE_ENV, HEAPSIZE, "2g"); + } + + @Test(expected = IllegalArgumentException.class) + public void testRejectEmptyPropertyValue() throws Exception { + updateUnit(OOZIE, OOZIE_ENV, HEAPSIZE, ""); + } + + private void stackUnitIs(String name, String unit) { + ValueAttributesInfo propertyValueAttributes = new ValueAttributesInfo(); + propertyValueAttributes.setUnit(unit); + stackConfigWithMetadata.put(name, new Stack.ConfigProperty(new StackConfigurationResponse( + name, + "any", + "any", + "any", + "any", + true, + Collections.emptySet(), + Collections.emptyMap(), + propertyValueAttributes, + Collections.emptySet() + ))); + } + + private String updateUnit(String serviceName, String configType, String propName, String propValue) throws InvalidTopologyException, ConfigurationTopologyException { + UnitUpdater updater = new UnitUpdater(serviceName, configType); + expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(stack.getConfigurationPropertiesWithMetadata(serviceName, configType)).andReturn(stackConfigWithMetadata).anyTimes(); + replayAll(); + return updater.updateForClusterCreate(propName, propValue, Collections.emptyMap(), clusterTopology); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/93fe8487/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/UnitValidatorTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/UnitValidatorTest.java b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/UnitValidatorTest.java new file mode 100644 index 0000000..334ee4b --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/topology/validators/UnitValidatorTest.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.topology.validators; + +import static com.google.common.collect.Sets.newHashSet; +import static java.util.Collections.emptyMap; +import static org.easymock.EasyMock.expect; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.ambari.server.controller.StackConfigurationResponse; +import org.apache.ambari.server.controller.internal.ConfigurationTopologyException; +import org.apache.ambari.server.controller.internal.Stack; +import org.apache.ambari.server.state.ValueAttributesInfo; +import org.apache.ambari.server.topology.Blueprint; +import org.apache.ambari.server.topology.ClusterTopology; +import org.apache.ambari.server.topology.Configuration; +import org.apache.ambari.server.topology.InvalidTopologyException; +import org.easymock.EasyMockRule; +import org.easymock.EasyMockSupport; +import org.easymock.Mock; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + + +public class UnitValidatorTest extends EasyMockSupport { + private static final String CONFIG_TYPE = "config-type"; + private static final String SERVICE = "service"; + @Rule public EasyMockRule mocks = new EasyMockRule(this); + private Map stackConfigWithMetadata = new HashMap<>(); + private UnitValidator validator; + private @Mock ClusterTopology clusterTopology; + private @Mock Blueprint blueprint; + private @Mock Stack stack; + + @Test(expected = IllegalArgumentException.class) + public void rejectsPropertyWithDifferentUnitThanStackUnit() throws Exception { + stackUnitIs("property1", "MB"); + propertyToBeValidatedIs("property1", "12G"); + validate("property1"); + } + + @Test + public void acceptsPropertyWithSameUnitThanStackUnit() throws Exception { + stackUnitIs("property1", "MB"); + propertyToBeValidatedIs("property1", "12m"); + validate("property1"); + } + + @Test + public void skipsValidatingIrrelevantProperty() throws Exception { + stackUnitIs("property1", "MB"); + propertyToBeValidatedIs("property1", "12g"); + validate("property2"); + } + + @Before + public void setUp() throws Exception { + expect(clusterTopology.getBlueprint()).andReturn(blueprint).anyTimes(); + expect(clusterTopology.getHostGroupInfo()).andReturn(Collections.emptyMap()).anyTimes(); + expect(blueprint.getStack()).andReturn(stack).anyTimes(); + expect(stack.getConfigurationPropertiesWithMetadata(SERVICE, CONFIG_TYPE)).andReturn(stackConfigWithMetadata).anyTimes(); + } + + private void propertyToBeValidatedIs(String propertyName, String propertyValue) throws InvalidTopologyException, ConfigurationTopologyException { + Map> propertiesToBeValidated = new HashMap>() {{ + put(CONFIG_TYPE, new HashMap(){{ + put(propertyName, propertyValue); + }}); + }}; + expect(clusterTopology.getConfiguration()).andReturn(new Configuration(propertiesToBeValidated, emptyMap())).anyTimes(); + replayAll(); + } + + private void validate(String propertyName) throws InvalidTopologyException { + validator = new UnitValidator(newHashSet(new UnitValidatedProperty(SERVICE, CONFIG_TYPE, propertyName))); + validator.validate(clusterTopology); + } + + private void stackUnitIs(String name, String unit) { + ValueAttributesInfo propertyValueAttributes = new ValueAttributesInfo(); + propertyValueAttributes.setUnit(unit); + stackConfigWithMetadata.put(name, new Stack.ConfigProperty(new StackConfigurationResponse( + name, + "any", + "any", + "any", + "any", + true, + Collections.emptySet(), + Collections.emptyMap(), + propertyValueAttributes, + Collections.emptySet() + ))); + } +} \ No newline at end of file