Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 536D5200C1A for ; Mon, 30 Jan 2017 07:04:08 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 52521160B58; Mon, 30 Jan 2017 06:04:08 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id D389F160B65 for ; Mon, 30 Jan 2017 07:04:06 +0100 (CET) Received: (qmail 35978 invoked by uid 500); 30 Jan 2017 06:03:55 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 33545 invoked by uid 99); 30 Jan 2017 06:03:53 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 30 Jan 2017 06:03:53 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id DC261DFEF5; Mon, 30 Jan 2017 06:03:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vvasudev@apache.org To: common-commits@hadoop.apache.org Date: Mon, 30 Jan 2017 06:04:33 -0000 Message-Id: <43c35cc0cdeb4bbf9a81c1b3c09731d2@git.apache.org> In-Reply-To: <9ef251a20c0f40bd9f677d9f1d48c278@git.apache.org> References: <9ef251a20c0f40bd9f677d9f1d48c278@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [41/52] [abbrv] hadoop git commit: YARN-5707. Add manager class for resource profiles. Contributed by Varun Vasudev. archived-at: Mon, 30 Jan 2017 06:04:08 -0000 YARN-5707. Add manager class for resource profiles. Contributed by Varun Vasudev. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bbfb7e7c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bbfb7e7c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bbfb7e7c Branch: refs/heads/YARN-3926 Commit: bbfb7e7c002a8d3d782b716229f7e58678164ae1 Parents: 7ccaf9d Author: Varun Vasudev Authored: Sat Oct 8 19:43:33 2016 +0530 Committer: Varun Vasudev Committed: Mon Jan 30 11:17:08 2017 +0530 ---------------------------------------------------------------------- .../hadoop/yarn/conf/YarnConfiguration.java | 23 +++ .../yarn/conf/TestYarnConfigurationFields.java | 4 + .../src/main/resources/yarn-default.xml | 16 ++ .../hadoop-yarn-server-resourcemanager/pom.xml | 5 + .../resource/ResourceProfilesManager.java | 46 +++++ .../resource/ResourceProfilesManagerImpl.java | 176 +++++++++++++++++++ .../resource/TestResourceProfiles.java | 142 +++++++++++++++ .../resources/profiles/illegal-profiles-1.json | 10 ++ .../resources/profiles/illegal-profiles-2.json | 10 ++ .../resources/profiles/illegal-profiles-3.json | 10 ++ .../resources/profiles/sample-profiles-1.json | 14 ++ .../resources/profiles/sample-profiles-2.json | 26 +++ 12 files changed, 482 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfb7e7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 660611a..c7ad5c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -828,6 +828,29 @@ public class YarnConfiguration extends Configuration { public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser."; /** + * Enable/disable resource profiles. + */ + @Public + @Unstable + public static final String RM_RESOURCE_PROFILES_ENABLED = + RM_PREFIX + "resource-profiles.enabled"; + @Public + @Unstable + public static final boolean DEFAULT_RM_RESOURCE_PROFILES_ENABLED = false; + + /** + * File containing resource profiles. + */ + @Public + @Unstable + public static final String RM_RESOURCE_PROFILES_SOURCE_FILE = + RM_PREFIX + "resource-profiles.source-file"; + @Public + @Unstable + public static final String DEFAULT_RM_RESOURCE_PROFILES_SOURCE_FILE = + "resource-profiles.json"; + + /** * Timeout in seconds for YARN node graceful decommission. * This is the maximal time to wait for running containers and applications * to complete before transition a DECOMMISSIONING node into DECOMMISSIONED. http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfb7e7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java index 3da4bab..4427390 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java @@ -84,6 +84,10 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase { // Used as Java command line properties, not XML configurationPrefixToSkipCompare.add("yarn.app.container"); + // Ignore default file name for resource profiles + configurationPropsToSkipCompare + .add(YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_SOURCE_FILE); + // Ignore NodeManager "work in progress" variables configurationPrefixToSkipCompare .add(YarnConfiguration.NM_NETWORK_RESOURCE_ENABLED); http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfb7e7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 49511a2..c48c900 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -951,6 +951,22 @@ 600000 + + + Flag to enable/disable resource profiles + + yarn.resourcemanager.resource-profiles.enabled + false + + + + + If resource profiles is enabled, source file for the profiles + + yarn.resourcemanager.resource-profiles.source-file + resource-profiles.json + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfb7e7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml index 6985d65..0c30d84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml @@ -338,6 +338,11 @@ src/test/resources/submit-reservation.json src/test/resources/delete-reservation.json src/test/resources/update-reservation.json + src/test/resources/profiles/sample-profiles-1.json + src/test/resources/profiles/sample-profiles-2.json + src/test/resources/profiles/illegal-profiles-1.json + src/test/resources/profiles/illegal-profiles-2.json + src/test/resources/profiles/illegal-profiles-3.json http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfb7e7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java new file mode 100644 index 0000000..af54f05 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.resource; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; + +import java.io.IOException; +import java.util.Map; + +/** + * Interface for the resource profiles manager. Provides an interface to get + * the list of available profiles and some helper functions. + */ +public interface ResourceProfilesManager { + + void init(Configuration config) throws IOException; + + Resource getProfile(String profile); + + Map getResourceProfiles(); + + void reloadProfiles() throws IOException; + + Resource getDefaultProfile(); + + Resource getMinimumProfile(); + + Resource getMaximumProfile(); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfb7e7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java new file mode 100644 index 0000000..4bef333 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.resource; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.codehaus.jackson.map.ObjectMapper; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class ResourceProfilesManagerImpl implements ResourceProfilesManager { + + private static final Log LOG = + LogFactory.getLog(ResourceProfilesManagerImpl.class); + + private final Map profiles = new ConcurrentHashMap<>(); + private Configuration conf; + + private static final String MEMORY = ResourceInformation.MEMORY_MB.getName(); + private static final String VCORES = ResourceInformation.VCORES.getName(); + + private static final String DEFAULT_PROFILE = "default"; + private static final String MINIMUM_PROFILE = "minimum"; + private static final String MAXIMUM_PROFILE = "maximum"; + + private static final String[] MANDATORY_PROFILES = + { DEFAULT_PROFILE, MINIMUM_PROFILE, MAXIMUM_PROFILE }; + + public void init(Configuration config) throws IOException { + conf = config; + loadProfiles(); + } + + private void loadProfiles() throws IOException { + boolean profilesEnabled = + conf.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, + YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED); + if (!profilesEnabled) { + return; + } + String sourceFile = + conf.get(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE, + YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_SOURCE_FILE); + String resourcesFile = sourceFile; + ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); + if (classLoader == null) { + classLoader = ResourceProfilesManagerImpl.class.getClassLoader(); + } + if (classLoader != null) { + URL tmp = classLoader.getResource(sourceFile); + if (tmp != null) { + resourcesFile = tmp.getPath(); + } + } + ObjectMapper mapper = new ObjectMapper(); + Map data = mapper.readValue(new File(resourcesFile), Map.class); + Iterator iterator = data.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = (Map.Entry) iterator.next(); + String key = entry.getKey().toString(); + if (entry.getValue() instanceof Map) { + Map value = (Map) entry.getValue(); + // ensure memory and vcores are specified + if (!value.containsKey(MEMORY) || !value.containsKey(VCORES)) { + throw new IOException( + "Illegal resource profile definition; profile '" + key + + "' must contain '" + MEMORY + "' and '" + VCORES + "'"); + } + Resource resource = parseResource(key, value); + profiles.put(key, resource); + LOG.info("Added profile '" + key + "' with resources " + resource); + } + } + // check to make sure mandatory profiles are present + for (String profile : MANDATORY_PROFILES) { + if (!profiles.containsKey(profile)) { + throw new IOException( + "Mandatory profile missing '" + profile + "' missing. " + + Arrays.toString(MANDATORY_PROFILES) + " must be present"); + } + } + LOG.info("Loaded profiles " + profiles.keySet()); + } + + private Resource parseResource(String key, Map value) throws IOException { + Resource resource = Resource.newInstance(0, 0); + Iterator iterator = value.entrySet().iterator(); + Map resourceTypes = + ResourceUtils.getResourceTypes(); + while (iterator.hasNext()) { + Map.Entry resourceEntry = (Map.Entry) iterator.next(); + String resourceName = resourceEntry.getKey().toString(); + ResourceInformation resourceValue = + fromString(resourceName, resourceEntry.getValue().toString()); + if (resourceName.equals(MEMORY)) { + resource.setMemorySize(resourceValue.getValue()); + continue; + } + if (resourceName.equals(VCORES)) { + resource.setVirtualCores(resourceValue.getValue().intValue()); + continue; + } + if (resourceTypes.containsKey(resourceName)) { + resource.setResourceInformation(resourceName, resourceValue); + } else { + throw new IOException("Unrecognized resource type '" + resourceName + + "'. Recognized resource types are '" + resourceTypes.keySet() + + "'"); + } + } + return resource; + } + + public Resource getProfile(String profile) { + return Resources.clone(profiles.get(profile)); + } + + public Map getResourceProfiles() { + return Collections.unmodifiableMap(profiles); + } + + @VisibleForTesting + public void reloadProfiles() throws IOException { + profiles.clear(); + loadProfiles(); + } + + public Resource getDefaultProfile() { + return getProfile(DEFAULT_PROFILE); + } + + public Resource getMinimumProfile() { + return getProfile(MINIMUM_PROFILE); + } + + public Resource getMaximumProfile() { + return getProfile(MAXIMUM_PROFILE); + } + + private ResourceInformation fromString(String name, String value) { + String units = ResourceUtils.getUnits(value); + Long resourceValue = + Long.valueOf(value.substring(0, value.length() - units.length())); + return ResourceInformation.newInstance(name, units, resourceValue); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfb7e7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java new file mode 100644 index 0000000..c542ed8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java @@ -0,0 +1,142 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.resource; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public class TestResourceProfiles { + + @Test + public void testProfilesEnabled() throws Exception { + ResourceProfilesManager manager = new ResourceProfilesManagerImpl(); + Configuration conf = new Configuration(); + // be default resource profiles should not be enabled + manager.init(conf); + Map profiles = manager.getResourceProfiles(); + Assert.assertTrue(profiles.isEmpty()); + conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true); + try { + manager.init(conf); + Assert.fail( + "Exception should be thrown due to missing resource profiles file"); + } catch (IOException ie) { + } + conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE, + "profiles/sample-profiles-1.json"); + manager.init(conf); + } + + @Test + public void testLoadProfiles() throws Exception { + ResourceProfilesManager manager = new ResourceProfilesManagerImpl(); + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true); + conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE, + "profiles/sample-profiles-1.json"); + manager.init(conf); + Map profiles = manager.getResourceProfiles(); + Map expected = new HashMap<>(); + expected.put("minimum", Resource.newInstance(1024, 1)); + expected.put("default", Resource.newInstance(2048, 2)); + expected.put("maximum", Resource.newInstance(4096, 4)); + + for (Map.Entry entry : expected.entrySet()) { + String profile = entry.getKey(); + Resource res = entry.getValue(); + Assert.assertTrue("Mandatory profile '" + profile + "' missing", + profiles.containsKey(profile)); + Assert.assertEquals("Profile " + profile + "' resources don't match", res, + manager.getProfile(profile)); + } + } + + @Test + public void testLoadProfilesMissingMandatoryProfile() throws Exception { + + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true); + + String[] badProfiles = { "profiles/illegal-profiles-1.json", + "profiles/illegal-profiles-2.json", + "profiles/illegal-profiles-3.json" }; + for (String file : badProfiles) { + ResourceProfilesManager manager = new ResourceProfilesManagerImpl(); + conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE, file); + try { + manager.init(conf); + Assert.fail("Bad profile '" + file + "' is not valid"); + } catch (IOException ie) { + } + } + } + + @Test + public void testGetProfile() throws Exception { + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true); + ResourceProfilesManager manager = new ResourceProfilesManagerImpl(); + conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE, + "profiles/sample-profiles-2.json"); + manager.init(conf); + Map expected = new HashMap<>(); + expected.put("minimum", Resource.newInstance(1024, 1)); + expected.put("default", Resource.newInstance(2048, 2)); + expected.put("maximum", Resource.newInstance(4096, 4)); + expected.put("small", Resource.newInstance(1024, 1)); + expected.put("medium", Resource.newInstance(2048, 1)); + expected.put("large", Resource.newInstance(4096, 4)); + + for (Map.Entry entry : expected.entrySet()) { + String profile = entry.getKey(); + Resource res = entry.getValue(); + Assert.assertEquals("Profile " + profile + "' resources don't match", res, + manager.getProfile(profile)); + } + } + + @Test + public void testGetMandatoryProfiles() throws Exception { + ResourceProfilesManager manager = new ResourceProfilesManagerImpl(); + Configuration conf = new Configuration(); + conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true); + conf.set(YarnConfiguration.RM_RESOURCE_PROFILES_SOURCE_FILE, + "profiles/sample-profiles-1.json"); + manager.init(conf); + Map expected = new HashMap<>(); + expected.put("minimum", Resource.newInstance(1024, 1)); + expected.put("default", Resource.newInstance(2048, 2)); + expected.put("maximum", Resource.newInstance(4096, 4)); + + Assert.assertEquals("Profile 'minimum' resources don't match", + expected.get("minimum"), manager.getMinimumProfile()); + Assert.assertEquals("Profile 'default' resources don't match", + expected.get("default"), manager.getDefaultProfile()); + Assert.assertEquals("Profile 'maximum' resources don't match", + expected.get("maximum"), manager.getMaximumProfile()); + + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfb7e7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/illegal-profiles-1.json ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/illegal-profiles-1.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/illegal-profiles-1.json new file mode 100644 index 0000000..b6aca96 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/illegal-profiles-1.json @@ -0,0 +1,10 @@ +{ + "minimum": { + "memoryMB" : 1024, + "vcores" : 1 + }, + "default" : { + "memoryMB" : 2048, + "vcores" : 2 + }, +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfb7e7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/illegal-profiles-2.json ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/illegal-profiles-2.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/illegal-profiles-2.json new file mode 100644 index 0000000..d62a311 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/illegal-profiles-2.json @@ -0,0 +1,10 @@ +{ + "minimum": { + "memoryMB" : 1024, + "vcores" : 1 + }, + "maximum" : { + "memoryMB": 4096, + "vcores" : 4 + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfb7e7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/illegal-profiles-3.json ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/illegal-profiles-3.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/illegal-profiles-3.json new file mode 100644 index 0000000..9ee74de --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/illegal-profiles-3.json @@ -0,0 +1,10 @@ +{ + "default" : { + "memoryMB" : 2048, + "vcores" : 2 + }, + "maximum" : { + "memoryMB": 4096, + "vcores" : 4 + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfb7e7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-1.json ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-1.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-1.json new file mode 100644 index 0000000..65b7360 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-1.json @@ -0,0 +1,14 @@ +{ + "minimum": { + "memory-mb" : 1024, + "vcores" : 1 + }, + "default" : { + "memory-mb" : 2048, + "vcores" : 2 + }, + "maximum" : { + "memory-mb": 4096, + "vcores" : 4 + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbfb7e7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-2.json ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-2.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-2.json new file mode 100644 index 0000000..c235671 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/profiles/sample-profiles-2.json @@ -0,0 +1,26 @@ +{ + "minimum": { + "memory-mb" : 1024, + "vcores" : 1 + }, + "default" : { + "memory-mb" : 2048, + "vcores" : 2 + }, + "maximum" : { + "memory-mb": 4096, + "vcores" : 4 + }, + "small" : { + "memory-mb": 1024, + "vcores": 1 + }, + "medium" : { + "memory-mb": 2048, + "vcores": 1 + }, + "large": { + "memory-mb" : 4096, + "vcores" : 4 + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org