Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 5D67C200D1A for ; Mon, 9 Oct 2017 20:46:48 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 5C00B160BE1; Mon, 9 Oct 2017 18:46:48 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 30CF2160BE0 for ; Mon, 9 Oct 2017 20:46:47 +0200 (CEST) Received: (qmail 37469 invoked by uid 500); 9 Oct 2017 18:46:33 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 34874 invoked by uid 99); 9 Oct 2017 18:46:32 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 09 Oct 2017 18:46:32 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B827EF5D6B; Mon, 9 Oct 2017 18:46:31 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jhung@apache.org To: common-commits@hadoop.apache.org Date: Mon, 09 Oct 2017 18:47:05 -0000 Message-Id: <2e2cbf5b718f462f93f954375c8931a0@git.apache.org> In-Reply-To: <45bf81dda43c4ffda60e010c9ae33578@git.apache.org> References: <45bf81dda43c4ffda60e010c9ae33578@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [36/50] [abbrv] hadoop git commit: YARN-6322: Disable queue refresh when configuration mutation is enabled. Contributed by Jonathan Hung archived-at: Mon, 09 Oct 2017 18:46:48 -0000 YARN-6322: Disable queue refresh when configuration mutation is enabled. Contributed by Jonathan Hung Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c8b208a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c8b208a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c8b208a Branch: refs/heads/YARN-5734 Commit: 4c8b208adb8c7639628676387e275856cf0842c9 Parents: 74ba6ff Author: Xuan Authored: Tue Aug 1 08:48:04 2017 -0700 Committer: Jonathan Hung Committed: Mon Oct 9 11:11:23 2017 -0700 ---------------------------------------------------------------------- .../server/resourcemanager/AdminService.java | 12 +++++++++- .../scheduler/MutableConfScheduler.java | 12 ++++++++-- .../scheduler/MutableConfigurationProvider.java | 4 +++- .../scheduler/capacity/CapacityScheduler.java | 9 ++++++-- .../conf/MutableCSConfigurationProvider.java | 11 +++++----- .../resourcemanager/TestRMAdminService.java | 23 ++++++++++++++++++++ .../TestMutableCSConfigurationProvider.java | 14 +++++++----- 7 files changed, 67 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c8b208a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 3457ae3..fd9e849 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -29,6 +29,7 @@ import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.ha.HAServiceProtocol; @@ -92,6 +93,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSyst import org.apache.hadoop.yarn.server.resourcemanager.resource.DynamicResourceConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeResourceUpdateEvent; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider; import com.google.common.annotations.VisibleForTesting; @@ -384,6 +387,12 @@ public class AdminService extends CompositeService implements RefreshQueuesResponse response = recordFactory.newRecordInstance(RefreshQueuesResponse.class); try { + ResourceScheduler scheduler = rm.getRMContext().getScheduler(); + if (scheduler instanceof MutableConfScheduler + && ((MutableConfScheduler) scheduler).isConfigurationMutable()) { + throw new IOException("Scheduler configuration is mutable. " + + operation + " is not allowed in this scenario."); + } refreshQueues(); RMAuditLogger.logSuccess(user.getShortUserName(), operation, "AdminService"); @@ -393,7 +402,8 @@ public class AdminService extends CompositeService implements } } - private void refreshQueues() throws IOException, YarnException { + @Private + public void refreshQueues() throws IOException, YarnException { rm.getRMContext().getScheduler().reinitialize(getConfig(), this.rm.getRMContext()); // refresh the reservation system http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c8b208a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java index 007dc29..313bf6a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfScheduler.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; @@ -33,10 +34,11 @@ public interface MutableConfScheduler extends ResourceScheduler { * Update the scheduler's configuration. * @param user Caller of this update * @param confUpdate configuration update - * @throws IOException if update is invalid + * @throws IOException if scheduler could not be reinitialized + * @throws YarnException if reservation system could not be reinitialized */ void updateConfiguration(UserGroupInformation user, - SchedConfUpdateInfo confUpdate) throws IOException; + SchedConfUpdateInfo confUpdate) throws IOException, YarnException; /** * Get the scheduler configuration. @@ -50,4 +52,10 @@ public interface MutableConfScheduler extends ResourceScheduler { * @return the queue object */ Queue getQueue(String queueName); + + /** + * Return whether the scheduler configuration is mutable. + * @return whether scheduler configuration is mutable or not. + */ + boolean isConfigurationMutable(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c8b208a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java index 1f13467..9baf1ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo; import java.io.IOException; @@ -39,8 +40,9 @@ public interface MutableConfigurationProvider { * @param user User issuing the request * @param confUpdate Key-value pairs for configurations to be updated. * @throws IOException if scheduler could not be reinitialized + * @throws YarnException if reservation system could not be reinitialized */ void mutateConfiguration(UserGroupInformation user, SchedConfUpdateInfo - confUpdate) throws IOException; + confUpdate) throws IOException, YarnException; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c8b208a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index 6d2de7e..51ee6a7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2620,8 +2620,8 @@ public class CapacityScheduler extends @Override public void updateConfiguration(UserGroupInformation user, - SchedConfUpdateInfo confUpdate) throws IOException { - if (csConfProvider instanceof MutableConfigurationProvider) { + SchedConfUpdateInfo confUpdate) throws IOException, YarnException { + if (isConfigurationMutable()) { ((MutableConfigurationProvider) csConfProvider).mutateConfiguration( user, confUpdate); } else { @@ -2629,4 +2629,9 @@ public class CapacityScheduler extends "provider does not support updating configuration."); } } + + @Override + public boolean isConfigurationMutable() { + return csConfProvider instanceof MutableConfigurationProvider; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c8b208a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java index 9ccc146..d03b2e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java @@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ConfigurationMutationACLPolicyFactory; @@ -58,7 +59,6 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, private YarnConfigurationStore confStore; private ConfigurationMutationACLPolicy aclMutationPolicy; private RMContext rmContext; - private Configuration conf; public MutableCSConfigurationProvider(RMContext rmContext) { this.rmContext = rmContext; @@ -96,7 +96,6 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, this.aclMutationPolicy = ConfigurationMutationACLPolicyFactory .getPolicy(config); aclMutationPolicy.init(config, rmContext); - this.conf = config; } @Override @@ -109,7 +108,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, @Override public synchronized void mutateConfiguration(UserGroupInformation user, - SchedConfUpdateInfo confUpdate) throws IOException { + SchedConfUpdateInfo confUpdate) throws IOException, YarnException { if (!aclMutationPolicy.isMutationAllowed(user, confUpdate)) { throw new AccessControlException("User is not admin of all modified" + " queues."); @@ -126,8 +125,8 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, } } try { - rmContext.getScheduler().reinitialize(conf, rmContext); - } catch (IOException e) { + rmContext.getRMAdminService().refreshQueues(); + } catch (IOException | YarnException e) { schedConf = oldConf; confStore.confirmMutation(id, false); throw e; @@ -148,7 +147,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider, } } try { - rmContext.getScheduler().reinitialize(conf, rmContext); + rmContext.getScheduler().reinitialize(schedConf, rmContext); } catch (IOException e) { schedConf = oldConf; confStore.confirmMutation(mutation.getId(), false); http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c8b208a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java index 588f16d..620ca58 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java @@ -198,6 +198,29 @@ public class TestRMAdminService { } @Test + public void testAdminRefreshQueuesWithMutableSchedulerConfiguration() { + configuration.set(CapacitySchedulerConfiguration.CS_CONF_PROVIDER, + CapacitySchedulerConfiguration.STORE_CS_CONF_PROVIDER); + + try { + rm = new MockRM(configuration); + rm.init(configuration); + rm.start(); + } catch (Exception ex) { + fail("Should not get any exceptions"); + } + + try { + rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance()); + fail("Expected exception while calling refreshQueues when scheduler" + + " configuration is mutable."); + } catch (Exception ex) { + assertTrue(ex.getMessage().endsWith("Scheduler configuration is " + + "mutable. refreshQueues is not allowed in this scenario.")); + } + } + + @Test public void testAdminRefreshNodesWithoutConfiguration() throws IOException, YarnException { configuration.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS, http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c8b208a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java index 9104f16..635a184 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java @@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.resourcemanager.AdminService; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -34,7 +36,6 @@ import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -import static org.mockito.Matchers.any; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -50,6 +51,7 @@ public class TestMutableCSConfigurationProvider { private SchedConfUpdateInfo goodUpdate; private SchedConfUpdateInfo badUpdate; private CapacityScheduler cs; + private AdminService adminService; private static final UserGroupInformation TEST_USER = UserGroupInformation .createUserForTesting("testUser", new String[] {}); @@ -61,6 +63,8 @@ public class TestMutableCSConfigurationProvider { when(rmContext.getScheduler()).thenReturn(cs); when(cs.getConfiguration()).thenReturn( new CapacitySchedulerConfiguration()); + adminService = mock(AdminService.class); + when(rmContext.getRMAdminService()).thenReturn(adminService); confProvider = new MutableCSConfigurationProvider(rmContext); goodUpdate = new SchedConfUpdateInfo(); Map goodUpdateMap = new HashMap<>(); @@ -78,22 +82,20 @@ public class TestMutableCSConfigurationProvider { } @Test - public void testInMemoryBackedProvider() throws IOException { + public void testInMemoryBackedProvider() throws IOException, YarnException { Configuration conf = new Configuration(); confProvider.init(conf); assertNull(confProvider.loadConfiguration(conf) .get("yarn.scheduler.capacity.root.a.goodKey")); - doNothing().when(cs).reinitialize(any(Configuration.class), - any(RMContext.class)); + doNothing().when(adminService).refreshQueues(); confProvider.mutateConfiguration(TEST_USER, goodUpdate); assertEquals("goodVal", confProvider.loadConfiguration(conf) .get("yarn.scheduler.capacity.root.a.goodKey")); assertNull(confProvider.loadConfiguration(conf).get( "yarn.scheduler.capacity.root.a.badKey")); - doThrow(new IOException()).when(cs).reinitialize(any(Configuration.class), - any(RMContext.class)); + doThrow(new IOException()).when(adminService).refreshQueues(); try { confProvider.mutateConfiguration(TEST_USER, badUpdate); } catch (IOException e) { --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org