Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 1B59A200C81 for ; Fri, 26 May 2017 16:46:18 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 19F95160BB8; Fri, 26 May 2017 14:46:18 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 12DF5160BAF for ; Fri, 26 May 2017 16:46:16 +0200 (CEST) Received: (qmail 7728 invoked by uid 500); 26 May 2017 14:46:16 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 7712 invoked by uid 99); 26 May 2017 14:46:14 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 26 May 2017 14:46:14 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 58AA8E0016; Fri, 26 May 2017 14:46:14 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jlowe@apache.org To: common-commits@hadoop.apache.org Message-Id: <2915a37ae7d14c4ca82c8ec125dbe1be@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-6641. Non-public resource localization on a bad disk causes subsequent containers failure. Contributed by Kuhu Shukla Date: Fri, 26 May 2017 14:46:14 +0000 (UTC) archived-at: Fri, 26 May 2017 14:46:18 -0000 Repository: hadoop Updated Branches: refs/heads/branch-2.8 bbef16b84 -> b89d59e21 YARN-6641. Non-public resource localization on a bad disk causes subsequent containers failure. Contributed by Kuhu Shukla Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b89d59e2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b89d59e2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b89d59e2 Branch: refs/heads/branch-2.8 Commit: b89d59e21eb7a7b465f666ccfadc9927fd12d967 Parents: bbef16b Author: Jason Lowe Authored: Fri May 26 09:45:49 2017 -0500 Committer: Jason Lowe Committed: Fri May 26 09:45:49 2017 -0500 ---------------------------------------------------------------------- .../localizer/LocalResourcesTrackerImpl.java | 13 ++-- .../localizer/ResourceLocalizationService.java | 10 +-- .../TestLocalResourcesTrackerImpl.java | 8 +-- .../TestResourceLocalizationService.java | 71 ++++++++++++++++++++ 4 files changed, 86 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b89d59e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java index 940c599..af34e92 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java @@ -94,14 +94,6 @@ class LocalResourcesTrackerImpl implements LocalResourcesTracker { public LocalResourcesTrackerImpl(String user, ApplicationId appId, Dispatcher dispatcher, boolean useLocalCacheDirectoryManager, - Configuration conf, NMStateStoreService stateStore) { - this(user, appId, dispatcher, - new ConcurrentHashMap(), - useLocalCacheDirectoryManager, conf, stateStore, null); - } - - public LocalResourcesTrackerImpl(String user, ApplicationId appId, - Dispatcher dispatcher, boolean useLocalCacheDirectoryManager, Configuration conf, NMStateStoreService stateStore, LocalDirsHandlerService dirHandler) { this(user, appId, dispatcher, @@ -528,4 +520,9 @@ class LocalResourcesTrackerImpl implements LocalResourcesTracker { } return mgr; } + + @VisibleForTesting + LocalDirsHandlerService getDirsHandler() { + return dirsHandler; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b89d59e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java index 61b6cdd..353831d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java @@ -293,7 +293,7 @@ public class ResourceLocalizationService extends CompositeService trackerState = userResources.getPrivateTrackerState(); if (!trackerState.isEmpty()) { LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, - null, dispatcher, true, super.getConfig(), stateStore); + null, dispatcher, true, super.getConfig(), stateStore, dirsHandler); LocalResourcesTracker oldTracker = privateRsrc.putIfAbsent(user, tracker); if (oldTracker != null) { @@ -309,7 +309,8 @@ public class ResourceLocalizationService extends CompositeService ApplicationId appId = appEntry.getKey(); String appIdStr = appId.toString(); LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, - appId, dispatcher, false, super.getConfig(), stateStore); + appId, dispatcher, false, super.getConfig(), stateStore, + dirsHandler); LocalResourcesTracker oldTracker = appRsrc.putIfAbsent(appIdStr, tracker); if (oldTracker != null) { @@ -447,10 +448,11 @@ public class ResourceLocalizationService extends CompositeService // 0) Create application tracking structs String userName = app.getUser(); privateRsrc.putIfAbsent(userName, new LocalResourcesTrackerImpl(userName, - null, dispatcher, true, super.getConfig(), stateStore)); + null, dispatcher, true, super.getConfig(), stateStore, dirsHandler)); String appIdStr = app.getAppId().toString(); appRsrc.putIfAbsent(appIdStr, new LocalResourcesTrackerImpl(app.getUser(), - app.getAppId(), dispatcher, false, super.getConfig(), stateStore)); + app.getAppId(), dispatcher, false, super.getConfig(), stateStore, + dirsHandler)); // 1) Signal container init // // This is handled by the ApplicationImpl state machine and allows http://git-wip-us.apache.org/repos/asf/hadoop/blob/b89d59e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java index 21ceaa4..5a97d8a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java @@ -529,7 +529,7 @@ public class TestLocalResourcesTrackerImpl { try { LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, - appId, dispatcher, false, conf, stateStore); + appId, dispatcher, false, conf, stateStore, null); // Container 1 needs lr1 resource ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, @@ -610,7 +610,7 @@ public class TestLocalResourcesTrackerImpl { try { LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, - appId, dispatcher, false, conf, stateStore); + appId, dispatcher, false, conf, stateStore, null); // Container 1 needs lr1 resource ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, @@ -672,7 +672,7 @@ public class TestLocalResourcesTrackerImpl { try { LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, - appId, dispatcher, false, conf, stateStore); + appId, dispatcher, false, conf, stateStore, null); // Container 1 needs lr1 resource ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, @@ -725,7 +725,7 @@ public class TestLocalResourcesTrackerImpl { try { LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, - appId, dispatcher, true, conf, stateStore); + appId, dispatcher, true, conf, stateStore, null); LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC); Assert.assertNull(tracker.getLocalizedResource(lr1)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b89d59e2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java index de1ce18..07fb7ba 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java @@ -2766,4 +2766,75 @@ public class TestResourceLocalizationService { } } + @Test + @SuppressWarnings("unchecked") + public void testDirHandler() throws Exception { + File f = new File(basedir.toString()); + String[] sDirs = new String[4]; + List localDirs = new ArrayList(sDirs.length); + for (int i = 0; i < 4; ++i) { + sDirs[i] = f.getAbsolutePath() + i; + localDirs.add(new Path(sDirs[i])); + } + conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs); + LocalizerTracker mockLocalizerTracker = mock(LocalizerTracker.class); + DrainDispatcher dispatcher = new DrainDispatcher(); + dispatcher.init(conf); + dispatcher.start(); + EventHandler applicationBus = mock(EventHandler.class); + dispatcher.register(ApplicationEventType.class, applicationBus); + EventHandler localizerBus = mock(EventHandler.class); + dispatcher.register(LocalizerEventType.class, localizerBus); + + ContainerExecutor exec = mock(ContainerExecutor.class); + LocalDirsHandlerService mockDirsHandler = + mock(LocalDirsHandlerService.class); + doReturn(new ArrayList(Arrays.asList(sDirs))).when( + mockDirsHandler).getLocalDirsForCleanup(); + // setup mocks + DeletionService delService = mock(DeletionService.class); + ResourceLocalizationService rawService = + new ResourceLocalizationService(dispatcher, exec, delService, + mockDirsHandler, nmContext); + ResourceLocalizationService spyService = spy(rawService); + doReturn(mockServer).when(spyService).createServer(); + doReturn(mockLocalizerTracker).when(spyService).createLocalizerTracker( + isA(Configuration.class)); + + final String user = "user0"; + // init application + final Application app = mock(Application.class); + final ApplicationId appId = + BuilderUtils.newApplicationId(314159265358979L, 3); + when(app.getUser()).thenReturn(user); + when(app.getAppId()).thenReturn(appId); + when(app.toString()).thenReturn(appId.toString()); + try { + spyService.init(conf); + spyService.start(); + + spyService.handle(new ApplicationLocalizationEvent( + LocalizationEventType.INIT_APPLICATION_RESOURCES, app)); + dispatcher.await(); + + LocalResourcesTracker appTracker = + spyService.getLocalResourcesTracker( + LocalResourceVisibility.APPLICATION, user, appId); + LocalResourcesTracker privTracker = + spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, + user, appId); + LocalResourcesTracker pubTracker = + spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, + user, appId); + Assert.assertNotNull("dirHandler for appTracker is null!", + ((LocalResourcesTrackerImpl)appTracker).getDirsHandler()); + Assert.assertNotNull("dirHandler for privTracker is null!", + ((LocalResourcesTrackerImpl)privTracker).getDirsHandler()); + Assert.assertNotNull("dirHandler for pubTracker is null!", + ((LocalResourcesTrackerImpl)pubTracker).getDirsHandler()); + } finally { + dispatcher.stop(); + delService.stop(); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org