hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aengin...@apache.org
Subject [16/50] [abbrv] hadoop git commit: YARN-4354. Public resource localization fails with NPE. Contributed by Jason Lowe.
Date Sat, 21 Nov 2015 07:56:20 GMT
YARN-4354. Public resource localization fails with NPE. Contributed by Jason Lowe.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/855d5292
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/855d5292
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/855d5292

Branch: refs/heads/HDFS-7240
Commit: 855d52927b6115e2cfbd97a94d6c1a3ddf0e94bb
Parents: c753617
Author: Junping Du <junping_du@apache.org>
Authored: Sun Nov 15 04:43:57 2015 -0800
Committer: Junping Du <junping_du@apache.org>
Committed: Sun Nov 15 04:43:57 2015 -0800

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 ++
 .../localizer/LocalResourcesTrackerImpl.java    | 10 +++-
 .../TestLocalResourcesTrackerImpl.java          | 56 ++++++++++++++++++--
 .../TestResourceLocalizationService.java        |  4 +-
 4 files changed, 67 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/855d5292/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2f4311e..747c729 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1181,6 +1181,9 @@ Release 2.7.2 - UNRELEASED
     YARN-4320. TestJobHistoryEventHandler fails as AHS in MiniYarnCluster no longer
     binds to default port 8188. (Varun Saxena via ozawa)
 
+    YARN-4354. Public resource localization fails with NPE. (Jason Lowe via 
+    junping_du)
+
 Release 2.7.1 - 2015-07-06
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/855d5292/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
index fb8f767..38fffe6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
@@ -185,14 +185,22 @@ class LocalResourcesTrackerImpl implements LocalResourcesTracker {
       break;
     }
 
+    if (rsrc == null) {
+      LOG.warn("Received " + event.getType() + " event for request " + req
+          + " but localized resource is missing");
+      return;
+    }
     rsrc.handle(event);
 
     // Remove the resource if its downloading and its reference count has
     // become 0 after RELEASE. This maybe because a container was killed while
     // localizing and no other container is referring to the resource.
+    // NOTE: This should NOT be done for public resources since the
+    //       download is not associated with a container-specific localizer.
     if (event.getType() == ResourceEventType.RELEASE) {
       if (rsrc.getState() == ResourceState.DOWNLOADING &&
-          rsrc.getRefCount() <= 0) {
+          rsrc.getRefCount() <= 0 &&
+          rsrc.getRequest().getVisibility() != LocalResourceVisibility.PUBLIC) {
         removeResource(req);
       }
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/855d5292/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
index 4eb8675..21ceaa4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
@@ -141,12 +141,12 @@ public class TestLocalResourcesTrackerImpl {
       tracker.handle(rel21Event);
 
       dispatcher.await();
-      verifyTrackedResourceCount(tracker, 1);
+      verifyTrackedResourceCount(tracker, 2);
 
       // Verify resource with non zero ref count is not removed.
       Assert.assertEquals(2, lr1.getRefCount());
       Assert.assertFalse(tracker.remove(lr1, mockDelService));
-      verifyTrackedResourceCount(tracker, 1);
+      verifyTrackedResourceCount(tracker, 2);
 
       // Localize resource1
       ResourceLocalizedEvent rle =
@@ -161,7 +161,7 @@ public class TestLocalResourcesTrackerImpl {
 
       // Verify resources in state LOCALIZED with ref-count=0 is removed.
       Assert.assertTrue(tracker.remove(lr1, mockDelService));
-      verifyTrackedResourceCount(tracker, 0);
+      verifyTrackedResourceCount(tracker, 1);
     } finally {
       if (dispatcher != null) {
         dispatcher.stop();
@@ -899,6 +899,56 @@ public class TestLocalResourcesTrackerImpl {
     }
   }
 
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testReleaseWhileDownloading() throws Exception {
+    String user = "testuser";
+    DrainDispatcher dispatcher = null;
+    try {
+      Configuration conf = new Configuration();
+      dispatcher = createDispatcher(conf);
+      EventHandler<LocalizerEvent> localizerEventHandler =
+          mock(EventHandler.class);
+      EventHandler<LocalizerEvent> containerEventHandler =
+          mock(EventHandler.class);
+      dispatcher.register(LocalizerEventType.class, localizerEventHandler);
+      dispatcher.register(ContainerEventType.class, containerEventHandler);
+
+      ContainerId cId = BuilderUtils.newContainerId(1, 1, 1, 1);
+      LocalizerContext lc = new LocalizerContext(user, cId, null);
+
+      LocalResourceRequest req =
+          createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
+      LocalizedResource lr = createLocalizedResource(req, dispatcher);
+      ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
+          new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
+      localrsrc.put(req, lr);
+      LocalResourcesTracker tracker =
+          new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc,
+              false, conf, new NMNullStateStoreService(), null);
+
+      // request the resource
+      ResourceEvent reqEvent =
+          new ResourceRequestEvent(req, LocalResourceVisibility.PUBLIC, lc);
+      tracker.handle(reqEvent);
+
+      // release the resource
+      ResourceEvent relEvent = new ResourceReleaseEvent(req, cId);
+      tracker.handle(relEvent);
+
+      // download completing after release
+      ResourceLocalizedEvent rle =
+          new ResourceLocalizedEvent(req, new Path("file:///tmp/r1"), 1);
+      tracker.handle(rle);
+
+      dispatcher.await();
+    } finally {
+      if (dispatcher != null) {
+        dispatcher.stop();
+      }
+    }
+  }
+
   private boolean createdummylocalizefile(Path path) {
     boolean ret = false;
     File file = new File(path.toUri().getRawPath().toString());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/855d5292/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index f9e4188..c14ec7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -487,8 +487,8 @@ public class TestResourceLocalizationService {
         Assert.assertEquals("Incorrect reference count", 0, lr.getRefCount());
         pubRsrcs.remove(lr.getRequest());
       }
-      Assert.assertEquals(2, pubRsrcs.size());
-      Assert.assertEquals(0, pubRsrcCount);
+      Assert.assertEquals(0, pubRsrcs.size());
+      Assert.assertEquals(2, pubRsrcCount);
 
       appRsrcCount = 0;
       for (LocalizedResource lr : appTracker) {


Mime
View raw message