hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject hadoop git commit: YARN-6837. Null LocalResource visibility or resource type can crash the nodemanager. Contributed by Jinjiang Ling
Date Thu, 20 Jul 2017 16:12:05 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 a39617df6 -> 6e45c543c


YARN-6837. Null LocalResource visibility or resource type can crash the nodemanager. Contributed
by Jinjiang Ling

(cherry picked from commit c8df3668ecc37c2d58cad35520a762eaec3c8539)

Conflicts:
	hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e45c543
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e45c543
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e45c543

Branch: refs/heads/branch-2.8
Commit: 6e45c543cbef17beaf27355d5602cb30faa15849
Parents: a39617d
Author: Jason Lowe <jlowe@yahoo-inc.com>
Authored: Thu Jul 20 11:03:04 2017 -0500
Committer: Jason Lowe <jlowe@yahoo-inc.com>
Committed: Thu Jul 20 11:11:41 2017 -0500

----------------------------------------------------------------------
 .../impl/pb/ContainerLaunchContextPBImpl.java   |  8 ++
 .../TestApplicationClientProtocolRecords.java   | 52 +++++++++++
 .../containermanager/ContainerManagerImpl.java  |  6 ++
 .../containermanager/TestContainerManager.java  | 90 ++++++++++++++++++++
 4 files changed, 156 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e45c543/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
index 3d1c08e..a2bc22b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
@@ -211,6 +211,14 @@ extends ContainerLaunchContext {
         throw new NullPointerException(
             "Null resource URL for local resource " + rsrcEntry.getKey() + " : "
                 + rsrcEntry.getValue());
+      } else if (rsrcEntry.getValue().getType() == null) {
+        throw new NullPointerException(
+            "Null resource type for local resource " + rsrcEntry.getKey() + " : "
+                + rsrcEntry.getValue());
+      } else if (rsrcEntry.getValue().getVisibility() == null) {
+          throw new NullPointerException(
+            "Null resource visibility for local resource " + rsrcEntry.getKey() + " : "
+                + rsrcEntry.getValue());
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e45c543/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java
index 3610263..9ef8e6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestApplicationClientProtocolRecords.java
@@ -22,10 +22,12 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceType;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.junit.Assert;
@@ -58,4 +60,54 @@ public class TestApplicationClientProtocolRecords {
       Assert.assertTrue(e.getMessage().contains("Null resource URL for local resource"));
     }
   }
+
+  /*
+   * This test validates the scenario in which the client sets a null value for
+   * local resource type.
+   */
+  @Test
+  public void testCLCPBImplNullResourceType() throws IOException {
+    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+    try {
+      LocalResource resource = recordFactory.newRecordInstance(LocalResource.class);
+      resource.setResource(URL.fromPath(new Path(".")));
+      resource.setSize(-1);
+      resource.setVisibility(LocalResourceVisibility.APPLICATION);
+      resource.setType(null);
+      resource.setTimestamp(System.currentTimeMillis());
+      Map<String, LocalResource> localResources =
+          new HashMap<String, LocalResource>();
+      localResources.put("null_type_resource", resource);
+      ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
+      containerLaunchContext.setLocalResources(localResources);
+      Assert.fail("Setting an invalid local resource should be an error!");
+    } catch (NullPointerException e) {
+      Assert.assertTrue(e.getMessage().contains("Null resource type for local resource"));
+    }
+  }
+
+  /*
+   * This test validates the scenario in which the client sets a null value for
+   * local resource type.
+   */
+  @Test
+  public void testCLCPBImplNullResourceVisibility() throws IOException {
+    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+    try {
+      LocalResource resource = recordFactory.newRecordInstance(LocalResource.class);
+      resource.setResource(URL.fromPath(new Path(".")));
+      resource.setSize(-1);
+      resource.setVisibility(null);
+      resource.setType(LocalResourceType.FILE);
+      resource.setTimestamp(System.currentTimeMillis());
+      Map<String, LocalResource> localResources =
+          new HashMap<String, LocalResource>();
+      localResources.put("null_visibility_resource", resource);
+      ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
+      containerLaunchContext.setLocalResources(localResources);
+      Assert.fail("Setting an invalid local resource should be an error!");
+    } catch (NullPointerException e) {
+      Assert.assertTrue(e.getMessage().contains("Null resource visibility for local resource"));
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e45c543/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 92575b2..f881586 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -905,6 +905,12 @@ public class ContainerManagerImpl extends CompositeService implements
       if (rsrc.getValue() == null || rsrc.getValue().getResource() == null) {
         throw new YarnException(
             "Null resource URL for local resource " + rsrc.getKey() + " : " + rsrc.getValue());
+      } else if (rsrc.getValue().getType() == null) {
+        throw new YarnException(
+            "Null resource type for local resource " + rsrc.getKey() + " : " + rsrc.getValue());
+      } else if (rsrc.getValue().getVisibility() == null) {
+        throw new YarnException(
+            "Null resource visibility for local resource " + rsrc.getKey() + " : " + rsrc.getValue());
       }
     }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e45c543/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
index d2e8f7e..bb32799 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
@@ -1379,4 +1379,94 @@ public class TestContainerManager extends BaseContainerManagerTest
{
     Assert.assertTrue(response.getFailedRequests().get(cId).getMessage()
         .contains("Null resource URL for local resource"));
   }
+
+  @Test
+  public void testStartContainerFailureWithNullTypeLocalResource()
+      throws Exception {
+    containerManager.start();
+    LocalResource rsrc_alpha =
+        recordFactory.newRecordInstance(LocalResource.class);
+    rsrc_alpha.setResource(URL.fromPath(new Path("./")));
+    rsrc_alpha.setSize(-1);
+    rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
+    rsrc_alpha.setType(null);
+    rsrc_alpha.setTimestamp(System.currentTimeMillis());
+    Map<String, LocalResource> localResources =
+        new HashMap<String, LocalResource>();
+    localResources.put("null_type_resource", rsrc_alpha);
+    ContainerLaunchContext containerLaunchContext =
+        recordFactory.newRecordInstance(ContainerLaunchContext.class);
+    ContainerLaunchContext spyContainerLaunchContext =
+        Mockito.spy(containerLaunchContext);
+    Mockito.when(spyContainerLaunchContext.getLocalResources())
+        .thenReturn(localResources);
+
+    ContainerId cId = createContainerId(0);
+    String user = "start_container_fail";
+    Token containerToken =
+        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
+            user, context.getContainerTokenSecretManager());
+    StartContainerRequest request = StartContainerRequest
+        .newInstance(spyContainerLaunchContext, containerToken);
+
+    // start containers
+    List<StartContainerRequest> startRequest =
+        new ArrayList<StartContainerRequest>();
+    startRequest.add(request);
+    StartContainersRequest requestList =
+        StartContainersRequest.newInstance(startRequest);
+
+    StartContainersResponse response =
+        containerManager.startContainers(requestList);
+    Assert.assertTrue(response.getFailedRequests().size() == 1);
+    Assert.assertTrue(response.getSuccessfullyStartedContainers().size() == 0);
+    Assert.assertTrue(response.getFailedRequests().containsKey(cId));
+    Assert.assertTrue(response.getFailedRequests().get(cId).getMessage()
+        .contains("Null resource type for local resource"));
+  }
+
+  @Test
+  public void testStartContainerFailureWithNullVisibilityLocalResource()
+      throws Exception {
+    containerManager.start();
+    LocalResource rsrc_alpha =
+        recordFactory.newRecordInstance(LocalResource.class);
+    rsrc_alpha.setResource(URL.fromPath(new Path("./")));
+    rsrc_alpha.setSize(-1);
+    rsrc_alpha.setVisibility(null);
+    rsrc_alpha.setType(LocalResourceType.FILE);
+    rsrc_alpha.setTimestamp(System.currentTimeMillis());
+    Map<String, LocalResource> localResources =
+        new HashMap<String, LocalResource>();
+    localResources.put("null_visibility_resource", rsrc_alpha);
+    ContainerLaunchContext containerLaunchContext =
+        recordFactory.newRecordInstance(ContainerLaunchContext.class);
+    ContainerLaunchContext spyContainerLaunchContext =
+        Mockito.spy(containerLaunchContext);
+    Mockito.when(spyContainerLaunchContext.getLocalResources())
+        .thenReturn(localResources);
+
+    ContainerId cId = createContainerId(0);
+    String user = "start_container_fail";
+    Token containerToken =
+        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
+            user, context.getContainerTokenSecretManager());
+    StartContainerRequest request = StartContainerRequest
+        .newInstance(spyContainerLaunchContext, containerToken);
+
+    // start containers
+    List<StartContainerRequest> startRequest =
+        new ArrayList<StartContainerRequest>();
+    startRequest.add(request);
+    StartContainersRequest requestList =
+        StartContainersRequest.newInstance(startRequest);
+
+    StartContainersResponse response =
+        containerManager.startContainers(requestList);
+    Assert.assertTrue(response.getFailedRequests().size() == 1);
+    Assert.assertTrue(response.getSuccessfullyStartedContainers().size() == 0);
+    Assert.assertTrue(response.getFailedRequests().containsKey(cId));
+    Assert.assertTrue(response.getFailedRequests().get(cId).getMessage()
+        .contains("Null resource visibility for local resource"));
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message