hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From z..@apache.org
Subject [49/50] [abbrv] hadoop git commit: YARN-3088. LinuxContainerExecutor.deleteAsUser can throw NPE if native executor returns an error. Contributed by Eric Payne
Date Mon, 26 Jan 2015 17:45:04 GMT
YARN-3088. LinuxContainerExecutor.deleteAsUser can throw NPE if native executor returns an
error. Contributed by Eric Payne


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b327379b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b327379b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b327379b

Branch: refs/heads/HDFS-EC
Commit: b327379b2aa5da9a48ff5258df1760d6523f04d7
Parents: 21f5c51
Author: Jason Lowe <jlowe@apache.org>
Authored: Mon Jan 26 15:40:21 2015 +0000
Committer: Zhe Zhang <zhz@apache.org>
Committed: Mon Jan 26 09:43:31 2015 -0800

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 ++
 .../nodemanager/LinuxContainerExecutor.java     |  9 +++-
 .../TestLinuxContainerExecutorWithMocks.java    | 49 +++++++++++++++++++-
 3 files changed, 58 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b327379b/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0808678..872f16e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -409,6 +409,9 @@ Release 2.7.0 - UNRELEASED
     YARN-3082. Non thread safe access to systemCredentials in NodeHeartbeatResponse
     processing. (Anubhav Dhoot via ozawa)
 
+    YARN-3088. LinuxContainerExecutor.deleteAsUser can throw NPE if native
+    executor returns an error (Eric Payne via jlowe)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b327379b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 4606f0c..d6e6894 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -392,18 +392,23 @@ public class LinuxContainerExecutor extends ContainerExecutor {
     verifyUsernamePattern(user);
     String runAsUser = getRunAsUser(user);
 
+    String dirString = dir == null ? "" : dir.toUri().getPath();
+
     List<String> command = new ArrayList<String>(
         Arrays.asList(containerExecutorExe,
                     runAsUser,
                     user,
                     Integer.toString(Commands.DELETE_AS_USER.getValue()),
-                    dir == null ? "" : dir.toUri().getPath()));
+                    dirString));
+    List<String> pathsToDelete = new ArrayList<String>();
     if (baseDirs == null || baseDirs.length == 0) {
       LOG.info("Deleting absolute path : " + dir);
+      pathsToDelete.add(dirString);
     } else {
       for (Path baseDir : baseDirs) {
         Path del = dir == null ? baseDir : new Path(baseDir, dir);
         LOG.info("Deleting path : " + del);
+        pathsToDelete.add(del.toString());
         command.add(baseDir.toUri().getPath());
       }
     }
@@ -419,7 +424,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
       }
     } catch (IOException e) {
       int exitCode = shExec.getExitCode();
-      LOG.error("DeleteAsUser for " + dir.toUri().getPath()
+      LOG.error("DeleteAsUser for " + StringUtils.join(" ", pathsToDelete)
           + " returned with exit code: " + exitCode, e);
       LOG.error("Output from LinuxContainerExecutor's deleteAsUser follows:");
       logOutput(shExec.getOutput());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b327379b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index d54367a..98ab8e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -319,10 +319,57 @@ public class TestLinuxContainerExecutorWithMocks {
     String cmd = String.valueOf(
         LinuxContainerExecutor.Commands.DELETE_AS_USER.getValue());
     Path dir = new Path("/tmp/testdir");
-    
+    Path testFile = new Path("testfile");
+    Path baseDir0 = new Path("/grid/0/BaseDir");
+    Path baseDir1 = new Path("/grid/1/BaseDir");
+
+    mockExec.deleteAsUser(appSubmitter, dir);
+    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+        appSubmitter, cmd, "/tmp/testdir"),
+        readMockParams());
+
+    mockExec.deleteAsUser(appSubmitter, null);
+    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+        appSubmitter, cmd, ""),
+        readMockParams());
+
+    mockExec.deleteAsUser(appSubmitter, testFile, baseDir0, baseDir1);
+    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+        appSubmitter, cmd, testFile.toString(), baseDir0.toString(), baseDir1.toString()),
+        readMockParams());
+
+    mockExec.deleteAsUser(appSubmitter, null, baseDir0, baseDir1);
+    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+        appSubmitter, cmd, "", baseDir0.toString(), baseDir1.toString()),
+        readMockParams());
+
+    File f = new File("./src/test/resources/mock-container-executer-with-error");
+    if (!FileUtil.canExecute(f)) {
+      FileUtil.setExecutable(f, true);
+    }
+    String executorPath = f.getAbsolutePath();
+    Configuration conf = new Configuration();
+    conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath);
+    mockExec.setConf(conf);
+
     mockExec.deleteAsUser(appSubmitter, dir);
     assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
         appSubmitter, cmd, "/tmp/testdir"),
         readMockParams());
+
+    mockExec.deleteAsUser(appSubmitter, null);
+    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+        appSubmitter, cmd, ""),
+        readMockParams());
+
+    mockExec.deleteAsUser(appSubmitter, testFile, baseDir0, baseDir1);
+    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+        appSubmitter, cmd, testFile.toString(), baseDir0.toString(), baseDir1.toString()),
+        readMockParams());
+
+    mockExec.deleteAsUser(appSubmitter, null, baseDir0, baseDir1);
+    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+        appSubmitter, cmd, "", baseDir0.toString(), baseDir1.toString()),
+        readMockParams());
   }
 }


Mime
View raw message