hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From epa...@apache.org
Subject hadoop git commit: YARN-4727. Unable to override the $HADOOP_CONF_DIR env variable for container. Contributed by Jason Lowe.
Date Wed, 13 Sep 2017 20:11:01 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 ecf352d64 -> fb3d53e00


YARN-4727. Unable to override the $HADOOP_CONF_DIR env variable for container. Contributed
by Jason Lowe.

(cherry picked from commit 3860be7961580ac20dd505d665b580f0a04ac4f8)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb3d53e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb3d53e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb3d53e0

Branch: refs/heads/branch-2
Commit: fb3d53e001725ab6c0182ba3be08e1b6342f9822
Parents: ecf352d
Author: Eric Payne <epayne@apache.org>
Authored: Wed Sep 13 14:41:55 2017 -0500
Committer: Eric Payne <epayne@apache.org>
Committed: Wed Sep 13 15:07:27 2017 -0500

----------------------------------------------------------------------
 .../launcher/ContainerLaunch.java               |  5 +--
 .../launcher/TestContainerLaunch.java           | 32 ++++++++++++++------
 2 files changed, 24 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb3d53e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index d29afd8..88f5925 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -1124,10 +1124,7 @@ public class ContainerLaunch implements Callable<Integer> {
     
     environment.put(Environment.PWD.name(), pwd.toString());
     
-    putEnvIfNotNull(environment, 
-        Environment.HADOOP_CONF_DIR.name(), 
-        System.getenv(Environment.HADOOP_CONF_DIR.name())
-        );
+    putEnvIfAbsent(environment, Environment.HADOOP_CONF_DIR.name());
 
     if (!Shell.WINDOWS) {
       environment.put("JVM_PID", "$$");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb3d53e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index ea46c29..c75ee04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -47,6 +47,7 @@ import java.util.StringTokenizer;
 import java.util.jar.JarFile;
 import java.util.jar.Manifest;
 
+import com.google.common.base.Supplier;
 import com.google.common.collect.Lists;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.hadoop.conf.Configuration;
@@ -55,6 +56,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.Shell.ExitCodeException;
 import org.apache.hadoop.util.StringUtils;
@@ -691,11 +693,15 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
     userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
     userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
     userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
+    final String userConfDir = "user_set_HADOOP_CONF_DIR";
+    userSetEnv.put(Environment.HADOOP_CONF_DIR.name(), userConfDir);
     containerLaunchContext.setEnvironment(userSetEnv);
 
     File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
     PrintWriter fileWriter = new PrintWriter(scriptFile);
     File processStartFile =
+        new File(tmpDir, "env_vars.tmp").getAbsoluteFile();
+    final File processFinalFile =
         new File(tmpDir, "env_vars.txt").getAbsoluteFile();
     if (Shell.WINDOWS) {
       fileWriter.println("@echo " + Environment.CONTAINER_ID.$() + "> "
@@ -716,6 +722,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
     	  + processStartFile);
       fileWriter.println("@echo " + Environment.HOME.$() + ">> "
           + processStartFile);
+      fileWriter.println("@echo " + Environment.HADOOP_CONF_DIR.$() + ">> "
+          + processStartFile);
       for (String serviceName : containerManager.getAuxServiceMetaData()
           .keySet()) {
         fileWriter.println("@echo %" + AuxiliaryServiceHelper.NM_AUX_SERVICE
@@ -723,6 +731,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
             + processStartFile);
       }
       fileWriter.println("@echo " + cId + ">> " + processStartFile);
+      fileWriter.println("@move /Y " + processStartFile + " "
+          + processFinalFile);
       fileWriter.println("@ping -n 100 127.0.0.1 >nul");
     } else {
       fileWriter.write("\numask 0"); // So that start file is readable by the test
@@ -744,6 +754,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
           + processStartFile);
       fileWriter.write("\necho $" + Environment.HOME.name() + " >> "
           + processStartFile);
+      fileWriter.write("\necho $" + Environment.HADOOP_CONF_DIR.name() + " >> "
+          + processStartFile);
       for (String serviceName : containerManager.getAuxServiceMetaData()
           .keySet()) {
         fileWriter.write("\necho $" + AuxiliaryServiceHelper.NM_AUX_SERVICE
@@ -751,6 +763,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
             + processStartFile);
       }
       fileWriter.write("\necho $$ >> " + processStartFile);
+      fileWriter.write("\nmv " + processStartFile + " " + processFinalFile);
       fileWriter.write("\nexec sleep 100");
     }
     fileWriter.close();
@@ -784,13 +797,12 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
         StartContainersRequest.newInstance(list);
     containerManager.startContainers(allRequests);
 
-    int timeoutSecs = 0;
-    while (!processStartFile.exists() && timeoutSecs++ < 20) {
-      Thread.sleep(1000);
-      LOG.info("Waiting for process start-file to be created");
-    }
-    Assert.assertTrue("ProcessStartFile doesn't exist!",
-        processStartFile.exists());
+    GenericTestUtils.waitFor(new Supplier<Boolean>() {
+      @Override
+      public Boolean get() {
+        return processFinalFile.exists();
+      }
+    }, 10, 20000);
 
     // Now verify the contents of the file
     List<String> localDirs = dirsHandler.getLocalDirs();
@@ -810,7 +822,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
       containerLogDirs.add(logDir + Path.SEPARATOR + relativeContainerLogDir);
     }
     BufferedReader reader =
-        new BufferedReader(new FileReader(processStartFile));
+        new BufferedReader(new FileReader(processFinalFile));
     Assert.assertEquals(cId.toString(), reader.readLine());
     Assert.assertEquals(context.getNodeId().getHost(), reader.readLine());
     Assert.assertEquals(String.valueOf(context.getNodeId().getPort()),
@@ -833,7 +845,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
               YarnConfiguration.NM_USER_HOME_DIR, 
               YarnConfiguration.DEFAULT_NM_USER_HOME_DIR),
         reader.readLine());
-
+    Assert.assertEquals(userConfDir, reader.readLine());
     for (String serviceName : containerManager.getAuxServiceMetaData().keySet()) {
       Assert.assertEquals(
           containerManager.getAuxServiceMetaData().get(serviceName),
@@ -872,6 +884,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
     	        YarnConfiguration.DEFAULT_NM_USER_HOME_DIR),
     	containerLaunchContext.getEnvironment()
     		.get(Environment.HOME.name()));
+    Assert.assertEquals(userConfDir, containerLaunchContext.getEnvironment()
+        .get(Environment.HADOOP_CONF_DIR.name()));
 
     // Get the pid of the process
     String pid = reader.readLine().trim();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message