hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yu...@apache.org
Subject hadoop git commit: YARN-7363. ContainerLocalizer don't have a valid log4j config in case of Linux container executor. (Contributed by Yufei Gu)
Date Mon, 27 Nov 2017 22:33:26 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 f335d509d -> c0ab3c840


YARN-7363. ContainerLocalizer don't have a valid log4j config in case of Linux container executor.
(Contributed by Yufei Gu)

(cherry picked from commit d8923cdbf1567aee10a54f144fef734d1465ebed)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0ab3c84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0ab3c84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0ab3c84

Branch: refs/heads/branch-3.0
Commit: c0ab3c8407b110dd9083d775e5ea823cbf73e7d4
Parents: f335d50
Author: Yufei Gu <yufei@apache.org>
Authored: Mon Nov 27 11:47:11 2017 -0800
Committer: Yufei Gu <yufei@apache.org>
Committed: Mon Nov 27 14:33:13 2017 -0800

----------------------------------------------------------------------
 .../hadoop/yarn/conf/YarnConfiguration.java     |  6 +++
 .../src/main/resources/yarn-default.xml         |  8 ++++
 .../nodemanager/LinuxContainerExecutor.java     | 28 +++++++++++-
 .../WindowsSecureContainerExecutor.java         |  2 +-
 .../localizer/ContainerLocalizer.java           | 46 +++++++++++++++++++-
 .../TestLinuxContainerExecutorWithMocks.java    | 19 +++++---
 6 files changed, 98 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ab3c84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 1a85c63..590caef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1588,6 +1588,12 @@ public class YarnConfiguration extends Configuration {
   public static final String NM_CONTAINER_LOCALIZER_JAVA_OPTS_DEFAULT =
       "-Xmx256m";
 
+  /** The log level of container localizer process. */
+  public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL=
+      NM_PREFIX + "container-localizer.log.level";
+  public static final String NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT =
+      "INFO";
+
   /** Prefix for runtime configuration constants. */
   public static final String LINUX_CONTAINER_RUNTIME_PREFIX = NM_PREFIX +
       "runtime.linux.";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ab3c84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index c304f7f..1291fad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1141,6 +1141,14 @@
 
   <property>
     <description>
+      The log level for container localizer while it is an independent process.
+    </description>
+    <name>yarn.nodemanager.container-localizer.log.level</name>
+    <value>INFO</value>
+  </property>
+
+  <property>
+    <description>
       Where to store container logs. An application's localized log directory
       will be found in ${yarn.nodemanager.log-dirs}/application_${appid}.
       Individual containers' log directories will be below this, in directories 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ab3c84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 9f1ec60..3950f59 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.ConfigurationException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
@@ -382,6 +383,10 @@ public class LinuxContainerExecutor extends ContainerExecutor {
     List<String> localizerArgs = new ArrayList<>();
 
     buildMainArgs(localizerArgs, user, appId, locId, nmAddr, localDirs);
+
+    Path containerLogDir = getContainerLogDir(dirsHandler, appId, locId);
+    localizerArgs = replaceWithContainerLogDir(localizerArgs, containerLogDir);
+
     initializeContainerOp.appendArgs(localizerArgs);
 
     try {
@@ -402,6 +407,27 @@ public class LinuxContainerExecutor extends ContainerExecutor {
     }
   }
 
+  private List<String> replaceWithContainerLogDir(List<String> commands,
+      Path containerLogDir) {
+    List<String> newCmds = new ArrayList<>(commands.size());
+
+    for (String item : commands) {
+      newCmds.add(item.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
+          containerLogDir.toString()));
+    }
+
+    return newCmds;
+  }
+
+  private Path getContainerLogDir(LocalDirsHandlerService dirsHandler,
+      String appId, String containerId) throws IOException {
+    String relativeContainerLogDir = ContainerLaunch
+        .getRelativeContainerLogDir(appId, containerId);
+
+    return dirsHandler.getLogPathForWrite(relativeContainerLogDir,
+        false);
+  }
+
   /**
    * Set up the {@link ContainerLocalizer}.
    *
@@ -417,7 +443,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
   public void buildMainArgs(List<String> command, String user, String appId,
       String locId, InetSocketAddress nmAddr, List<String> localDirs) {
     ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
-      localDirs);
+        localDirs, super.getConf());
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ab3c84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
index 7c124c3..e934472 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/WindowsSecureContainerExecutor.java
@@ -702,7 +702,7 @@ public class WindowsSecureContainerExecutor extends DefaultContainerExecutor
{
     command.addAll(ContainerLocalizer.getJavaOpts(getConf()));
 
     ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
-        localDirs);
+        localDirs, super.getConf());
 
     String cmdLine = StringUtils.join(command, " ");
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ab3c84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
index 2378c45..541911a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
@@ -63,6 +63,8 @@ import org.apache.hadoop.util.DiskValidatorFactory;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.concurrent.HadoopExecutors;
 import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.records.LocalResource;
 import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
 import org.apache.hadoop.yarn.api.records.SerializedException;
@@ -408,8 +410,12 @@ public class ContainerLocalizer {
    */
   public static void buildMainArgs(List<String> command,
       String user, String appId, String locId,
-      InetSocketAddress nmAddr, List<String> localDirs) {
-    
+      InetSocketAddress nmAddr, List<String> localDirs, Configuration conf) {
+
+    String logLevel = conf.get(YarnConfiguration.
+            NM_CONTAINER_LOCALIZER_LOG_LEVEL,
+        YarnConfiguration.NM_CONTAINER_LOCALIZER_LOG_LEVEL_DEFAULT);
+    addLog4jSystemProperties(logLevel, command);
     command.add(ContainerLocalizer.class.getName());
     command.add(user);
     command.add(appId);
@@ -421,6 +427,16 @@ public class ContainerLocalizer {
     }
   }
 
+  private static void addLog4jSystemProperties(
+      String logLevel, List<String> command) {
+    command.add("-Dlog4j.configuration=container-log4j.properties");
+    command.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "=" +
+        ApplicationConstants.LOG_DIR_EXPANSION_VAR);
+    command.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_SIZE + "=0");
+    command.add("-Dhadoop.root.logger=" + logLevel + ",CLA");
+    command.add("-Dhadoop.root.logfile=container-localizer-syslog");
+  }
+
   public static void main(String[] argv) throws Throwable {
     Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
     int nRet = 0;
@@ -431,6 +447,7 @@ public class ContainerLocalizer {
     // MKDIR $x/$user/appcache/$appid/filecache
     // LOAD $x/$user/appcache/$appid/appTokens
     try {
+      createLogDir();
       String user = argv[0];
       String appId = argv[1];
       String locId = argv[2];
@@ -466,6 +483,31 @@ public class ContainerLocalizer {
     }
   }
 
+  /**
+   * Create the log directory, if the directory exists, make sure its permission
+   * is 750.
+   */
+  private static void createLogDir() {
+    FileContext localFs;
+    try {
+      localFs = FileContext.getLocalFSFileContext(new Configuration());
+
+      String logDir = System.getProperty(
+          YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR);
+
+      if (logDir != null && !logDir.trim().isEmpty()) {
+        Path containerLogPath = new Path(logDir);
+        FsPermission containerLogDirPerm= new FsPermission((short)0750);
+        localFs.mkdir(containerLogPath, containerLogDirPerm, true);
+        // set permission again to make sure the permission is correct
+        // in case the directory is already there.
+        localFs.setPermission(containerLogPath, containerLogDirPerm);
+      }
+    } catch (IOException e) {
+      throw new YarnRuntimeException("Unable to create the log dir", e);
+    }
+  }
+
   private static void initDirs(Configuration conf, String user, String appId,
       FileContext lfs, List<Path> localDirs) throws IOException {
     if (null == localDirs || 0 == localDirs.size()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ab3c84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index 9fa22d8..938e75a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -272,7 +272,7 @@ public class TestLinuxContainerExecutorWithMocks {
           .build());
 
       List<String> result=readMockParams();
-      Assert.assertEquals(result.size(), 19);
+      Assert.assertEquals(result.size(), 24);
       Assert.assertEquals(result.get(0), YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER);
       Assert.assertEquals(result.get(1), "test");
       Assert.assertEquals(result.get(2), "0" );
@@ -280,12 +280,17 @@ public class TestLinuxContainerExecutorWithMocks {
       Assert.assertEquals(result.get(4), "/bin/nmPrivateCTokensPath");
       Assert.assertEquals(result.get(8), "-classpath" );
       Assert.assertEquals(result.get(11), "-Xmx256m" );
-      Assert.assertEquals(result.get(12),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer"
);
-      Assert.assertEquals(result.get(13), "test");
-      Assert.assertEquals(result.get(14), "application_0");
-      Assert.assertEquals(result.get(15),"12345" );
-      Assert.assertEquals(result.get(16),"localhost" );
-      Assert.assertEquals(result.get(17),"8040" );
+      Assert.assertEquals(result.get(12), "-Dlog4j.configuration=container-log4j.properties"
);
+      Assert.assertEquals(result.get(13), "-Dyarn.app.container.log.dir=${yarn.log.dir}/userlogs/application_0/12345");
+      Assert.assertEquals(result.get(14), "-Dyarn.app.container.log.filesize=0");
+      Assert.assertEquals(result.get(15), "-Dhadoop.root.logger=INFO,CLA");
+      Assert.assertEquals(result.get(16), "-Dhadoop.root.logfile=container-localizer-syslog");
+      Assert.assertEquals(result.get(17),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer"
);
+      Assert.assertEquals(result.get(18), "test");
+      Assert.assertEquals(result.get(19), "application_0");
+      Assert.assertEquals(result.get(20),"12345" );
+      Assert.assertEquals(result.get(21),"localhost" );
+      Assert.assertEquals(result.get(22),"8040" );
 
     } catch (InterruptedException e) {
       LOG.error("Error:"+e.getMessage(),e);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message