hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rkan...@apache.org
Subject hadoop git commit: YARN-4579. Allow DefaultContainerExecutor container log directory permissions to be configurable (rchiang via rkanter)
Date Fri, 26 Feb 2016 00:37:33 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk c2460dad6 -> d7fdec1e6


YARN-4579. Allow DefaultContainerExecutor container log directory permissions to be configurable
(rchiang via rkanter)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d7fdec1e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d7fdec1e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d7fdec1e

Branch: refs/heads/trunk
Commit: d7fdec1e6b465395d2faca6e404e329d20f6c3d8
Parents: c2460da
Author: Robert Kanter <rkanter@apache.org>
Authored: Thu Feb 25 16:35:58 2016 -0800
Committer: Robert Kanter <rkanter@apache.org>
Committed: Thu Feb 25 16:36:38 2016 -0800

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 ++
 .../hadoop/yarn/conf/YarnConfiguration.java     |  7 +++++
 .../src/main/resources/yarn-default.xml         | 10 ++++++
 .../nodemanager/DefaultContainerExecutor.java   | 32 +++++++++++++++++---
 .../TestDefaultContainerExecutor.java           | 21 +++++++++----
 5 files changed, 62 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7fdec1e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 14837a8..8708e0c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -852,6 +852,9 @@ Release 2.8.0 - UNRELEASED
     YARN-4682. AMRM client to log when AMRM token updated.
     (Prabhu Joseph via stevel)
 
+    YARN-4579. Allow DefaultContainerExecutor container log directory
+    permissions to be configurable (rchiang via rkanter)
+
   OPTIMIZATIONS
 
     YARN-3339. TestDockerContainerExecutor should pull a single image and not

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7fdec1e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 69e5ba5..cef6932 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -764,6 +764,13 @@ public class YarnConfiguration extends Configuration {
       NM_PREFIX + "logaggregation.threadpool-size-max";
   public static final int DEFAULT_NM_LOG_AGGREGATION_THREAD_POOL_SIZE = 100;
 
+  /** Default permissions for container logs. */
+  public static final String NM_DEFAULT_CONTAINER_EXECUTOR_PREFIX =
+      NM_PREFIX + "default-container-executor.";
+  public static final String NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS =
+      NM_DEFAULT_CONTAINER_EXECUTOR_PREFIX + "log-dirs.permissions";
+  public static final String NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS_DEFAULT =
"710";
+
   public static final String NM_RESOURCEMANAGER_MINIMUM_VERSION =
       NM_PREFIX + "resourcemanager.minimum.version";
   public static final String DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION = "NONE";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7fdec1e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 72c89c9..cc08802 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1073,6 +1073,16 @@
   </property>
 
   <property>
+    <description>
+      The permissions settings used for the creation of container
+      directories when using DefaultContainerExecutor.  This follows
+      standard user/group/all permissions format.
+    </description>
+    <name>yarn.nodemanager.default-container-executor.log-dirs.permissions</name>
+    <value>710</value>
+  </property>
+
+  <property>
     <description>Whether to enable log aggregation. Log aggregation collects
       each container's logs and moves these logs onto a file-system, for e.g.
       HDFS, after the application completes. Users can configure the

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7fdec1e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 9ccde5f..49398e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -74,6 +74,8 @@ public class DefaultContainerExecutor extends ContainerExecutor {
 
   protected final FileContext lfs;
 
+  private String logDirPermissions = null;
+
   public DefaultContainerExecutor() {
     try {
       this.lfs = FileContext.getLocalFSFileContext();
@@ -509,9 +511,6 @@ public class DefaultContainerExecutor extends ContainerExecutor {
   /** Permissions for user app dir.
    * $local.dir/usercache/$user/appcache/$appId */
   static final short APPDIR_PERM = (short)0710;
-  /** Permissions for user log dir.
-   * $logdir/$user/$appId */
-  static final short LOGDIR_PERM = (short)0710;
 
   private long getDiskFreeSpace(Path base) throws IOException {
     return lfs.getFsStatus(base).getRemaining();
@@ -702,7 +701,8 @@ public class DefaultContainerExecutor extends ContainerExecutor {
       throws IOException {
 
     boolean appLogDirStatus = false;
-    FsPermission appLogDirPerms = new FsPermission(LOGDIR_PERM);
+    FsPermission appLogDirPerms = new
+        FsPermission(getLogDirPermissions());
     for (String rootLogDir : logDirs) {
       // create $log.dir/$appid
       Path appLogDir = new Path(rootLogDir, appId);
@@ -727,7 +727,8 @@ public class DefaultContainerExecutor extends ContainerExecutor {
       List<String> logDirs, String user) throws IOException {
 
     boolean containerLogDirStatus = false;
-    FsPermission containerLogDirPerms = new FsPermission(LOGDIR_PERM);
+    FsPermission containerLogDirPerms = new
+        FsPermission(getLogDirPermissions());
     for (String rootLogDir : logDirs) {
       // create $log.dir/$appid/$containerid
       Path appLogDir = new Path(rootLogDir, appId);
@@ -750,6 +751,27 @@ public class DefaultContainerExecutor extends ContainerExecutor {
   }
 
   /**
+   * Return default container log directory permissions.
+   */
+  @VisibleForTesting
+  public String getLogDirPermissions() {
+    if (this.logDirPermissions==null) {
+      this.logDirPermissions = getConf().get(
+          YarnConfiguration.NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS,
+          YarnConfiguration.NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS_DEFAULT);
+    }
+    return this.logDirPermissions;
+  }
+
+  /**
+   * Clear the internal variable for repeatable testing.
+   */
+  @VisibleForTesting
+  public void clearLogDirPermissions() {
+    this.logDirPermissions = null;
+  }
+
+  /**
    * @return the list of paths of given local directories
    */
   private static List<Path> getPaths(List<String> dirs) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7fdec1e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
index 4404a7c..5696ae8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
@@ -168,8 +168,7 @@ public class TestDefaultContainerExecutor {
         DefaultContainerExecutor.FILECACHE_PERM);
     final FsPermission appDirPerm = new FsPermission(
         DefaultContainerExecutor.APPDIR_PERM);
-    final FsPermission logDirPerm = new FsPermission(
-        DefaultContainerExecutor.LOGDIR_PERM);
+
     List<String> localDirs = new ArrayList<String>();
     localDirs.add(new Path(BASE_TMP_PATH, "localDirA").toString());
     localDirs.add(new Path(BASE_TMP_PATH, "localDirB").toString());
@@ -181,6 +180,7 @@ public class TestDefaultContainerExecutor {
     conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
     FileContext lfs = FileContext.getLocalFSFileContext(conf);
     DefaultContainerExecutor executor = new DefaultContainerExecutor(lfs);
+    executor.setConf(conf);
     executor.init();
 
     try {
@@ -208,11 +208,20 @@ public class TestDefaultContainerExecutor {
         Assert.assertEquals(appDirPerm, stats.getPermission());
       }
 
-      executor.createAppLogDirs(appId, logDirs, user);
+      String[] permissionsArray = { "000", "111", "555", "710", "777" };
+
+      for (String perm : permissionsArray ) {
+        conf.set(YarnConfiguration.NM_DEFAULT_CONTAINER_EXECUTOR_LOG_DIRS_PERMISSIONS, perm);
+        executor.clearLogDirPermissions();
+        FsPermission logDirPerm = new FsPermission(
+            executor.getLogDirPermissions());
+        executor.createAppLogDirs(appId, logDirs, user);
 
-      for (String dir : logDirs) {
-        FileStatus stats = lfs.getFileStatus(new Path(dir, appId));
-        Assert.assertEquals(logDirPerm, stats.getPermission());
+        for (String dir : logDirs) {
+          FileStatus stats = lfs.getFileStatus(new Path(dir, appId));
+          Assert.assertEquals(logDirPerm, stats.getPermission());
+          lfs.delete(new Path(dir, appId), true);
+        }
       }
     } finally {
       deleteTmpFiles();


Mime
View raw message