hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject hadoop git commit: MAPREDUCE-6288. Changed permissions on JobHistory server's done directory so that user's client can load the conf files directly. Contributed by Robert Kanter.
Date Mon, 30 Mar 2015 17:29:20 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 35af6f180 -> fa30500d7


MAPREDUCE-6288. Changed permissions on JobHistory server's done directory so that user's client
can load the conf files directly. Contributed by Robert Kanter.

(cherry picked from commit 5358b83167777a7108b32c9900fb0d01ca0fe961)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fa30500d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fa30500d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fa30500d

Branch: refs/heads/branch-2.7
Commit: fa30500d7277fcb1ac194bc8a95c9f9fa8a3d164
Parents: 35af6f1
Author: Vinod Kumar Vavilapalli <vinodkv@apache.org>
Authored: Mon Mar 30 10:27:19 2015 -0700
Committer: Vinod Kumar Vavilapalli <vinodkv@apache.org>
Committed: Mon Mar 30 10:29:05 2015 -0700

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt            |  4 ++
 .../v2/jobhistory/JobHistoryUtils.java          |  4 +-
 .../mapreduce/v2/hs/HistoryFileManager.java     | 31 ++++++++-
 .../mapreduce/v2/hs/TestHistoryFileManager.java | 73 ++++++++++++++++++++
 4 files changed, 108 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa30500d/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index da29b4e..a9b739a 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -196,6 +196,10 @@ Release 2.7.0 - UNRELEASED
     MAPREDUCE-6285. ClientServiceDelegate should not retry upon
     AuthenticationException. (Jonathan Eagles via ozawa)
 
+    MAPREDUCE-6288. Changed permissions on JobHistory server's done directory
+    so that user's client can load the conf files directly. (Robert Kanter via
+    vinodkv)
+
 Release 2.6.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa30500d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
index e279c03..8966e4e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JobHistoryUtils.java
@@ -72,7 +72,7 @@ public class JobHistoryUtils {
    * Permissions for the history done dir and derivatives.
    */
   public static final FsPermission HISTORY_DONE_DIR_PERMISSION =
-    FsPermission.createImmutable((short) 0770); 
+    FsPermission.createImmutable((short) 0771);
 
   public static final FsPermission HISTORY_DONE_FILE_PERMISSION =
     FsPermission.createImmutable((short) 0770); // rwx------
@@ -81,7 +81,7 @@ public class JobHistoryUtils {
    * Umask for the done dir and derivatives.
    */
   public static final FsPermission HISTORY_DONE_DIR_UMASK = FsPermission
-      .createImmutable((short) (0770 ^ 0777));
+      .createImmutable((short) (0771 ^ 0777));
 
   
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa30500d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 6b9f146..77b3867 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -571,8 +571,10 @@ public class HistoryFileManager extends AbstractService {
           new Path(doneDirPrefix));
       doneDirFc = FileContext.getFileContext(doneDirPrefixPath.toUri(), conf);
       doneDirFc.setUMask(JobHistoryUtils.HISTORY_DONE_DIR_UMASK);
-      mkdir(doneDirFc, doneDirPrefixPath, new FsPermission(
-          JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION));
+      FsPermission doneDirPerm = new FsPermission(
+          JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION);
+      mkdir(doneDirFc, doneDirPrefixPath, doneDirPerm);
+      checkHistoryDirsPermissions(doneDirFc, doneDirPrefixPath, doneDirPerm);
     } catch (ConnectException ex) {
       if (logWait) {
         LOG.info("Waiting for FileSystem at " +
@@ -659,6 +661,31 @@ public class HistoryFileManager extends AbstractService {
     }
   }
 
+  private void checkHistoryDirsPermissions(FileContext fc, Path donePath,
+      FsPermission fsp) throws IOException {
+    FileStatus fsStatus = fc.getFileStatus(donePath);
+    if (fsStatus.getPermission().toShort() != fsp.toShort()) {
+      fc.setPermission(donePath, fsp);
+    }
+    List<FileStatus> dirs = findTimestampedDirectories();
+    for (FileStatus dir : dirs) {
+      setPermission(fc, donePath, fsp, dir);
+    }
+  }
+
+  private void setPermission(FileContext fc, Path donePath, FsPermission fsp,
+       FileStatus dir) throws IOException {
+    FsPermission fsPerm = dir.getPermission();
+    if (fsPerm.toShort() != fsp.toShort()) {
+      fc.setPermission(dir.getPath(), fsp);
+      Path parentPath = dir.getPath().getParent();
+      if (!parentPath.equals(donePath)) {
+        FileStatus parentDir = fc.getFileStatus(parentPath);
+        setPermission(fc, donePath, fsp, parentDir);
+      }
+    }
+  }
+
   /**
    * Populates index data structures. Should only be called at initialization
    * times.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fa30500d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
index e2e943a..032bb72 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
@@ -23,6 +23,9 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.util.UUID;
 
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
 import org.junit.Assert;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -109,6 +112,76 @@ public class TestHistoryFileManager {
   }
 
   @Test
+  public void testUpdateDirPermissions() throws Exception {
+    DistributedFileSystem fs = dfsCluster.getFileSystem();
+    fs.setSafeMode( HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+    Assert.assertFalse(dfsCluster.getFileSystem().isInSafeMode());
+    Configuration conf = dfsCluster.getConfiguration(0);
+    conf.set(JHAdminConfig.MR_HISTORY_DONE_DIR, getDoneDirNameForTest());
+    conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR, getIntermediateDoneDirNameForTest());
+    Path p1a = new Path(getDoneDirNameForTest(), "2013");
+    Path p1b = new Path(p1a, "02");
+    Path p1c = new Path(p1b, "15");
+    Path p1d = new Path(p1c, "000000");
+    Path p2a = new Path(getDoneDirNameForTest(), "2013");
+    Path p2b = new Path(p2a, "03");
+    Path p2c = new Path(p2b, "14");
+    Path p2d = new Path(p2c, "000001");
+    FsPermission oldPerms = new FsPermission((short) 0770);
+    fs.mkdirs(p1d);
+    fs.mkdirs(p2d);
+    fs.setPermission(p1a, oldPerms);
+    fs.setPermission(p1b, oldPerms);
+    fs.setPermission(p1c, oldPerms);
+    fs.setPermission(p1d, oldPerms);
+    fs.setPermission(p2a, oldPerms);
+    fs.setPermission(p2b, oldPerms);
+    fs.setPermission(p2c, oldPerms);
+    fs.setPermission(p2d, oldPerms);
+    Path p1File = new Path(p1d, "foo.jhist");
+    Assert.assertTrue(fs.createNewFile(p1File));
+    fs.setPermission(p1File, JobHistoryUtils.HISTORY_DONE_FILE_PERMISSION);
+    Path p2File = new Path(p2d, "bar.jhist");
+    Assert.assertTrue(fs.createNewFile(p2File));
+    fs.setPermission(p2File, JobHistoryUtils.HISTORY_DONE_FILE_PERMISSION);
+    Assert.assertEquals(oldPerms, fs.getFileStatus(p1a).getPermission());
+    Assert.assertEquals(oldPerms, fs.getFileStatus(p1b).getPermission());
+    Assert.assertEquals(oldPerms, fs.getFileStatus(p1c).getPermission());
+    Assert.assertEquals(oldPerms, fs.getFileStatus(p1d).getPermission());
+    Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_FILE_PERMISSION,
+        fs.getFileStatus(p1File).getPermission());
+    Assert.assertEquals(oldPerms, fs.getFileStatus(p2a).getPermission());
+    Assert.assertEquals(oldPerms, fs.getFileStatus(p2b).getPermission());
+    Assert.assertEquals(oldPerms, fs.getFileStatus(p2c).getPermission());
+    Assert.assertEquals(oldPerms, fs.getFileStatus(p2d).getPermission());
+    Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_FILE_PERMISSION,
+        fs.getFileStatus(p2File).getPermission());
+    HistoryFileManager hfm = new HistoryFileManager();
+    hfm.conf = conf;
+    Assert.assertEquals(true, hfm.tryCreatingHistoryDirs(false));
+    Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION,
+        fs.getFileStatus(p1a).getPermission());
+    Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION,
+        fs.getFileStatus(p1b).getPermission());
+    Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION,
+        fs.getFileStatus(p1c).getPermission());
+    Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION,
+        fs.getFileStatus(p1d).getPermission());
+    Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_FILE_PERMISSION,
+        fs.getFileStatus(p2File).getPermission());
+    Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION,
+        fs.getFileStatus(p2a).getPermission());
+    Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION,
+        fs.getFileStatus(p2b).getPermission());
+    Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION,
+        fs.getFileStatus(p2c).getPermission());
+    Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_DIR_PERMISSION,
+        fs.getFileStatus(p2d).getPermission());
+    Assert.assertEquals(JobHistoryUtils.HISTORY_DONE_FILE_PERMISSION,
+        fs.getFileStatus(p2File).getPermission());
+  }
+
+  @Test
   public void testCreateDirsWithAdditionalFileSystem() throws Exception {
     dfsCluster.getFileSystem().setSafeMode(
         HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);


Mime
View raw message