hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject hadoop git commit: MAPREDUCE-7159. FrameworkUploader: ensure proper permissions of generated framework tar.gz if restrictive umask is used. Contributed by Peter Bacsko
Date Thu, 06 Dec 2018 21:01:42 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.2 8c70728f7 -> 298a1f29e


MAPREDUCE-7159. FrameworkUploader: ensure proper permissions of generated framework tar.gz
if restrictive umask is used. Contributed by Peter Bacsko

(cherry picked from commit 9886c27c71147a2bcef87b42ff21c2bbe98c8ab4)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/298a1f29
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/298a1f29
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/298a1f29

Branch: refs/heads/branch-3.2
Commit: 298a1f29e8cd145e41206364fb77230dd187ddfe
Parents: 8c70728
Author: Jason Lowe <jlowe@apache.org>
Authored: Thu Dec 6 14:48:17 2018 -0600
Committer: Jason Lowe <jlowe@apache.org>
Committed: Thu Dec 6 14:57:45 2018 -0600

----------------------------------------------------------------------
 .../hadoop-mapreduce-client-uploader/pom.xml    |  6 ++
 .../mapred/uploader/FrameworkUploader.java      | 52 ++++++++++++++++++
 .../mapred/uploader/TestFrameworkUploader.java  | 58 ++++++++++++++++++++
 3 files changed, 116 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/298a1f29/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml
index 79ae97f..82784f0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/pom.xml
@@ -40,6 +40,12 @@
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-hdfs</artifactId>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-hdfs-client</artifactId>
         </dependency>
     </dependencies>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/298a1f29/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
index d2116c0..6f914f7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/main/java/org/apache/hadoop/mapred/uploader/FrameworkUploader.java
@@ -26,12 +26,16 @@ import org.apache.commons.compress.archivers.ArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringUtils;
@@ -71,6 +75,10 @@ public class FrameworkUploader implements Runnable {
       LoggerFactory.getLogger(FrameworkUploader.class);
   private Configuration conf = new Configuration();
 
+  // Minimal required permissions for the uploaded framework
+  private static final FsPermission FRAMEWORK_PERMISSION =
+      new FsPermission(0644);
+
   @VisibleForTesting
   String input = null;
   @VisibleForTesting
@@ -99,6 +107,7 @@ public class FrameworkUploader implements Runnable {
   List<Pattern> blacklistedFiles = new LinkedList<>();
 
   private OutputStream targetStream = null;
+  private FSDataOutputStream fsDataStream = null;
   private String alias = null;
 
   @VisibleForTesting
@@ -203,11 +212,50 @@ public class FrameworkUploader implements Runnable {
         targetStream = fileSystem.create(targetPath, true);
       }
 
+      if (!FRAMEWORK_PERMISSION.equals(
+          FRAMEWORK_PERMISSION.applyUMask(FsPermission.getUMask(conf)))) {
+        LOG.info("Modifying permissions to " + FRAMEWORK_PERMISSION);
+        fileSystem.setPermission(targetPath, FRAMEWORK_PERMISSION);
+      }
+
+      fsDataStream = (FSDataOutputStream) targetStream;
       if (targetPath.getName().endsWith("gz") ||
           targetPath.getName().endsWith("tgz")) {
         LOG.info("Creating GZip");
         targetStream = new GZIPOutputStream(targetStream);
       }
+
+      Path current = targetPath.getParent();
+      // Walk the path backwards to verify that the uploaded
+      // framework is accessible for all users
+      while (current != null) {
+        try {
+          FileStatus fstat = fileSystem.getFileStatus(current);
+          FsPermission perm = fstat.getPermission();
+
+          // Note: READ is not necessary to enter the directory.
+          // We need to check only the EXECUTE flag
+          boolean userCanEnter = perm.getUserAction()
+              .implies(FsAction.EXECUTE);
+          boolean groupCanEnter = perm.getGroupAction()
+              .implies(FsAction.EXECUTE);
+          boolean othersCanEnter = perm.getOtherAction()
+              .implies(FsAction.EXECUTE);
+
+          if (!userCanEnter || !groupCanEnter || !othersCanEnter) {
+            LOG.warn("Path " + current + " is not accessible"
+                + " for all users. Current permissions are: " + perm);
+            LOG.warn("Please set EXECUTE permissions on this directory");
+          }
+          current = current.getParent();
+        } catch (AccessControlException e) {
+          LOG.warn("Path " + current + " is not accessible,"
+              + " cannot retrieve permissions");
+          LOG.warn("Please set EXECUTE permissions on this directory");
+          LOG.debug("Stack trace", e);
+          break;
+        }
+      }
     }
   }
 
@@ -290,6 +338,10 @@ public class FrameworkUploader implements Runnable {
           out.closeArchiveEntry();
         }
       }
+
+      // Necessary to see proper replication counts in endUpload()
+      fsDataStream.hflush();
+
       endUpload();
     } finally {
       if (targetStream != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/298a1f29/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
index 9c72f72..6e6a902 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-uploader/src/test/java/org/apache/hadoop/mapred/uploader/TestFrameworkUploader.java
@@ -24,6 +24,14 @@ import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.junit.Assert;
 import org.junit.Assume;
 import org.junit.Before;
@@ -470,4 +478,54 @@ public class TestFrameworkUploader {
 
   }
 
+  @Test
+  public void testPermissionSettingsOnRestrictiveUmask()
+      throws Exception {
+    File parent = new File(testDir);
+    parent.deleteOnExit();
+    MiniDFSCluster cluster = null;
+
+    try {
+      Assert.assertTrue("Directory creation failed", parent.mkdirs());
+      Configuration hdfsConf = new HdfsConfiguration();
+      String namenodeDir = new File(MiniDFSCluster.getBaseDirectory(),
+          "name").getAbsolutePath();
+      hdfsConf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, namenodeDir);
+      hdfsConf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, namenodeDir);
+      hdfsConf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "027");
+      cluster = new MiniDFSCluster.Builder(hdfsConf)
+          .numDataNodes(1).build();
+      DistributedFileSystem dfs = cluster.getFileSystem();
+      cluster.waitActive();
+
+      File file1 = new File(parent, "a.jar");
+      file1.createNewFile();
+      File file2 = new File(parent, "b.jar");
+      file2.createNewFile();
+      File file3 = new File(parent, "c.jar");
+      file3.createNewFile();
+
+      FrameworkUploader uploader = new FrameworkUploader();
+      uploader.whitelist = "";
+      uploader.blacklist = "";
+      uploader.input = parent.getAbsolutePath() + File.separatorChar + "*";
+      String hdfsUri = hdfsConf.get(FS_DEFAULT_NAME_KEY);
+      String targetPath = "/test.tar.gz";
+      uploader.target = hdfsUri + targetPath;
+      uploader.acceptableReplication = 1;
+      uploader.setConf(hdfsConf);
+
+      uploader.collectPackages();
+      uploader.buildPackage();
+
+      FileStatus fileStatus = dfs.getFileStatus(new Path(targetPath));
+      FsPermission perm = fileStatus.getPermission();
+      Assert.assertEquals("Permissions", new FsPermission(0644), perm);
+    } finally {
+      if (cluster != null) {
+        cluster.close();
+      }
+      FileUtils.deleteDirectory(parent);
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message