hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cnaur...@apache.org
Subject [1/3] hadoop git commit: HADOOP-12774. s3a should use UGI.getCurrentUser.getShortname() for username. Contributed by Steve Loughran.
Date Tue, 25 Oct 2016 05:18:49 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 5b7cbb5a3 -> 5c2f67bda
  refs/heads/branch-2.8 78970e0db -> 02f0472d1
  refs/heads/trunk d8fa1cfa6 -> 3372e9403


HADOOP-12774. s3a should use UGI.getCurrentUser.getShortname() for username. Contributed by
Steve Loughran.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3372e940
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3372e940
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3372e940

Branch: refs/heads/trunk
Commit: 3372e940303149d6258e0b72c54d72f080f0daa2
Parents: d8fa1cf
Author: Chris Nauroth <cnauroth@apache.org>
Authored: Mon Oct 24 21:54:06 2016 -0700
Committer: Chris Nauroth <cnauroth@apache.org>
Committed: Mon Oct 24 21:54:06 2016 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/fs/s3a/Listing.java  |  5 +--
 .../org/apache/hadoop/fs/s3a/S3AFileStatus.java | 33 ++++++++++++++------
 .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 29 ++++++++++++-----
 .../java/org/apache/hadoop/fs/s3a/S3AUtils.java |  8 +++--
 .../hadoop/fs/s3a/ITestS3AConfiguration.java    | 23 +++++++++++++-
 5 files changed, 74 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3372e940/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
index 4120b20..30d8e6f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
@@ -298,7 +298,7 @@ public class Listing {
         // Skip over keys that are ourselves and old S3N _$folder$ files
         if (acceptor.accept(keyPath, summary) && filter.accept(keyPath)) {
           FileStatus status = createFileStatus(keyPath, summary,
-              owner.getDefaultBlockSize(keyPath));
+              owner.getDefaultBlockSize(keyPath), owner.getUsername());
           LOG.debug("Adding: {}", status);
           stats.add(status);
           added++;
@@ -312,7 +312,8 @@ public class Listing {
       for (String prefix : objects.getCommonPrefixes()) {
         Path keyPath = owner.keyToQualifiedPath(prefix);
         if (acceptor.accept(keyPath, prefix) && filter.accept(keyPath)) {
-          FileStatus status = new S3AFileStatus(true, false, keyPath);
+          FileStatus status = new S3AFileStatus(false, keyPath,
+              owner.getUsername());
           LOG.debug("Adding directory: {}", status);
           added++;
           stats.add(status);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3372e940/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java
index 75a6500..b0f08e3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileStatus.java
@@ -33,28 +33,41 @@ import org.apache.hadoop.fs.Path;
 public class S3AFileStatus extends FileStatus {
   private boolean isEmptyDirectory;
 
-  // Directories
-  public S3AFileStatus(boolean isdir, boolean isemptydir, Path path) {
-    super(0, isdir, 1, 0, 0, path);
+  /**
+   * Create a directory status.
+   * @param isemptydir is this an empty directory?
+   * @param path the path
+   * @param owner the owner
+   */
+  public S3AFileStatus(boolean isemptydir,
+      Path path,
+      String owner) {
+    super(0, true, 1, 0, 0, path);
     isEmptyDirectory = isemptydir;
+    setOwner(owner);
+    setGroup(owner);
   }
 
-  // Files
+  /**
+   * A simple file.
+   * @param length file length
+   * @param modification_time mod time
+   * @param path path
+   * @param blockSize block size
+   * @param owner owner
+   */
   public S3AFileStatus(long length, long modification_time, Path path,
-      long blockSize) {
+      long blockSize, String owner) {
     super(length, false, 1, blockSize, modification_time, path);
     isEmptyDirectory = false;
+    setOwner(owner);
+    setGroup(owner);
   }
 
   public boolean isEmptyDirectory() {
     return isEmptyDirectory;
   }
 
-  @Override
-  public String getOwner() {
-    return System.getProperty("user.name");
-  }
-
   /** Compare if this object is equal to another object.
    * @param   o the object to be compared.
    * @return  true if two file status has the same path name; false if not.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3372e940/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 9908ba7..6030fe4 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -87,6 +87,7 @@ import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.StorageStatistics;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.fs.s3native.S3xLoginHelper;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Progressable;
 import org.apache.hadoop.util.ReflectionUtils;
 
@@ -120,6 +121,7 @@ public class S3AFileSystem extends FileSystem {
   public static final int DEFAULT_BLOCKSIZE = 32 * 1024 * 1024;
   private URI uri;
   private Path workingDir;
+  private String username;
   private AmazonS3 s3;
   private String bucket;
   private int maxKeys;
@@ -160,7 +162,9 @@ public class S3AFileSystem extends FileSystem {
       instrumentation = new S3AInstrumentation(name);
 
       uri = S3xLoginHelper.buildFSURI(name);
-      workingDir = new Path("/user", System.getProperty("user.name"))
+      // Username is the current user at the time the FS was instantiated.
+      username = UserGroupInformation.getCurrentUser().getShortUserName();
+      workingDir = new Path("/user", username)
           .makeQualified(this.uri, this.getWorkingDirectory());
 
       bucket = name.getHost();
@@ -1389,6 +1393,14 @@ public class S3AFileSystem extends FileSystem {
   }
 
   /**
+   * Get the username of the FS.
+   * @return the short name of the user who instantiated the FS
+   */
+  public String getUsername() {
+    return username;
+  }
+
+  /**
    *
    * Make the given path and all non-existent parents into
    * directories. Has the semantics of Unix {@code 'mkdir -p'}.
@@ -1479,14 +1491,14 @@ public class S3AFileSystem extends FileSystem {
 
         if (objectRepresentsDirectory(key, meta.getContentLength())) {
           LOG.debug("Found exact file: fake directory");
-          return new S3AFileStatus(true, true,
-              path);
+          return new S3AFileStatus(true, path, username);
         } else {
           LOG.debug("Found exact file: normal file");
           return new S3AFileStatus(meta.getContentLength(),
               dateToLong(meta.getLastModified()),
               path,
-              getDefaultBlockSize(path));
+              getDefaultBlockSize(path),
+              username);
         }
       } catch (AmazonServiceException e) {
         if (e.getStatusCode() != 404) {
@@ -1504,7 +1516,7 @@ public class S3AFileSystem extends FileSystem {
 
           if (objectRepresentsDirectory(newKey, meta.getContentLength())) {
             LOG.debug("Found file (with /): fake directory");
-            return new S3AFileStatus(true, true, path);
+            return new S3AFileStatus(true, path, username);
           } else {
             LOG.warn("Found file (with /): real file? should not happen: {}",
                 key);
@@ -1512,7 +1524,8 @@ public class S3AFileSystem extends FileSystem {
             return new S3AFileStatus(meta.getContentLength(),
                 dateToLong(meta.getLastModified()),
                 path,
-                getDefaultBlockSize(path));
+                getDefaultBlockSize(path),
+                username);
           }
         } catch (AmazonServiceException e) {
           if (e.getStatusCode() != 404) {
@@ -1549,10 +1562,10 @@ public class S3AFileSystem extends FileSystem {
           }
         }
 
-        return new S3AFileStatus(true, false, path);
+        return new S3AFileStatus(false, path, username);
       } else if (key.isEmpty()) {
         LOG.debug("Found root directory");
-        return new S3AFileStatus(true, true, path);
+        return new S3AFileStatus(true, path, username);
       }
     } catch (AmazonServiceException e) {
       if (e.getStatusCode() != 404) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3372e940/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index f926f34..56e0c37 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -241,17 +241,19 @@ public final class S3AUtils {
    * @param keyPath path to entry
    * @param summary summary from AWS
    * @param blockSize block size to declare.
+   * @param owner owner of the file
    * @return a status entry
    */
   public static S3AFileStatus createFileStatus(Path keyPath,
       S3ObjectSummary summary,
-      long blockSize) {
+      long blockSize,
+      String owner) {
     if (objectRepresentsDirectory(summary.getKey(), summary.getSize())) {
-      return new S3AFileStatus(true, true, keyPath);
+      return new S3AFileStatus(true, keyPath, owner);
     } else {
       return new S3AFileStatus(summary.getSize(),
           dateToLong(summary.getLastModified()), keyPath,
-          blockSize);
+          blockSize, owner);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3372e940/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index 30d4bf6..04057a9 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -44,8 +44,10 @@ import static org.junit.Assert.fail;
 
 import java.io.File;
 import java.net.URI;
+import java.security.PrivilegedExceptionAction;
 
 import org.apache.hadoop.security.ProviderUtils;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.alias.CredentialProvider;
 import org.apache.hadoop.security.alias.CredentialProviderFactory;
 import org.apache.hadoop.util.VersionInfo;
@@ -436,7 +438,7 @@ public class ITestS3AConfiguration {
     dir1.mkdirs();
     dir2.mkdirs();
     conf = new Configuration();
-    conf.set(Constants.BUFFER_DIR, dir1 +", " + dir2);
+    conf.set(Constants.BUFFER_DIR, dir1 + ", " + dir2);
     fs = S3ATestUtils.createTestFileSystem(conf);
     File tmp1 = fs.createTmpFileForWrite("out-", 1024, conf);
     tmp1.delete();
@@ -446,6 +448,25 @@ public class ITestS3AConfiguration {
         tmp1.getParent(), tmp2.getParent());
   }
 
+  @Test
+  public void testUsernameFromUGI() throws Throwable {
+    final String alice = "alice";
+    UserGroupInformation fakeUser =
+        UserGroupInformation.createUserForTesting(alice,
+            new String[]{"users", "administrators"});
+    conf = new Configuration();
+    fs = fakeUser.doAs(new PrivilegedExceptionAction<S3AFileSystem>() {
+      @Override
+      public S3AFileSystem run() throws Exception{
+        return S3ATestUtils.createTestFileSystem(conf);
+      }
+    });
+    assertEquals("username", alice, fs.getUsername());
+    S3AFileStatus status = fs.getFileStatus(new Path("/"));
+    assertEquals("owner in " + status, alice, status.getOwner());
+    assertEquals("group in " + status, alice, status.getGroup());
+  }
+
   /**
    * Reads and returns a field from an object using reflection.  If the field
    * cannot be found, is null, or is not the expected type, then this method


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message