hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject hadoop git commit: HADOOP-15943. AliyunOSS: add missing owner & group attributes for oss FileStatus. Contributed by wujinhu.
Date Fri, 23 Nov 2018 06:38:40 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.1 fdb95ef15 -> f94d40341


HADOOP-15943. AliyunOSS: add missing owner & group attributes for oss FileStatus. Contributed
by wujinhu.

(cherry picked from commit 5ff0cf86a940fd83f1425794921cc075b19f1108)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f94d4034
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f94d4034
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f94d4034

Branch: refs/heads/branch-3.1
Commit: f94d40341ff36d8b06ef7c9aa0792d90a82862f0
Parents: fdb95ef
Author: Weiwei Yang <wwei@apache.org>
Authored: Fri Nov 23 13:56:54 2018 +0800
Committer: Weiwei Yang <wwei@apache.org>
Committed: Fri Nov 23 14:27:15 2018 +0800

----------------------------------------------------------------------
 .../fs/aliyun/oss/AliyunOSSFileSystem.java      | 25 +++++++------
 .../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 11 +++---
 .../hadoop/fs/aliyun/oss/OSSFileStatus.java     | 38 ++++++++++++++++++++
 .../oss/TestAliyunOSSFileSystemContract.java    | 13 +++++++
 4 files changed, 72 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f94d4034/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index 1f95153..8e748db 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.fs.PathIOException;
 import org.apache.hadoop.fs.RemoteIterator;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.BlockingThreadPoolExecutorService;
 import org.apache.hadoop.util.Progressable;
 
@@ -70,6 +71,7 @@ public class AliyunOSSFileSystem extends FileSystem {
       LoggerFactory.getLogger(AliyunOSSFileSystem.class);
   private URI uri;
   private String bucket;
+  private String username;
   private Path workingDir;
   private int blockOutputActiveBlocks;
   private AliyunOSSFileSystemStore store;
@@ -259,7 +261,7 @@ public class AliyunOSSFileSystem extends FileSystem {
 
     // Root always exists
     if (key.length() == 0) {
-      return new FileStatus(0, true, 1, 0, 0, qualifiedPath);
+      return new OSSFileStatus(0, true, 1, 0, 0, qualifiedPath, username);
     }
 
     ObjectMetadata meta = store.getObjectMetadata(key);
@@ -273,17 +275,17 @@ public class AliyunOSSFileSystem extends FileSystem {
       ObjectListing listing = store.listObjects(key, 1, null, false);
       if (CollectionUtils.isNotEmpty(listing.getObjectSummaries()) ||
           CollectionUtils.isNotEmpty(listing.getCommonPrefixes())) {
-        return new FileStatus(0, true, 1, 0, 0, qualifiedPath);
+        return new OSSFileStatus(0, true, 1, 0, 0, qualifiedPath, username);
       } else {
         throw new FileNotFoundException(path + ": No such file or directory!");
       }
     } else if (objectRepresentsDirectory(key, meta.getContentLength())) {
-      return new FileStatus(0, true, 1, 0, meta.getLastModified().getTime(),
-          qualifiedPath);
+      return new OSSFileStatus(0, true, 1, 0, meta.getLastModified().getTime(),
+          qualifiedPath, username);
     } else {
-      return new FileStatus(meta.getContentLength(), false, 1,
+      return new OSSFileStatus(meta.getContentLength(), false, 1,
           getDefaultBlockSize(path), meta.getLastModified().getTime(),
-          qualifiedPath);
+          qualifiedPath, username);
     }
   }
 
@@ -330,15 +332,16 @@ public class AliyunOSSFileSystem extends FileSystem {
 
     bucket = name.getHost();
     uri = java.net.URI.create(name.getScheme() + "://" + name.getAuthority());
-    workingDir = new Path("/user",
-        System.getProperty("user.name")).makeQualified(uri, null);
+    // Username is the current user at the time the FS was instantiated.
+    username = UserGroupInformation.getCurrentUser().getShortUserName();
+    workingDir = new Path("/user", username).makeQualified(uri, null);
     long keepAliveTime = longOption(conf,
         KEEPALIVE_TIME_KEY, KEEPALIVE_TIME_DEFAULT, 0);
     blockOutputActiveBlocks = intOption(conf,
         UPLOAD_ACTIVE_BLOCKS_KEY, UPLOAD_ACTIVE_BLOCKS_DEFAULT, 1);
 
     store = new AliyunOSSFileSystemStore();
-    store.initialize(name, conf, statistics);
+    store.initialize(name, conf, username, statistics);
     maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
 
     int threadNum = AliyunOSSUtils.intPositiveOption(conf,
@@ -423,9 +426,9 @@ public class AliyunOSSFileSystem extends FileSystem {
             if (LOG.isDebugEnabled()) {
               LOG.debug("Adding: fi: " + keyPath);
             }
-            result.add(new FileStatus(objectSummary.getSize(), false, 1,
+            result.add(new OSSFileStatus(objectSummary.getSize(), false, 1,
                 getDefaultBlockSize(keyPath),
-                objectSummary.getLastModified().getTime(), keyPath));
+                objectSummary.getLastModified().getTime(), keyPath, username));
           }
         }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f94d4034/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
index 72b4300..dd62515 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -82,6 +82,7 @@ import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
 public class AliyunOSSFileSystemStore {
   public static final Logger LOG =
       LoggerFactory.getLogger(AliyunOSSFileSystemStore.class);
+  private String username;
   private FileSystem.Statistics statistics;
   private OSSClient ossClient;
   private String bucketName;
@@ -90,8 +91,9 @@ public class AliyunOSSFileSystemStore {
   private int maxKeys;
   private String serverSideEncryptionAlgorithm;
 
-  public void initialize(URI uri, Configuration conf,
+  public void initialize(URI uri, Configuration conf, String user,
                          FileSystem.Statistics stat) throws IOException {
+    this.username = user;
     statistics = stat;
     ClientConfiguration clientConf = new ClientConfiguration();
     clientConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS_KEY,
@@ -572,9 +574,9 @@ public class AliyunOSSFileSystemStore {
           String key = summary.getKey();
           Path path = fs.makeQualified(new Path("/" + key));
           if (filter.accept(path) && acceptor.accept(path, summary)) {
-            FileStatus status = new FileStatus(summary.getSize(),
+            FileStatus status = new OSSFileStatus(summary.getSize(),
                 key.endsWith("/"), 1, fs.getDefaultBlockSize(path),
-                summary.getLastModified().getTime(), path);
+                summary.getLastModified().getTime(), path, username);
             stats.add(status);
           }
         }
@@ -582,7 +584,8 @@ public class AliyunOSSFileSystemStore {
         for (String commonPrefix : listing.getCommonPrefixes()) {
           Path path = fs.makeQualified(new Path("/" + commonPrefix));
           if (filter.accept(path) && acceptor.accept(path, commonPrefix)) {
-            FileStatus status = new FileStatus(0, true, 1, 0, 0, path);
+            FileStatus status = new OSSFileStatus(0, true, 1, 0, 0,
+                path, username);
             stats.add(status);
           }
         }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f94d4034/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSFileStatus.java
b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSFileStatus.java
new file mode 100644
index 0000000..13ed8d8
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/OSSFileStatus.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * This class is used by listStatus for oss files.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class OSSFileStatus extends FileStatus {
+  public OSSFileStatus(long length, boolean isdir, int blockReplication,
+      long blocksize, long modTime, Path path, String user) {
+    super(length, isdir, blockReplication, blocksize, modTime, path);
+    setOwner(user);
+    setGroup(user);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f94d4034/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
index 0570146..81759a5 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystemContractBaseTest;
 import org.apache.hadoop.fs.Path;
 
+import org.apache.hadoop.security.UserGroupInformation;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -83,6 +84,18 @@ public class TestAliyunOSSFileSystemContract
   }
 
   @Test
+  public void testListStatus() throws IOException {
+    Path file = this.path("/test/hadoop/file");
+    this.createFile(file);
+    assertTrue("File exists", this.fs.exists(file));
+    FileStatus fs = this.fs.getFileStatus(file);
+    assertEquals(fs.getOwner(),
+        UserGroupInformation.getCurrentUser().getShortUserName());
+    assertEquals(fs.getGroup(),
+        UserGroupInformation.getCurrentUser().getShortUserName());
+  }
+
+  @Test
   public void testDeleteSubdir() throws IOException {
     Path parentDir = this.path("/test/hadoop");
     Path file = this.path("/test/hadoop/file");


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message