hadoop-ozone-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From x...@apache.org
Subject [hadoop-ozone] branch HDDS-2665-ofs updated: HDDS-2929. Implement ofs://: temp directory mount (#610)
Date Sat, 07 Mar 2020 01:33:12 GMT
This is an automated email from the ASF dual-hosted git repository.

xyao pushed a commit to branch HDDS-2665-ofs
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git


The following commit(s) were added to refs/heads/HDDS-2665-ofs by this push:
     new c410c4d  HDDS-2929. Implement ofs://: temp directory mount (#610)
c410c4d is described below

commit c410c4d329a839547bbf9491b5bf4562efcda59e
Author: Siyao Meng <50227127+smengcl@users.noreply.github.com>
AuthorDate: Fri Mar 6 17:33:01 2020 -0800

    HDDS-2929. Implement ofs://: temp directory mount (#610)
---
 .../src/main/compose/ozonesecure/docker-config     |   1 +
 .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 115 +++++++++++++++++----
 .../ozone/BasicRootedOzoneClientAdapterImpl.java   |   6 +-
 .../java/org/apache/hadoop/fs/ozone/OFSPath.java   |  48 +++++++--
 .../org/apache/hadoop/fs/ozone/TestOFSPath.java    |  22 ++--
 5 files changed, 159 insertions(+), 33 deletions(-)

diff --git a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
index b8ddc97..0570b46 100644
--- a/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
+++ b/hadoop-ozone/dist/src/main/compose/ozonesecure/docker-config
@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+CORE-SITE.XML_fs.ofs.impl=org.apache.hadoop.fs.ozone.RootedOzoneFileSystem
 CORE-SITE.XML_fs.o3fs.impl=org.apache.hadoop.fs.ozone.OzoneFileSystem
 
 OZONE-SITE.XML_ozone.om.address=om
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index a881f10..b0550e1 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -29,12 +29,19 @@ import org.apache.hadoop.fs.contract.ContractTestUtils;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneAcl;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.TestDataUtil;
 import org.apache.hadoop.ozone.client.ObjectStore;
 import org.apache.hadoop.ozone.client.OzoneBucket;
 import org.apache.hadoop.ozone.client.OzoneKeyDetails;
 import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.VolumeArgs;
+import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
+import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLIdentityType;
+import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType;
+import org.apache.hadoop.ozone.security.acl.OzoneAclConfig;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.After;
 import org.junit.Assert;
@@ -45,6 +52,7 @@ import org.junit.rules.Timeout;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
@@ -53,8 +61,10 @@ import java.util.TreeSet;
 import java.util.stream.Collectors;
 
 import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
 import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.BUCKET_NOT_FOUND;
 
 /**
  * Ozone file system tests that are not covered by contract tests.
@@ -65,10 +75,11 @@ public class TestRootedOzoneFileSystem {
   @Rule
   public Timeout globalTimeout = new Timeout(300_000);
 
-  private static MiniOzoneCluster cluster = null;
-  private static FileSystem fs;
-  private static RootedOzoneFileSystem ofs;
-  private static ObjectStore objectStore;
+  private OzoneConfiguration conf;
+  private MiniOzoneCluster cluster = null;
+  private FileSystem fs;
+  private RootedOzoneFileSystem ofs;
+  private ObjectStore objectStore;
   private static BasicRootedOzoneClientAdapterImpl adapter;
 
   private String volumeName;
@@ -79,7 +90,7 @@ public class TestRootedOzoneFileSystem {
 
   @Before
   public void init() throws Exception {
-    OzoneConfiguration conf = new OzoneConfiguration();
+    conf = new OzoneConfiguration();
     cluster = MiniOzoneCluster.newBuilder(conf)
         .setNumDatanodes(3)
         .build();
@@ -117,12 +128,13 @@ public class TestRootedOzoneFileSystem {
 
   @Test
   public void testOzoneFsServiceLoader() throws IOException {
-    OzoneConfiguration conf = new OzoneConfiguration();
+    OzoneConfiguration confTestLoader = new OzoneConfiguration();
     // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
     //  hence this workaround.
-    conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
-    Assert.assertEquals(
-        FileSystem.getFileSystemClass(OzoneConsts.OZONE_OFS_URI_SCHEME, conf),
+    confTestLoader.set("fs.ofs.impl",
+        "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
+    Assert.assertEquals(FileSystem.getFileSystemClass(
+        OzoneConsts.OZONE_OFS_URI_SCHEME, confTestLoader),
         RootedOzoneFileSystem.class);
   }
 
@@ -661,32 +673,97 @@ public class TestRootedOzoneFileSystem {
     // FileSystem because we can't change LISTING_PAGE_SIZE. Use adapter instead
 
     // numEntries > 5
-    FileStatus[] fileStatusesOver = customListStatus(
-        new Path("/"), false, "", 8);
+    FileStatus[] fileStatusesOver = customListStatus(new Path("/"),
+        false, "", 8);
     // There are only 5 volumes
     Assert.assertEquals(5, fileStatusesOver.length);
 
     // numEntries = 5
-    FileStatus[] fileStatusesExact = customListStatus(
-        new Path("/"), false, "", 5);
+    FileStatus[] fileStatusesExact = customListStatus(new Path("/"),
+        false, "", 5);
     Assert.assertEquals(5, fileStatusesExact.length);
 
     // numEntries < 5
-    FileStatus[] fileStatusesLimit1 = customListStatus(
-        new Path("/"), false, "", 3);
+    FileStatus[] fileStatusesLimit1 = customListStatus(new Path("/"),
+        false, "", 3);
     // Should only return 3 volumes even though there are more than that due to
     // the specified limit
     Assert.assertEquals(3, fileStatusesLimit1.length);
 
     // Get the last entry in the list as startPath
-    String nextStartPath = fileStatusesLimit1[fileStatusesLimit1.length - 1]
-        .getPath().toString();
-    FileStatus[] fileStatusesLimit2 = customListStatus(
-        new Path("/"), false, nextStartPath, 3);
+    String nextStartPath =
+        fileStatusesLimit1[fileStatusesLimit1.length - 1].getPath().toString();
+    FileStatus[] fileStatusesLimit2 = customListStatus(new Path("/"),
+        false, nextStartPath, 3);
     // Note: at the time of writing this test, OmMetadataManagerImpl#listVolumes
     //  excludes startVolume (startPath) from the result. Might change.
     Assert.assertEquals(fileStatusesOver.length,
         fileStatusesLimit1.length + fileStatusesLimit2.length);
   }
 
+   /*
+   * OFS: Test /tmp mount behavior.
+   */
+  @Test
+  public void testTempMount() throws Exception {
+    // Prep
+    // Use ClientProtocol to pass in volume ACL, ObjectStore won't do it
+    ClientProtocol proxy = objectStore.getClientProxy();
+    // Get default acl rights for user
+    OzoneAclConfig aclConfig = conf.getObject(OzoneAclConfig.class);
+    ACLType userRights = aclConfig.getUserDefaultRights();
+    // Construct ACL for world access
+    OzoneAcl aclWorldAccess = new OzoneAcl(ACLIdentityType.WORLD, "",
+        userRights, ACCESS);
+    // Construct VolumeArgs
+    VolumeArgs volumeArgs = new VolumeArgs.Builder()
+        .setAcls(Collections.singletonList(aclWorldAccess)).build();
+    // Sanity check
+    Assert.assertNull(volumeArgs.getOwner());
+    Assert.assertNull(volumeArgs.getAdmin());
+    Assert.assertNull(volumeArgs.getQuota());
+    Assert.assertEquals(0, volumeArgs.getMetadata().size());
+    Assert.assertEquals(1, volumeArgs.getAcls().size());
+    // Create volume "tmp" with world access. allow non-admin to create buckets
+    proxy.createVolume(OFSPath.OFS_MOUNT_TMP_VOLUMENAME, volumeArgs);
+
+    OzoneVolume vol = objectStore.getVolume(OFSPath.OFS_MOUNT_TMP_VOLUMENAME);
+    Assert.assertNotNull(vol);
+
+    // Begin test
+    String hashedUsername = OFSPath.getTempMountBucketNameOfCurrentUser();
+
+    // Expect failure since temp bucket for current user is not created yet
+    try {
+      vol.getBucket(hashedUsername);
+    } catch (OMException ex) {
+      // Expect BUCKET_NOT_FOUND
+      if (!ex.getResult().equals(BUCKET_NOT_FOUND)) {
+        Assert.fail("Temp bucket for current user shouldn't have been created");
+      }
+    }
+
+    // Write under /tmp/, OFS will create the temp bucket if not exist
+    fs.mkdirs(new Path("/tmp/dir1"));
+
+    try (FSDataOutputStream stream = ofs.create(new Path("/tmp/dir1/file1"))) {
+      stream.write(1);
+    }
+
+    // Verify temp bucket creation
+    OzoneBucket bucket = vol.getBucket(hashedUsername);
+    Assert.assertNotNull(bucket);
+    // Verify dir1 creation
+    FileStatus[] fileStatuses = fs.listStatus(new Path("/tmp/"));
+    Assert.assertEquals(1, fileStatuses.length);
+    Assert.assertEquals(
+        "/tmp/dir1", fileStatuses[0].getPath().toUri().getPath());
+    // Verify file1 creation
+    FileStatus[] fileStatusesInDir1 =
+        fs.listStatus(new Path("/tmp/dir1"));
+    Assert.assertEquals(1, fileStatusesInDir1.length);
+    Assert.assertEquals("/tmp/dir1/file1",
+        fileStatusesInDir1[0].getPath().toUri().getPath());
+  }
+
 }
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index 3baba25..64f8581 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -213,6 +213,7 @@ public class BasicRootedOzoneClientAdapterImpl
     try {
       bucket = proxy.getBucketDetails(volumeStr, bucketStr);
     } catch (OMException ex) {
+      // Note: always create bucket if volumeStr matches "tmp" so -put works
       if (createIfNotExist) {
         // Note: getBucketDetails always throws BUCKET_NOT_FOUND, even if
         // the volume doesn't exist.
@@ -292,7 +293,8 @@ public class BasicRootedOzoneClientAdapterImpl
     OFSPath ofsPath = new OFSPath(pathStr);
     String key = ofsPath.getKeyName();
     try {
-      OzoneBucket bucket = getBucket(ofsPath, false);
+      // Hadoop CopyCommands class always sets recursive to true
+      OzoneBucket bucket = getBucket(ofsPath, recursive);
       OzoneOutputStream ozoneOutputStream = bucket.createFile(
           key, 0, replicationType, replicationFactor, overWrite, recursive);
       return new OzoneFSOutputStream(ozoneOutputStream.getOutputStream());
@@ -458,6 +460,8 @@ public class BasicRootedOzoneClientAdapterImpl
     } catch (OMException e) {
       if (e.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) {
         throw new FileNotFoundException(key + ": No such file or directory!");
+      } else if (e.getResult() == OMException.ResultCodes.BUCKET_NOT_FOUND) {
+        throw new FileNotFoundException(key + ": Bucket doesn't exist!");
       }
       throw e;
     }
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
index c9ae6a7..f7d4ded 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
@@ -17,13 +17,17 @@
  */
 package org.apache.hadoop.fs.ozone;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.http.ParseException;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.apache.yetus.audience.InterfaceStability;
-
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.io.IOException;
 import java.util.StringTokenizer;
 
 import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
@@ -44,7 +48,7 @@ class OFSPath {
    * /vol1/buc2/dir3/key4  vol1           buc2           (empty)      dir3/key4
    * /vol1/buc2            vol1           buc2           (empty)      (empty)
    * /vol1                 vol1           (empty)        (empty)      (empty)
-   * /tmp/dir3/key4        tempVolume     tempBucket     tmp          dir3/key4
+   * /tmp/dir3/key4        tmp            <username>     tmp          dir3/key4
    *
    * Note the leading '/' doesn't matter.
    */
@@ -53,6 +57,9 @@ class OFSPath {
   private String mountName = "";
   private String keyName = "";
   private static final String OFS_MOUNT_NAME_TMP = "tmp";
+  // Hard-code the volume name to tmp for the first implementation
+  @VisibleForTesting
+  static final String OFS_MOUNT_TMP_VOLUMENAME = "tmp";
 
   OFSPath(Path path) {
     String pathStr = path.toUri().getPath();
@@ -83,10 +90,15 @@ class OFSPath {
       // TODO: Compare a keyword list instead for future expansion.
       if (firstToken.equals(OFS_MOUNT_NAME_TMP)) {
         mountName = firstToken;
-        // TODO: Retrieve volume and bucket of the mount from user protobuf.
-        //  Leave them hard-coded just for now. Will be addressed in HDDS-2929
-        volumeName = "tempVolume";
-        bucketName = "tempBucket";
+        // TODO: In the future, may retrieve volume and bucket from
+        //  UserVolumeInfo on the server side. TBD.
+        volumeName = OFS_MOUNT_TMP_VOLUMENAME;
+        try {
+          bucketName = getTempMountBucketNameOfCurrentUser();
+        } catch (IOException ex) {
+          throw new ParseException(
+              "Failed to get temp bucket name for current user.");
+        }
       } else if (numToken >= 2) {
         // Regular volume and bucket path
         volumeName = firstToken;
@@ -95,7 +107,6 @@ class OFSPath {
         // Volume only
         volumeName = firstToken;
       }
-//    } else {  // TODO: Implement '/' case for ls.
     }
 
     // Compose key name
@@ -174,4 +185,27 @@ class OFSPath {
   public boolean isVolume() {
     return this.getBucketName().isEmpty() && !this.getVolumeName().isEmpty();
   }
+
+  /**
+   * Get the bucket name of temp for given username.
+   * @param username Input user name String. Mustn't be null.
+   * @return Username MD5 hash in hex digits.
+   */
+  @VisibleForTesting
+  static String getTempMountBucketName(String username) {
+    Preconditions.checkNotNull(username);
+    // TODO: Improve this to "slugify(username)-md5(username)" for better
+    //  readability?
+    return DigestUtils.md5Hex(username);
+  }
+
+  /**
+   * Get the bucket name of temp for the current user from UserGroupInformation.
+   * @return Username MD5 hash in hex digits.
+   * @throws IOException When UserGroupInformation.getCurrentUser() fails.
+   */
+  static String getTempMountBucketNameOfCurrentUser() throws IOException {
+    String username = UserGroupInformation.getCurrentUser().getUserName();
+    return getTempMountBucketName(username);
+  }
 }
diff --git a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java
b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java
index 05e1ce2..c46c09f 100644
--- a/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java
+++ b/hadoop-ozone/ozonefs/src/test/java/org/apache/hadoop/fs/ozone/TestOFSPath.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.ozone;
 import org.junit.Assert;
 import org.junit.Test;
 
+import java.io.IOException;
+
 /**
  * Testing basic functions of utility class OFSPath.
  */
@@ -103,11 +105,19 @@ public class TestOFSPath {
 
   @Test
   public void testParsingMount() {
+    String bucketName;
+    try {
+      bucketName = OFSPath.getTempMountBucketNameOfCurrentUser();
+    } catch (IOException ex) {
+      Assert.fail("Failed to get the current user name, "
+          + "thus failed to get temp bucket name.");
+      bucketName = "";  // Make javac happy
+    }
     // Mount only
     OFSPath ofsPath = new OFSPath("/tmp/");
-    // TODO: Subject to change in HDDS-2929.
-    Assert.assertEquals("tempVolume", ofsPath.getVolumeName());
-    Assert.assertEquals("tempBucket", ofsPath.getBucketName());
+    Assert.assertEquals(
+        OFSPath.OFS_MOUNT_TMP_VOLUMENAME, ofsPath.getVolumeName());
+    Assert.assertEquals(bucketName, ofsPath.getBucketName());
     Assert.assertEquals("tmp", ofsPath.getMountName());
     Assert.assertEquals("", ofsPath.getKeyName());
     Assert.assertEquals("/tmp", ofsPath.getNonKeyPath());
@@ -115,9 +125,9 @@ public class TestOFSPath {
 
     // Mount with key
     ofsPath = new OFSPath("/tmp/key1");
-    // TODO: Subject to change in HDDS-2929.
-    Assert.assertEquals("tempVolume", ofsPath.getVolumeName());
-    Assert.assertEquals("tempBucket", ofsPath.getBucketName());
+    Assert.assertEquals(
+        OFSPath.OFS_MOUNT_TMP_VOLUMENAME, ofsPath.getVolumeName());
+    Assert.assertEquals(bucketName, ofsPath.getBucketName());
     Assert.assertEquals("tmp", ofsPath.getMountName());
     Assert.assertEquals("key1", ofsPath.getKeyName());
     Assert.assertEquals("/tmp", ofsPath.getNonKeyPath());


---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org


Mime
View raw message