hadoop-ozone-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From x...@apache.org
Subject [hadoop-ozone] branch HDDS-2665-ofs updated: HDDS-2979. Implement ofs://: Fix getFileStatus for mkdir volume (#528)
Date Fri, 07 Feb 2020 00:51:23 GMT
This is an automated email from the ASF dual-hosted git repository.

xyao pushed a commit to branch HDDS-2665-ofs
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git


The following commit(s) were added to refs/heads/HDDS-2665-ofs by this push:
     new 3e4782d  HDDS-2979. Implement ofs://: Fix getFileStatus for mkdir volume (#528)
3e4782d is described below

commit 3e4782d3aaeb5b4d4b7429f39a8ce867d3481be4
Author: Siyao Meng <50227127+smengcl@users.noreply.github.com>
AuthorDate: Thu Feb 6 16:51:17 2020 -0800

    HDDS-2979. Implement ofs://: Fix getFileStatus for mkdir volume (#528)
---
 .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 138 +++++++++++----------
 .../ozone/BasicRootedOzoneClientAdapterImpl.java   |  44 ++++++-
 2 files changed, 116 insertions(+), 66 deletions(-)

diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index 97d739b..97f840f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -25,17 +25,18 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.ozone.MiniOzoneCluster;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.TestDataUtil;
 import org.apache.hadoop.ozone.client.ObjectStore;
 import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClientException;
 import org.apache.hadoop.ozone.client.OzoneKeyDetails;
 import org.apache.hadoop.ozone.client.OzoneVolume;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -48,17 +49,10 @@ import java.util.TreeSet;
 
 import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 /**
  * Ozone file system tests that are not covered by contract tests.
- *
- * TODO: Refactor this and TestOzoneFileSystem to eliminate most
- *  code duplication.
+ * TODO: Refactor this and TestOzoneFileSystem later to reduce code duplication.
  */
 public class TestRootedOzoneFileSystem {
 
@@ -66,19 +60,13 @@ public class TestRootedOzoneFileSystem {
   public Timeout globalTimeout = new Timeout(300_000);
 
   private static MiniOzoneCluster cluster = null;
-
   private static FileSystem fs;
   private static RootedOzoneFileSystem ofs;
-
   private static ObjectStore objectStore;
 
   private String volumeName;
   private String bucketName;
-
-  private String rootPath;
-
   // Store path commonly used by tests that test functionality within a bucket
-  private String testBucketStr;
   private Path testBucketPath;
 
   @Before
@@ -94,11 +82,11 @@ public class TestRootedOzoneFileSystem {
     OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
     volumeName = bucket.getVolumeName();
     bucketName = bucket.getName();
-    testBucketStr = "/" + volumeName + "/" + bucketName;
+    String testBucketStr = "/" + volumeName + "/" + bucketName;
     testBucketPath = new Path(testBucketStr);
 
-    rootPath = String.format("%s://%s/", OzoneConsts.OZONE_OFS_URI_SCHEME,
-        conf.get(OZONE_OM_ADDRESS_KEY));
+    String rootPath = String.format("%s://%s/",
+        OzoneConsts.OZONE_OFS_URI_SCHEME, conf.get(OZONE_OM_ADDRESS_KEY));
 
     // Set the fs.defaultFS and start the filesystem
     conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, rootPath);
@@ -123,7 +111,7 @@ public class TestRootedOzoneFileSystem {
     // Note: FileSystem#loadFileSystems won't load OFS class due to META-INF
     //  hence this workaround.
     conf.set("fs.ofs.impl", "org.apache.hadoop.fs.ozone.RootedOzoneFileSystem");
-    assertEquals(
+    Assert.assertEquals(
         FileSystem.getFileSystemClass(OzoneConsts.OZONE_OFS_URI_SCHEME, conf),
         RootedOzoneFileSystem.class);
   }
@@ -138,7 +126,7 @@ public class TestRootedOzoneFileSystem {
 
     OzoneKeyDetails key = getKey(child, false);
     OFSPath childOFSPath = new OFSPath(child);
-    assertEquals(key.getName(), childOFSPath.getKeyName());
+    Assert.assertEquals(key.getName(), childOFSPath.getKeyName());
 
     // Creating a child should not add parent keys to the bucket
     try {
@@ -148,9 +136,11 @@ public class TestRootedOzoneFileSystem {
     }
 
     // List status on the parent should show the child file
-    assertEquals("List status of parent should include the 1 child file", 1L,
-        (long)fs.listStatus(parent).length);
-    assertTrue("Parent directory does not appear to be a directory",
+    Assert.assertEquals(
+        "List status of parent should include the 1 child file",
+        1L, fs.listStatus(parent).length);
+    Assert.assertTrue(
+        "Parent directory does not appear to be a directory",
         fs.getFileStatus(parent).isDirectory());
   }
 
@@ -171,17 +161,17 @@ public class TestRootedOzoneFileSystem {
     }
 
     // Delete the child key
-    assertTrue(fs.delete(child, false));
+    Assert.assertTrue(fs.delete(child, false));
 
     // Deleting the only child should create the parent dir key if it does
     // not exist
     OFSPath parentOFSPath = new OFSPath(parent);
     String parentKey = parentOFSPath.getKeyName() + "/";
     OzoneKeyDetails parentKeyInfo = getKey(parent, true);
-    assertEquals(parentKey, parentKeyInfo.getName());
+    Assert.assertEquals(parentKey, parentKeyInfo.getName());
 
     // Recursive delete with DeleteIterator
-    assertTrue(fs.delete(grandparent, true));
+    Assert.assertTrue(fs.delete(grandparent, true));
   }
 
   @Test
@@ -191,21 +181,22 @@ public class TestRootedOzoneFileSystem {
     Path file2 = new Path(parent, "key2");
 
     FileStatus[] fileStatuses = ofs.listStatus(testBucketPath);
-    assertEquals("Should be empty", 0, fileStatuses.length);
+    Assert.assertEquals("Should be empty", 0, fileStatuses.length);
 
     ContractTestUtils.touch(fs, file1);
     ContractTestUtils.touch(fs, file2);
 
     fileStatuses = ofs.listStatus(testBucketPath);
-    assertEquals("Should have created parent",
+    Assert.assertEquals("Should have created parent",
         1, fileStatuses.length);
-    assertEquals("Parent path doesn't match",
+    Assert.assertEquals("Parent path doesn't match",
         fileStatuses[0].getPath().toUri().getPath(), parent.toString());
 
     // ListStatus on a directory should return all subdirs along with
     // files, even if there exists a file and sub-dir with the same name.
     fileStatuses = ofs.listStatus(parent);
-    assertEquals("FileStatus did not return all children of the directory",
+    Assert.assertEquals(
+        "FileStatus did not return all children of the directory",
         2, fileStatuses.length);
 
     // ListStatus should return only the immediate children of a directory.
@@ -214,7 +205,8 @@ public class TestRootedOzoneFileSystem {
     ContractTestUtils.touch(fs, file3);
     ContractTestUtils.touch(fs, file4);
     fileStatuses = ofs.listStatus(parent);
-    assertEquals("FileStatus did not return all children of the directory",
+    Assert.assertEquals(
+        "FileStatus did not return all children of the directory",
         3, fileStatuses.length);
   }
 
@@ -264,23 +256,26 @@ public class TestRootedOzoneFileSystem {
     OFSPath ofsPathDir1 = new OFSPath(dir12);
     String key = ofsPathDir1.getKeyName() + "/";
     OzoneKeyDetails ozoneKeyDetails = ozoneBucket.getKey(key);
-    assertEquals(key, ozoneKeyDetails.getName());
+    Assert.assertEquals(key, ozoneKeyDetails.getName());
 
     // Verify that directories are created.
     FileStatus[] fileStatuses = ofs.listStatus(root);
-    assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir1.toString());
-    assertEquals(fileStatuses[1].getPath().toUri().getPath(), dir2.toString());
+    Assert.assertEquals(
+        fileStatuses[0].getPath().toUri().getPath(), dir1.toString());
+    Assert.assertEquals(
+        fileStatuses[1].getPath().toUri().getPath(), dir2.toString());
 
     fileStatuses = ofs.listStatus(dir1);
-    assertEquals(fileStatuses[0].getPath().toUri().getPath(), dir12.toString());
+    Assert.assertEquals(
+        fileStatuses[0].getPath().toUri().getPath(), dir12.toString());
     fileStatuses = ofs.listStatus(dir12);
-    assertEquals(fileStatuses.length, 0);
+    Assert.assertEquals(fileStatuses.length, 0);
     fileStatuses = ofs.listStatus(dir2);
-    assertEquals(fileStatuses.length, 0);
+    Assert.assertEquals(fileStatuses.length, 0);
   }
 
   /**
-   * OFS: Tests mkdir on a volume and bucket that doesn't exist.
+   * OFS: Test mkdir on a volume and bucket that doesn't exist.
    */
   @Test
   public void testMkdirNonExistentVolumeBucket() throws Exception {
@@ -294,20 +289,20 @@ public class TestRootedOzoneFileSystem {
     Iterator<? extends OzoneVolume> iterVol =
         objectStore.listVolumesByUser(null, volumeNameLocal, null);
     OzoneVolume ozoneVolume = iterVol.next();
-    assertNotNull(ozoneVolume);
-    assertEquals(volumeNameLocal, ozoneVolume.getName());
+    Assert.assertNotNull(ozoneVolume);
+    Assert.assertEquals(volumeNameLocal, ozoneVolume.getName());
 
     Iterator<? extends OzoneBucket> iterBuc =
         ozoneVolume.listBuckets("bucket-");
     OzoneBucket ozoneBucket = iterBuc.next();
-    assertNotNull(ozoneBucket);
-    assertEquals(bucketNameLocal, ozoneBucket.getName());
+    Assert.assertNotNull(ozoneBucket);
+    Assert.assertEquals(bucketNameLocal, ozoneBucket.getName());
 
     // TODO: Use listStatus to check volume and bucket creation in HDDS-2928.
   }
 
   /**
-   * OFS: Tests mkdir on a volume that doesn't exist.
+   * OFS: Test mkdir on a volume that doesn't exist.
    */
   @Test
   public void testMkdirNonExistentVolume() throws Exception {
@@ -319,17 +314,31 @@ public class TestRootedOzoneFileSystem {
     Iterator<? extends OzoneVolume> iterVol =
         objectStore.listVolumesByUser(null, volumeNameLocal, null);
     OzoneVolume ozoneVolume = iterVol.next();
-    assertNotNull(ozoneVolume);
-    assertEquals(volumeNameLocal, ozoneVolume.getName());
+    Assert.assertNotNull(ozoneVolume);
+    Assert.assertEquals(volumeNameLocal, ozoneVolume.getName());
 
     // TODO: Use listStatus to check volume and bucket creation in HDDS-2928.
   }
 
   /**
-   * Tests listStatus operation in a bucket.
+   * OFS: Test getFileStatus on root.
    */
   @Test
-  public void testListStatusOnRoot() throws Exception {
+  public void testGetFileStatusRoot() throws Exception {
+    Path root = new Path("/");
+    FileStatus fileStatus = fs.getFileStatus(root);
+    Assert.assertNotNull(fileStatus);
+    Assert.assertNull(fileStatus.getPath());
+    Assert.assertTrue(fileStatus.isDirectory());
+    Assert.assertEquals(
+        new FsPermission((short)00755), fileStatus.getPermission());
+  }
+
+  /**
+   * Test listStatus operation in a bucket.
+   */
+  @Test
+  public void testListStatusInBucket() throws Exception {
     Path root = new Path("/" + volumeName + "/" + bucketName);
     Path dir1 = new Path(root, "dir1");
     Path dir12 = new Path(dir1, "dir12");
@@ -341,14 +350,15 @@ public class TestRootedOzoneFileSystem {
     // exist) and dir2 only. dir12 is not an immediate child of root and
     // hence should not be listed.
     FileStatus[] fileStatuses = ofs.listStatus(root);
-    assertEquals("FileStatus should return only the immediate children", 2,
-        fileStatuses.length);
+    Assert.assertEquals(
+        "FileStatus should return only the immediate children",
+        2, fileStatuses.length);
 
     // Verify that dir12 is not included in the result of the listStatus on root
     String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
     String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
-    assertFalse(fileStatus1.equals(dir12.toString()));
-    assertFalse(fileStatus2.equals(dir12.toString()));
+    Assert.assertNotEquals(fileStatus1, dir12.toString());
+    Assert.assertNotEquals(fileStatus2, dir12.toString());
   }
 
   /**
@@ -366,12 +376,12 @@ public class TestRootedOzoneFileSystem {
     }
 
     FileStatus[] fileStatuses = ofs.listStatus(root);
-    assertEquals(
+    Assert.assertEquals(
         "Total directories listed do not match the existing directories",
         numDirs, fileStatuses.length);
 
     for (int i=0; i < numDirs; i++) {
-      assertTrue(paths.contains(fileStatuses[i].getPath().getName()));
+      Assert.assertTrue(paths.contains(fileStatuses[i].getPath().getName()));
     }
   }
 
@@ -401,16 +411,17 @@ public class TestRootedOzoneFileSystem {
     fs.mkdirs(dir2);
 
     FileStatus[] fileStatuses = ofs.listStatus(dir1);
-    assertEquals("FileStatus should return only the immediate children", 2,
-        fileStatuses.length);
+    Assert.assertEquals(
+        "FileStatus should return only the immediate children",
+        2, fileStatuses.length);
 
     // Verify that the two children of /dir1 returned by listStatus operation
     // are /dir1/dir11 and /dir1/dir12.
     String fileStatus1 = fileStatuses[0].getPath().toUri().getPath();
     String fileStatus2 = fileStatuses[1].getPath().toUri().getPath();
-    assertTrue(fileStatus1.equals(dir11.toString()) ||
+    Assert.assertTrue(fileStatus1.equals(dir11.toString()) ||
         fileStatus1.equals(dir12.toString()));
-    assertTrue(fileStatus2.equals(dir11.toString()) ||
+    Assert.assertTrue(fileStatus2.equals(dir11.toString()) ||
         fileStatus2.equals(dir12.toString()));
   }
 
@@ -427,15 +438,15 @@ public class TestRootedOzoneFileSystem {
     fs.mkdirs(target);
     fs.mkdirs(leafInsideInterimPath);
 
-    assertTrue(fs.rename(leafInsideInterimPath, leafInTarget));
+    Assert.assertTrue(fs.rename(leafInsideInterimPath, leafInTarget));
 
     // after rename listStatus for interimPath should succeed and
     // interimPath should have no children
     FileStatus[] statuses = fs.listStatus(interimPath);
-    assertNotNull("liststatus returns a null array", statuses);
-    assertEquals("Statuses array is not empty", 0, statuses.length);
+    Assert.assertNotNull("liststatus returns a null array", statuses);
+    Assert.assertEquals("Statuses array is not empty", 0, statuses.length);
     FileStatus fileStatus = fs.getFileStatus(interimPath);
-    assertEquals("FileStatus does not point to interimPath",
+    Assert.assertEquals("FileStatus does not point to interimPath",
         interimPath.getName(), fileStatus.getPath().getName());
   }
 
@@ -458,14 +469,15 @@ public class TestRootedOzoneFileSystem {
     Path leafInTargetInAnotherBucket = new Path(bucket2, "leaf");
     try {
       fs.rename(leafInsideInterimPath, leafInTargetInAnotherBucket);
-      fail("Should have thrown exception when renaming to a different bucket");
+      Assert.fail(
+          "Should have thrown exception when renaming to a different bucket");
     } catch (IOException ignored) {
       // Test passed. Exception thrown as expected.
     }
   }
 
   private OzoneKeyDetails getKey(Path keyPath, boolean isDirectory)
-      throws IOException, OzoneClientException {
+      throws IOException {
     String key = ofs.pathToKey(keyPath);
     if (isDirectory) {
       key = key + "/";
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index 36559d2..a1871fe 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -26,6 +26,7 @@ import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
+import com.google.common.base.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.crypto.key.KeyProvider;
@@ -80,8 +81,8 @@ public class BasicRootedOzoneClientAdapterImpl
       LoggerFactory.getLogger(BasicRootedOzoneClientAdapterImpl.class);
 
   private OzoneClient ozoneClient;
-  private ClientProtocol proxy;
   private ObjectStore objectStore;
+  private ClientProtocol proxy;
   private ReplicationType replicationType;
   private ReplicationFactor replicationFactor;
   private boolean securityEnabled;
@@ -196,6 +197,14 @@ public class BasicRootedOzoneClientAdapterImpl
    */
   private OzoneBucket getBucket(String volumeStr, String bucketStr,
       boolean createIfNotExist) throws IOException {
+    Preconditions.checkNotNull(volumeStr);
+    Preconditions.checkNotNull(bucketStr);
+    
+    if (bucketStr.isEmpty()) {
+      // throw FileNotFoundException in this case to make Hadoop common happy
+      throw new FileNotFoundException(
+          "getBucket: Invalid argument: given bucket string is empty.");
+    }
 
     OzoneBucket bucket;
     try {
@@ -348,11 +357,11 @@ public class BasicRootedOzoneClientAdapterImpl
     LOG.trace("creating dir for path: {}", pathStr);
     incrementCounter(Statistic.OBJECTS_CREATED);
     OFSPath ofsPath = new OFSPath(pathStr);
-    if (ofsPath.getVolumeName().length() == 0) {
+    if (ofsPath.getVolumeName().isEmpty()) {
       // Volume name unspecified, invalid param, return failure
       return false;
     }
-    if (ofsPath.getBucketName().length() == 0) {
+    if (ofsPath.getBucketName().isEmpty()) {
       // Create volume only
       objectStore.createVolume(ofsPath.getVolumeName());
       return true;
@@ -430,6 +439,12 @@ public class BasicRootedOzoneClientAdapterImpl
     incrementCounter(Statistic.OBJECTS_QUERY);
     OFSPath ofsPath = new OFSPath(path);
     String key = ofsPath.getKeyName();
+    // getFileStatus is called for root
+    if (ofsPath.getVolumeName().isEmpty() &&
+        ofsPath.getBucketName().isEmpty()) {
+      // Generate a FileStatusAdapter for root
+      return rootFileStatusAdapter();
+    }
     try {
       OzoneBucket bucket = getBucket(ofsPath, false);
       OzoneFileStatus status = bucket.getFileStatus(key);
@@ -648,4 +663,27 @@ public class BasicRootedOzoneClientAdapterImpl
         status.getPath()
     );
   }
+
+  /**
+   * Generate a FileStatusAdapter for OFS root.
+   * @return FileStatusAdapter for root.
+   */
+  private static FileStatusAdapter rootFileStatusAdapter() {
+    // Note that most fields are mimicked from HDFS FileStatus for root,
+    //  except modification time, permission, owner and group.
+    // TODO: Revisit the return value.
+    return new FileStatusAdapter(
+        0L,
+        null,
+        true,
+        (short)0,
+        0L,
+        0L,
+        0L,
+        (short)00755,
+        null,
+        null,
+        null
+    );
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org


Mime
View raw message