hadoop-ozone-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From x...@apache.org
Subject [hadoop-ozone] branch HDDS-2665-ofs updated: HDDS-2928. Implement ofs://: listStatus (#547)
Date Tue, 25 Feb 2020 05:59:53 GMT
This is an automated email from the ASF dual-hosted git repository.

xyao pushed a commit to branch HDDS-2665-ofs
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git


The following commit(s) were added to refs/heads/HDDS-2665-ofs by this push:
     new 7c35d76  HDDS-2928. Implement ofs://: listStatus (#547)
7c35d76 is described below

commit 7c35d76b2ec22dbd869ec7332e1c333567821e11
Author: Siyao Meng <50227127+smengcl@users.noreply.github.com>
AuthorDate: Mon Feb 24 21:59:44 2020 -0800

    HDDS-2928. Implement ofs://: listStatus (#547)
---
 .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 142 +++++++++++++++++--
 .../ozone/BasicRootedOzoneClientAdapterImpl.java   | 154 +++++++++++++++++----
 .../fs/ozone/BasicRootedOzoneFileSystem.java       |   6 +-
 .../java/org/apache/hadoop/fs/ozone/OFSPath.java   |  16 +++
 4 files changed, 280 insertions(+), 38 deletions(-)

diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
index 97f840f..033efdb 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestRootedOzoneFileSystem.java
@@ -43,11 +43,15 @@ import org.junit.Test;
 import org.junit.rules.Timeout;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Set;
 import java.util.TreeSet;
+import java.util.stream.Collectors;
 
 import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
 import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
 
 /**
@@ -68,6 +72,7 @@ public class TestRootedOzoneFileSystem {
   private String bucketName;
   // Store path commonly used by tests that test functionality within a bucket
   private Path testBucketPath;
+  private String rootPath;
 
   @Before
   public void init() throws Exception {
@@ -82,10 +87,11 @@ public class TestRootedOzoneFileSystem {
     OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(cluster);
     volumeName = bucket.getVolumeName();
     bucketName = bucket.getName();
-    String testBucketStr = "/" + volumeName + "/" + bucketName;
+    String testBucketStr =
+        OZONE_URI_DELIMITER + volumeName + OZONE_URI_DELIMITER + bucketName;
     testBucketPath = new Path(testBucketStr);
 
-    String rootPath = String.format("%s://%s/",
+    rootPath = String.format("%s://%s/",
         OzoneConsts.OZONE_OFS_URI_SCHEME, conf.get(OZONE_OM_ADDRESS_KEY));
 
     // Set the fs.defaultFS and start the filesystem
@@ -213,7 +219,7 @@ public class TestRootedOzoneFileSystem {
   /**
    * OFS: Helper function for tests. Return a volume name that doesn't exist.
    */
-  private String getRandomNonExistVolumeName() throws Exception {
+  private String getRandomNonExistVolumeName() throws IOException {
     final int numDigit = 5;
     long retriesLeft = Math.round(Math.pow(10, 5));
     String name = null;
@@ -230,7 +236,7 @@ public class TestRootedOzoneFileSystem {
       }
     }
     if (retriesLeft <= 0) {
-      throw new Exception(
+      Assert.fail(
           "Failed to generate random volume name that doesn't exist already.");
     }
     return name;
@@ -328,10 +334,10 @@ public class TestRootedOzoneFileSystem {
     Path root = new Path("/");
     FileStatus fileStatus = fs.getFileStatus(root);
     Assert.assertNotNull(fileStatus);
-    Assert.assertNull(fileStatus.getPath());
+    Assert.assertEquals(new Path(rootPath), fileStatus.getPath());
     Assert.assertTrue(fileStatus.isDirectory());
-    Assert.assertEquals(
-        new FsPermission((short)00755), fileStatus.getPermission());
+    Assert.assertEquals(FsPermission.getDirDefault(),
+        fileStatus.getPermission());
   }
 
   /**
@@ -465,7 +471,8 @@ public class TestRootedOzoneFileSystem {
     fs.mkdirs(leafInsideInterimPath);
 
     // Attempt to rename the key to a different bucket
-    Path bucket2 = new Path("/" + volumeName + "/" + bucketName + "test");
+    Path bucket2 = new Path(OZONE_URI_DELIMITER + volumeName +
+        OZONE_URI_DELIMITER + bucketName + "test");
     Path leafInTargetInAnotherBucket = new Path(bucket2, "leaf");
     try {
       fs.rename(leafInsideInterimPath, leafInTargetInAnotherBucket);
@@ -480,7 +487,7 @@ public class TestRootedOzoneFileSystem {
       throws IOException {
     String key = ofs.pathToKey(keyPath);
     if (isDirectory) {
-      key = key + "/";
+      key = key + OZONE_URI_DELIMITER;
     }
     OFSPath ofsPath = new OFSPath(key);
     String keyInBucket = ofsPath.getKeyName();
@@ -492,4 +499,121 @@ public class TestRootedOzoneFileSystem {
     GenericTestUtils.assertExceptionContains("KEY_NOT_FOUND", ex);
   }
 
+  /**
+   * Helper function for testListStatusRootAndVolume*.
+   */
+  private Path createRandomVolumeBucketWithDirs() throws IOException {
+    String volume1 = getRandomNonExistVolumeName();
+    String bucket1 = "bucket-" + RandomStringUtils.randomNumeric(5);
+    Path bucketPath1 = new Path(
+        OZONE_URI_DELIMITER + volume1 + OZONE_URI_DELIMITER + bucket1);
+
+    Path dir1 = new Path(bucketPath1, "dir1");
+    fs.mkdirs(dir1);  // Intentionally creating this "in-the-middle" dir key
+    Path subdir1 = new Path(dir1, "subdir1");
+    fs.mkdirs(subdir1);
+    Path dir2 = new Path(bucketPath1, "dir2");
+    fs.mkdirs(dir2);
+
+    return bucketPath1;
+  }
+
+  /**
+   * OFS: Test non-recursive listStatus on root and volume.
+   */
+  @Test
+  public void testListStatusRootAndVolumeNonRecursive() throws Exception {
+    Path bucketPath1 = createRandomVolumeBucketWithDirs();
+    createRandomVolumeBucketWithDirs();
+    // listStatus("/volume/bucket")
+    FileStatus[] fileStatusBucket = ofs.listStatus(bucketPath1);
+    Assert.assertEquals(2, fileStatusBucket.length);
+    // listStatus("/volume")
+    Path volume = new Path(
+        OZONE_URI_DELIMITER + new OFSPath(bucketPath1).getVolumeName());
+    FileStatus[] fileStatusVolume = ofs.listStatus(volume);
+    Assert.assertEquals(1, fileStatusVolume.length);
+    // listStatus("/")
+    Path root = new Path(OZONE_URI_DELIMITER);
+    FileStatus[] fileStatusRoot = ofs.listStatus(root);
+    Assert.assertEquals(2, fileStatusRoot.length);
+  }
+
+  /**
+   * Helper function to do FileSystem#listStatus recursively.
+   * Simulate what FsShell does, using DFS.
+   */
+  private void listStatusRecursiveHelper(Path curPath, List<FileStatus> result)
+      throws IOException {
+    FileStatus[] startList = ofs.listStatus(curPath);
+    for (FileStatus fileStatus : startList) {
+      result.add(fileStatus);
+      if (fileStatus.isDirectory()) {
+        Path nextPath = fileStatus.getPath();
+        listStatusRecursiveHelper(nextPath, result);
+      }
+    }
+  }
+
+  /**
+   * Helper function to call adapter impl for listStatus (with recursive).
+   */
+  private List<FileStatus> listStatusCallAdapterHelper(String pathStr)
+      throws IOException {
+    // FileSystem interface does not support recursive listStatus, use adapter
+    BasicRootedOzoneClientAdapterImpl adapter =
+        (BasicRootedOzoneClientAdapterImpl) ofs.getAdapter();
+    return adapter.listStatus(pathStr, true, "", 1000,
+        ofs.getUri(), ofs.getWorkingDirectory(), ofs.getUsername())
+        .stream().map(ofs::convertFileStatus).collect(Collectors.toList());
+  }
+
+  /**
+   * Helper function to compare recursive listStatus results from adapter
+   * and (simulated) FileSystem.
+   */
+  private void listStatusCheckHelper(Path path) throws IOException {
+    // Get recursive listStatus result directly from adapter impl
+    List<FileStatus> statusesFromAdapter =
+        listStatusCallAdapterHelper(path.toString());
+    // Get recursive listStatus result with FileSystem API by simulating FsShell
+    List<FileStatus> statusesFromFS = new ArrayList<>();
+    listStatusRecursiveHelper(path, statusesFromFS);
+    // Compare. The results would be in the same order due to assumptions:
+    // 1. They are both using DFS internally;
+    // 2. They both return ordered results.
+    Assert.assertEquals(statusesFromAdapter.size(), statusesFromFS.size());
+    final int n = statusesFromFS.size();
+    for (int i = 0; i < n; i++) {
+      FileStatus statusFromAdapter = statusesFromAdapter.get(i);
+      FileStatus statusFromFS = statusesFromFS.get(i);
+      Assert.assertEquals(statusFromAdapter.getPath(), statusFromFS.getPath());
+      Assert.assertEquals(statusFromAdapter.getLen(), statusFromFS.getLen());
+      Assert.assertEquals(statusFromAdapter.isDirectory(),
+          statusFromFS.isDirectory());
+      // TODO: When HDDS-3054 is in, uncomment the lines below.
+      //  As of now the modification time almost certainly won't match.
+//      Assert.assertEquals(statusFromAdapter.getModificationTime(),
+//          statusFromFS.getModificationTime());
+    }
+  }
+
+  /**
+   * OFS: Test recursive listStatus on root and volume.
+   */
+  @Test
+  public void testListStatusRootAndVolumeRecursive() throws Exception {
+    Path bucketPath1 = createRandomVolumeBucketWithDirs();
+    createRandomVolumeBucketWithDirs();
+    // listStatus("/volume/bucket")
+    listStatusCheckHelper(bucketPath1);
+    // listStatus("/volume")
+    Path volume = new Path(
+        OZONE_URI_DELIMITER + new OFSPath(bucketPath1).getVolumeName());
+    listStatusCheckHelper(volume);
+    // listStatus("/")
+    Path root = new Path(OZONE_URI_DELIMITER);
+    listStatusCheckHelper(root);
+  }
+
 }
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
index a1871fe..69a350c 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.crypto.key.KeyProvider;
 import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdds.client.ReplicationFactor;
 import org.apache.hadoop.hdds.client.ReplicationType;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -58,6 +59,7 @@ import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
     .BUCKET_ALREADY_EXISTS;
 import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes
@@ -434,16 +436,16 @@ public class BasicRootedOzoneClientAdapterImpl
   }
 
   public FileStatusAdapter getFileStatus(String path, URI uri,
-      Path qualifiedPath, String userName)
-      throws IOException {
+      Path qualifiedPath, String userName) throws IOException {
     incrementCounter(Statistic.OBJECTS_QUERY);
     OFSPath ofsPath = new OFSPath(path);
     String key = ofsPath.getKeyName();
-    // getFileStatus is called for root
-    if (ofsPath.getVolumeName().isEmpty() &&
-        ofsPath.getBucketName().isEmpty()) {
-      // Generate a FileStatusAdapter for root
-      return rootFileStatusAdapter();
+    if (ofsPath.isRoot()) {
+      return getFileStatusAdapterForRoot(uri);
+    }
+    if (ofsPath.isVolume()) {
+      OzoneVolume volume = objectStore.getVolume(ofsPath.getVolumeName());
+      return getFileStatusAdapterForVolume(volume, uri);
     }
     try {
       OzoneBucket bucket = getBucket(ofsPath, false);
@@ -452,11 +454,9 @@ public class BasicRootedOzoneClientAdapterImpl
       //  BasicRootedOzoneFileSystem#getFileStatus. No need to prepend here.
       makeQualified(status, uri, qualifiedPath, userName);
       return toFileStatusAdapter(status);
-
     } catch (OMException e) {
       if (e.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) {
-        throw new
-            FileNotFoundException(key + ": No such file or directory!");
+        throw new FileNotFoundException(key + ": No such file or directory!");
       }
       throw e;
     }
@@ -487,6 +487,59 @@ public class BasicRootedOzoneClientAdapterImpl
   }
 
   /**
+   * Helper for OFS listStatus on root.
+   */
+  private List<FileStatusAdapter> listStatusRoot(
+      boolean recursive, String startPath, long numEntries,
+      URI uri, Path workingDir, String username) throws IOException {
+
+    OFSPath ofsStartPath = new OFSPath(startPath);
+    // list volumes
+    Iterator<? extends OzoneVolume> iter = objectStore.listVolumesByUser(
+        username, ofsStartPath.getVolumeName(), null);
+    List<FileStatusAdapter> res = new ArrayList<>();
+    // TODO: Test continuation
+    while (iter.hasNext() && res.size() <= numEntries) {
+      OzoneVolume volume = iter.next();
+      res.add(getFileStatusAdapterForVolume(volume, uri));
+      if (recursive) {
+        String pathStrNextVolume = volume.getName();
+        // TODO: Check startPath
+        res.addAll(listStatus(pathStrNextVolume, recursive, startPath,
+            numEntries - res.size(), uri, workingDir, username));
+      }
+    }
+    return res;
+  }
+
+  /**
+   * Helper for OFS listStatus on a volume.
+   */
+  private List<FileStatusAdapter> listStatusVolume(String volumeStr,
+      boolean recursive, String startPath, long numEntries,
+      URI uri, Path workingDir, String username) throws IOException {
+
+    OFSPath ofsStartPath = new OFSPath(startPath);
+    // list buckets in the volume
+    OzoneVolume volume = objectStore.getVolume(volumeStr);
+    Iterator<? extends OzoneBucket> iter =
+        volume.listBuckets(null, ofsStartPath.getBucketName());
+    List<FileStatusAdapter> res = new ArrayList<>();
+    // TODO: Test continuation
+    while (iter.hasNext() && res.size() <= numEntries) {
+      OzoneBucket bucket = iter.next();
+      res.add(getFileStatusAdapterForBucket(bucket, uri, username));
+      if (recursive) {
+        String pathStrNext = volumeStr + OZONE_URI_DELIMITER + bucket.getName();
+        // TODO: Check startPath
+        res.addAll(listStatus(pathStrNext, recursive, startPath,
+            numEntries - res.size(), uri, workingDir, username));
+      }
+    }
+    return res;
+  }
+
+  /**
    * OFS listStatus implementation.
    *
    * @param pathStr Path for the listStatus to operate on.
@@ -511,9 +564,18 @@ public class BasicRootedOzoneClientAdapterImpl
 
     incrementCounter(Statistic.OBJECTS_LIST);
     OFSPath ofsPath = new OFSPath(pathStr);
-    // TODO: Subject to change in HDDS-2928.
-    String keyName = ofsPath.getKeyName();
+    if (ofsPath.isRoot()) {
+      return listStatusRoot(
+          recursive, startPath, numEntries, uri, workingDir, username);
+    }
     OFSPath ofsStartPath = new OFSPath(startPath);
+    if (ofsPath.isVolume()) {
+      String startBucket = ofsStartPath.getBucketName();
+      return listStatusVolume(ofsPath.getVolumeName(),
+          recursive, startBucket, numEntries, uri, workingDir, username);
+    }
+
+    String keyName = ofsPath.getKeyName();
     // Internally we need startKey to be passed into bucket.listStatus
     String startKey = ofsStartPath.getKeyName();
     try {
@@ -665,25 +727,65 @@ public class BasicRootedOzoneClientAdapterImpl
   }
 
   /**
+   * Generate a FileStatusAdapter for a volume.
+   * @param ozoneVolume OzoneVolume object
+   * @param uri Full URI to OFS root.
+   * @return FileStatusAdapter for a volume.
+   */
+  private static FileStatusAdapter getFileStatusAdapterForVolume(
+      OzoneVolume ozoneVolume, URI uri) {
+    String pathStr = uri.toString() +
+        OZONE_URI_DELIMITER + ozoneVolume.getName();
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("getFileStatusAdapterForVolume: ozoneVolume={}, pathStr={}",
+          ozoneVolume.getName(), pathStr);
+    }
+    Path path = new Path(pathStr);
+    return new FileStatusAdapter(0L, path, true, (short)0, 0L,
+        ozoneVolume.getCreationTime().getEpochSecond() * 1000, 0L,
+        FsPermission.getDirDefault().toShort(),
+        // TODO: Revisit owner and admin
+        ozoneVolume.getOwner(), ozoneVolume.getAdmin(), path
+    );
+  }
+
+  /**
+   * Generate a FileStatusAdapter for a bucket.
+   * @param ozoneBucket OzoneBucket object.
+   * @param uri Full URI to OFS root.
+   * @return FileStatusAdapter for a bucket.
+   */
+  private static FileStatusAdapter getFileStatusAdapterForBucket(
+      OzoneBucket ozoneBucket, URI uri, String username) {
+    String pathStr = uri.toString() +
+        OZONE_URI_DELIMITER + ozoneBucket.getVolumeName() +
+        OZONE_URI_DELIMITER + ozoneBucket.getName();
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("getFileStatusAdapterForBucket: ozoneBucket={}, pathStr={}, "
+              + "username={}", ozoneBucket.getVolumeName() + OZONE_URI_DELIMITER
+              + ozoneBucket.getName(), pathStr, username);
+    }
+    Path path = new Path(pathStr);
+    return new FileStatusAdapter(0L, path, true, (short)0, 0L,
+        ozoneBucket.getCreationTime().getEpochSecond() * 1000, 0L,
+        FsPermission.getDirDefault().toShort(),  // TODO: derive from ACLs later
+        // TODO: revisit owner and group
+        username, username, path);
+  }
+
+  /**
    * Generate a FileStatusAdapter for OFS root.
+   * @param uri Full URI to OFS root.
    * @return FileStatusAdapter for root.
    */
-  private static FileStatusAdapter rootFileStatusAdapter() {
+  private static FileStatusAdapter getFileStatusAdapterForRoot(URI uri) {
     // Note that most fields are mimicked from HDFS FileStatus for root,
     //  except modification time, permission, owner and group.
-    // TODO: Revisit the return value.
-    return new FileStatusAdapter(
-        0L,
-        null,
-        true,
-        (short)0,
-        0L,
-        0L,
-        0L,
-        (short)00755,
-        null,
-        null,
-        null
+    Path path = new Path(uri.toString() + OZONE_URI_DELIMITER);
+    return new FileStatusAdapter(0L, path, true, (short)0, 0L,
+        System.currentTimeMillis(), 0L,
+        FsPermission.getDirDefault().toShort(),
+        null, null, null
     );
   }
 }
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index 35256c8..9274ffc 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.fs.ozone;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -763,9 +764,8 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
     return true;
   }
 
-  private FileStatus convertFileStatus(
-      FileStatusAdapter fileStatusAdapter) {
-
+  @VisibleForTesting
+  FileStatus convertFileStatus(FileStatusAdapter fileStatusAdapter) {
     Path symLink = null;
     try {
       fileStatusAdapter.getSymlink();
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
index 748d0b4..02b8e02 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OFSPath.java
@@ -142,4 +142,20 @@ class OFSPath {
   public boolean isInSameBucketAs(OFSPath p2) {
     return isInSameBucketAsInternal(this, p2);
   }
+
+  /**
+   * If both volume and bucket names are empty, the given path is root.
+   * i.e. /
+   */
+  public boolean isRoot() {
+    return this.getVolumeName().isEmpty() && this.getBucketName().isEmpty();
+  }
+
+  /**
+   * If bucket name is empty but volume name is not, the given path is volume.
+   * e.g. /volume1
+   */
+  public boolean isVolume() {
+    return this.getBucketName().isEmpty() && !this.getVolumeName().isEmpty();
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org


Mime
View raw message