hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject [32/51] [abbrv] hadoop git commit: HADOOP-14730. Support protobuf FileStatus in AdlFileSystem.
Date Wed, 09 Aug 2017 17:36:52 GMT
HADOOP-14730. Support protobuf FileStatus in AdlFileSystem.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/55a181f8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/55a181f8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/55a181f8

Branch: refs/heads/HADOOP-13345
Commit: 55a181f845adcdcc9b008e9906ade1544fc220e4
Parents: 8d3fd81
Author: Chris Douglas <cdouglas@apache.org>
Authored: Mon Aug 7 21:31:28 2017 -0700
Committer: Chris Douglas <cdouglas@apache.org>
Committed: Mon Aug 7 21:31:28 2017 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/fs/adl/AdlFileStatus.java | 69 ++++++++++++++++++++
 .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 27 ++------
 .../apache/hadoop/fs/adl/TestGetFileStatus.java | 57 ++++++++--------
 .../apache/hadoop/fs/adl/TestListStatus.java    |  8 ++-
 4 files changed, 105 insertions(+), 56 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/55a181f8/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileStatus.java
b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileStatus.java
new file mode 100644
index 0000000..70c005d
--- /dev/null
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileStatus.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.hadoop.fs.adl;
+
+import com.microsoft.azure.datalake.store.DirectoryEntry;
+import com.microsoft.azure.datalake.store.DirectoryEntryType;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE;
+import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_REPLICATION_FACTOR;
+
+/**
+ * Shim class supporting linking against 2.x clients.
+ */
+class AdlFileStatus extends FileStatus {
+
+  private static final long serialVersionUID = 0x01fcbe5e;
+
+  private boolean hasAcl = false;
+
+  AdlFileStatus(DirectoryEntry entry, Path path, boolean hasAcl) {
+    this(entry, path, entry.user, entry.group, hasAcl);
+  }
+
+  AdlFileStatus(DirectoryEntry entry, Path path,
+                String owner, String group, boolean hasAcl) {
+    super(entry.length, DirectoryEntryType.DIRECTORY == entry.type,
+        ADL_REPLICATION_FACTOR, ADL_BLOCK_SIZE,
+        entry.lastModifiedTime.getTime(), entry.lastAccessTime.getTime(),
+        new AdlPermission(hasAcl, Short.parseShort(entry.permission, 8)),
+        owner, group, null, path);
+    this.hasAcl = hasAcl;
+  }
+
+  @Override
+  public boolean hasAcl() {
+    return hasAcl;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    // satisfy findbugs
+    return super.equals(o);
+  }
+
+  @Override
+  public int hashCode() {
+    // satisfy findbugs
+    return super.hashCode();
+  }
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/55a181f8/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index 0de538e..76ce43e 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -29,7 +29,6 @@ import com.google.common.annotations.VisibleForTesting;
 import com.microsoft.azure.datalake.store.ADLStoreClient;
 import com.microsoft.azure.datalake.store.ADLStoreOptions;
 import com.microsoft.azure.datalake.store.DirectoryEntry;
-import com.microsoft.azure.datalake.store.DirectoryEntryType;
 import com.microsoft.azure.datalake.store.IfExists;
 import com.microsoft.azure.datalake.store.LatencyTracker;
 import com.microsoft.azure.datalake.store.UserGroupRepresentation;
@@ -606,30 +605,12 @@ public class AdlFileSystem extends FileSystem {
   }
 
   private FileStatus toFileStatus(final DirectoryEntry entry, final Path f) {
-    boolean isDirectory = entry.type == DirectoryEntryType.DIRECTORY;
-    long lastModificationData = entry.lastModifiedTime.getTime();
-    long lastAccessTime = entry.lastAccessTime.getTime();
-    // set aclBit from ADLS backend response if
-    // ADL_SUPPORT_ACL_BIT_IN_FSPERMISSION is true.
-    final boolean aclBit = aclBitStatus ? entry.aclBit : false;
-
-    FsPermission permission = new AdlPermission(aclBit,
-        Short.valueOf(entry.permission, 8));
-    String user = entry.user;
-    String group = entry.group;
-
-    FileStatus status;
+    Path p = makeQualified(f);
+    boolean aclBit = aclBitStatus ? entry.aclBit : false;
     if (overrideOwner) {
-      status = new FileStatus(entry.length, isDirectory, ADL_REPLICATION_FACTOR,
-          ADL_BLOCK_SIZE, lastModificationData, lastAccessTime, permission,
-          userName, "hdfs", this.makeQualified(f));
-    } else {
-      status = new FileStatus(entry.length, isDirectory, ADL_REPLICATION_FACTOR,
-          ADL_BLOCK_SIZE, lastModificationData, lastAccessTime, permission,
-          user, group, this.makeQualified(f));
+      return new AdlFileStatus(entry, p, userName, "hdfs", aclBit);
     }
-
-    return status;
+    return new AdlFileStatus(entry, p, aclBit);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/55a181f8/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
index 0ea4b86..d9e22db 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
@@ -42,8 +42,8 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys.ADL_BLOCK_SIZE;
  * org.apache.hadoop.fs.adl.live testing package.
  */
 public class TestGetFileStatus extends AdlMockWebServer {
-  private static final Logger LOG = LoggerFactory
-      .getLogger(TestGetFileStatus.class);
+  private static final Logger LOG =
+      LoggerFactory.getLogger(TestGetFileStatus.class);
 
   @Test
   public void getFileStatusReturnsAsExpected()
@@ -72,33 +72,30 @@ public class TestGetFileStatus extends AdlMockWebServer {
         fileStatus.isErasureCoded());
   }
 
-    @Test
-    public void getFileStatusAclBit()
-            throws URISyntaxException, IOException {
-        // With ACLBIT set to true
-        getMockServer().enqueue(new MockResponse().setResponseCode(200)
-                .setBody(TestADLResponseData.getGetFileStatusJSONResponse(true)));
-        long startTime = Time.monotonicNow();
-        FileStatus fileStatus = getMockAdlFileSystem()
-                .getFileStatus(new Path("/test1/test2"));
-        long endTime = Time.monotonicNow();
-        LOG.debug("Time : " + (endTime - startTime));
-        Assert.assertTrue(fileStatus.isFile());
-        Assert.assertEquals(true, fileStatus.getPermission().getAclBit());
-        Assert.assertEquals(fileStatus.hasAcl(),
-            fileStatus.getPermission().getAclBit());
+  @Test
+  public void getFileStatusAclBit() throws URISyntaxException, IOException {
+    // With ACLBIT set to true
+    getMockServer().enqueue(new MockResponse().setResponseCode(200)
+            .setBody(TestADLResponseData.getGetFileStatusJSONResponse(true)));
+    long startTime = Time.monotonicNow();
+    FileStatus fileStatus = getMockAdlFileSystem()
+            .getFileStatus(new Path("/test1/test2"));
+    long endTime = Time.monotonicNow();
+    LOG.debug("Time : " + (endTime - startTime));
+    Assert.assertTrue(fileStatus.isFile());
+    Assert.assertTrue(fileStatus.hasAcl());
+    Assert.assertTrue(fileStatus.getPermission().getAclBit());
 
-        // With ACLBIT set to false
-        getMockServer().enqueue(new MockResponse().setResponseCode(200)
-                .setBody(TestADLResponseData.getGetFileStatusJSONResponse(false)));
-        startTime = Time.monotonicNow();
-        fileStatus = getMockAdlFileSystem()
-                .getFileStatus(new Path("/test1/test2"));
-        endTime = Time.monotonicNow();
-        LOG.debug("Time : " + (endTime - startTime));
-        Assert.assertTrue(fileStatus.isFile());
-        Assert.assertEquals(false, fileStatus.getPermission().getAclBit());
-        Assert.assertEquals(fileStatus.hasAcl(),
-            fileStatus.getPermission().getAclBit());
-    }
+    // With ACLBIT set to false
+    getMockServer().enqueue(new MockResponse().setResponseCode(200)
+            .setBody(TestADLResponseData.getGetFileStatusJSONResponse(false)));
+    startTime = Time.monotonicNow();
+    fileStatus = getMockAdlFileSystem()
+            .getFileStatus(new Path("/test1/test2"));
+    endTime = Time.monotonicNow();
+    LOG.debug("Time : " + (endTime - startTime));
+    Assert.assertTrue(fileStatus.isFile());
+    Assert.assertFalse(fileStatus.hasAcl());
+    Assert.assertFalse(fileStatus.getPermission().getAclBit());
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/55a181f8/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java
b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java
index dac8886..db32476 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestListStatus.java
@@ -102,7 +102,7 @@ public class TestListStatus extends AdlMockWebServer {
   }
 
   @Test
-  public void listStatusAclBit()
+  public void listStatusAcl()
           throws URISyntaxException, IOException {
     // With ACLBIT set to true
     getMockServer().enqueue(new MockResponse().setResponseCode(200)
@@ -115,7 +115,8 @@ public class TestListStatus extends AdlMockWebServer {
     LOG.debug("Time : " + (endTime - startTime));
     for (int i = 0; i < ls.length; i++) {
       Assert.assertTrue(ls[i].isDirectory());
-      Assert.assertEquals(true, ls[i].getPermission().getAclBit());
+      Assert.assertTrue(ls[i].hasAcl());
+      Assert.assertTrue(ls[i].getPermission().getAclBit());
     }
 
     // With ACLBIT set to false
@@ -129,7 +130,8 @@ public class TestListStatus extends AdlMockWebServer {
     LOG.debug("Time : " + (endTime - startTime));
     for (int i = 0; i < ls.length; i++) {
       Assert.assertTrue(ls[i].isDirectory());
-      Assert.assertEquals(false, ls[i].getPermission().getAclBit());
+      Assert.assertFalse(ls[i].hasAcl());
+      Assert.assertFalse(ls[i].getPermission().getAclBit());
     }
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message