hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cdoug...@apache.org
Subject [10/20] hadoop git commit: HADOOP-14017. User friendly name for ADLS user and group. Contributed by Vishwajeet Dusane
Date Thu, 09 Mar 2017 23:27:22 GMT
HADOOP-14017. User friendly name for ADLS user and group. Contributed by Vishwajeet Dusane

(cherry picked from commit 924def78544a64449785f305cb6984c3559aea4d)
(cherry picked from commit acf20c8318270eaa8fdf6259ec7b07344e3bd7e5)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/518705b8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/518705b8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/518705b8

Branch: refs/heads/branch-2.8
Commit: 518705b8e3b6435894def73dc37104cdba38fe00
Parents: f0df481
Author: Mingliang Liu <liuml07@apache.org>
Authored: Tue Feb 21 13:44:42 2017 -0800
Committer: Chris Douglas <cdouglas@apache.org>
Committed: Thu Mar 9 15:21:33 2017 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/fs/adl/AdlConfKeys.java   |  4 +++
 .../org/apache/hadoop/fs/adl/AdlFileSystem.java | 23 +++++++++++---
 .../src/site/markdown/index.md                  | 26 +++++++++++++++
 .../fs/adl/TestValidateConfiguration.java       |  9 ++++++
 .../apache/hadoop/fs/adl/live/TestMetadata.java | 33 ++++++++++++++++++++
 5 files changed, 91 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/518705b8/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
index 21120df..7d31103 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlConfKeys.java
@@ -87,6 +87,10 @@ public final class AdlConfKeys {
       "adl.feature.support.acl.bit";
   static final boolean ADL_SUPPORT_ACL_BIT_IN_FSPERMISSION_DEFAULT = true;
 
+  static final String ADL_ENABLEUPN_FOR_OWNERGROUP_KEY =
+      "adl.feature.ownerandgroup.enableupn";
+  static final boolean ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT = false;
+
   private AdlConfKeys() {
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/518705b8/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index fb0feda..e0e273e 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -32,6 +32,7 @@ import com.microsoft.azure.datalake.store.DirectoryEntry;
 import com.microsoft.azure.datalake.store.DirectoryEntryType;
 import com.microsoft.azure.datalake.store.IfExists;
 import com.microsoft.azure.datalake.store.LatencyTracker;
+import com.microsoft.azure.datalake.store.UserGroupRepresentation;
 import com.microsoft.azure.datalake.store.oauth2.AccessTokenProvider;
 import com.microsoft.azure.datalake.store.oauth2.ClientCredsTokenProvider;
 import com.microsoft.azure.datalake.store.oauth2.RefreshTokenBasedTokenProvider;
@@ -80,6 +81,8 @@ public class AdlFileSystem extends FileSystem {
   private ADLStoreClient adlClient;
   private Path workingDirectory;
   private boolean aclBitStatus;
+  private UserGroupRepresentation oidOrUpn;
+
 
   // retained for tests
   private AccessTokenProvider tokenProvider;
@@ -181,6 +184,11 @@ public class AdlFileSystem extends FileSystem {
     if (!trackLatency) {
       LatencyTracker.disable();
     }
+
+    boolean enableUPN = conf.getBoolean(ADL_ENABLEUPN_FOR_OWNERGROUP_KEY,
+        ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT);
+    oidOrUpn = enableUPN ? UserGroupRepresentation.UPN :
+        UserGroupRepresentation.OID;
   }
 
   /**
@@ -439,7 +447,8 @@ public class AdlFileSystem extends FileSystem {
   @Override
   public FileStatus getFileStatus(final Path f) throws IOException {
     statistics.incrementReadOps(1);
-    DirectoryEntry entry = adlClient.getDirectoryEntry(toRelativeFilePath(f));
+    DirectoryEntry entry =
+        adlClient.getDirectoryEntry(toRelativeFilePath(f), oidOrUpn);
     return toFileStatus(entry, f);
   }
 
@@ -456,7 +465,7 @@ public class AdlFileSystem extends FileSystem {
   public FileStatus[] listStatus(final Path f) throws IOException {
     statistics.incrementReadOps(1);
     List<DirectoryEntry> entries =
-        adlClient.enumerateDirectory(toRelativeFilePath(f));
+        adlClient.enumerateDirectory(toRelativeFilePath(f), oidOrUpn);
     return toFileStatuses(entries, f);
   }
 
@@ -749,8 +758,8 @@ public class AdlFileSystem extends FileSystem {
   @Override
   public AclStatus getAclStatus(final Path path) throws IOException {
     statistics.incrementReadOps(1);
-    com.microsoft.azure.datalake.store.acl.AclStatus adlStatus = adlClient
-        .getAclStatus(toRelativeFilePath(path));
+    com.microsoft.azure.datalake.store.acl.AclStatus adlStatus =
+        adlClient.getAclStatus(toRelativeFilePath(path), oidOrUpn);
     AclStatus.Builder aclStatusBuilder = new AclStatus.Builder();
     aclStatusBuilder.owner(adlStatus.owner);
     aclStatusBuilder.group(adlStatus.group);
@@ -963,4 +972,10 @@ public class AdlFileSystem extends FileSystem {
     }
     return new String(passchars);
   }
+
+  @VisibleForTesting
+  public void setUserGroupRepresentationAsUPN(boolean enableUPN) {
+    oidOrUpn = enableUPN ? UserGroupRepresentation.UPN :
+        UserGroupRepresentation.OID;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/518705b8/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
index 5037db6..6d9e173 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
+++ b/hadoop-tools/hadoop-azure-datalake/src/site/markdown/index.md
@@ -26,6 +26,7 @@
         * [Protecting the Credentials with Credential Providers](#Credential_Provider)
     * [Enabling ADL Filesystem](#Enabling_ADL)
     * [Accessing adl URLs](#Accessing_adl_URLs)
+    * [User/Group Representation](#OIDtoUPNConfiguration)
 * [Testing the hadoop-azure Module](#Testing_the_hadoop-azure_Module)
 
 ## <a name="Introduction" />Introduction
@@ -42,6 +43,8 @@ The jar file is named azure-datalake-store.jar.
 * Can act as a source of data in a MapReduce job, or a sink.
 * Tested on both Linux and Windows.
 * Tested for scale.
+* API setOwner/setAcl/removeAclEntries/modifyAclEntries accepts UPN or OID
+  (Object ID) as user and group name.
 
 ## <a name="Limitations" />Limitations
 Partial or no support for the following operations :
@@ -221,6 +224,29 @@ commands demonstrate access to a storage account named `youraccount`.
 
     > hadoop fs -cat adl://yourcontainer.azuredatalakestore.net/testDir/testFile
     test file content
+
+### <a name="OIDtoUPNConfiguration" />User/Group Representation
+The hadoop-azure-datalake module provides support for configuring how
+User/Group information is represented during
+getFileStatus/listStatus/getAclStatus.
+
+Add the following properties to your core-site.xml
+
+        <property>
+          <name>adl.feature.ownerandgroup.enableupn</name>
+          <value>true</value>
+          <description>
+            When true : User and Group in FileStatus/AclStatus response is
+            represented as user friendly name as per Azure AD profile.
+
+            When false (default) : User and Group in FileStatus/AclStatus
+            response is represented by the unique identifier from Azure AD
+            profile (Object ID as GUID).
+
+            For performance optimization, Recommended default value.
+          </description>
+        </property>
+
 ## <a name="Testing_the_hadoop-azure_Module" />Testing the azure-datalake-store Module
 The hadoop-azure module includes a full suite of unit tests. Most of the tests will run without
additional configuration by running mvn test. This includes tests against mocked storage,
which is an in-memory emulation of Azure Data Lake Storage.
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/518705b8/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
index e3025b2..4cabaa3 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestValidateConfiguration.java
@@ -27,6 +27,10 @@ import static org.apache.hadoop.fs.adl.AdlConfKeys
 import static org.apache.hadoop.fs.adl.AdlConfKeys
     .ADL_DEBUG_SET_LOCAL_USER_AS_OWNER_DEFAULT;
 import static org.apache.hadoop.fs.adl.AdlConfKeys
+    .ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
+    .ADL_ENABLEUPN_FOR_OWNERGROUP_KEY;
+import static org.apache.hadoop.fs.adl.AdlConfKeys
     .ADL_EXPERIMENT_POSITIONAL_READ_DEFAULT;
 import static org.apache.hadoop.fs.adl.AdlConfKeys
     .ADL_EXPERIMENT_POSITIONAL_READ_KEY;
@@ -99,5 +103,10 @@ public class TestValidateConfiguration {
     Assert.assertEquals(false, ADL_DEBUG_SET_LOCAL_USER_AS_OWNER_DEFAULT);
     Assert.assertEquals(4 * 1024 * 1024, DEFAULT_READ_AHEAD_BUFFER_SIZE);
     Assert.assertEquals(4 * 1024 * 1024, DEFAULT_WRITE_AHEAD_BUFFER_SIZE);
+
+    Assert.assertEquals("adl.feature.ownerandgroup.enableupn",
+        ADL_ENABLEUPN_FOR_OWNERGROUP_KEY);
+    Assert.assertEquals(false,
+        ADL_ENABLEUPN_FOR_OWNERGROUP_DEFAULT);
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/518705b8/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestMetadata.java
b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestMetadata.java
index 3b9e7da..dbcaa39 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestMetadata.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestMetadata.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.adl.AdlFileSystem;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Assume;
@@ -32,6 +33,8 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.util.UUID;
 
+import static org.junit.Assert.fail;
+
 /**
  * This class is responsible for testing ContentSummary, ListStatus on
  * file/folder.
@@ -107,5 +110,35 @@ public class TestMetadata {
         .assertEquals(path.makeQualified(fs.getUri(), fs.getWorkingDirectory()),
             statuses[0].getPath());
   }
+
+  @Test
+  public void testUserRepresentationConfiguration() throws IOException {
+    // Validating actual user/group OID or friendly name is outside scope of
+    // this test.
+    Path path = new Path(parent, "a.txt");
+    AdlFileSystem fs = (AdlFileSystem) adlStore;
+
+    // When set to true, User/Group information should be user friendly name.
+    // That is non GUID value.
+    fs.setUserGroupRepresentationAsUPN(false);
+    fs.createNewFile(path);
+    Assert.assertTrue(fs.isFile(path));
+    FileStatus fileStatus = fs.getFileStatus(path);
+    UUID.fromString(fileStatus.getGroup());
+    UUID.fromString(fileStatus.getOwner());
+
+    // When set to false, User/Group information should be AAD represented
+    // unique OID. That is GUID value.
+    // Majority of the cases, user friendly name would not be GUID value.
+    fs.setUserGroupRepresentationAsUPN(true);
+    fileStatus = fs.getFileStatus(path);
+    try {
+      UUID.fromString(fileStatus.getGroup());
+      UUID.fromString(fileStatus.getOwner());
+      fail("Expected user friendly name to be non guid value.");
+    } catch (IllegalArgumentException e) {
+      // expected to fail since
+    }
+  }
 }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message