hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jgho...@apache.org
Subject hadoop git commit: HDFS-8542. WebHDFS getHomeDirectory behavior does not match specification. Contributed by Kanaka Kumar Avvaru.
Date Mon, 22 Jun 2015 23:43:30 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 3bf44b862 -> f6157a4a0


HDFS-8542. WebHDFS getHomeDirectory behavior does not match specification. Contributed by
 Kanaka Kumar Avvaru.

(cherry picked from commit fac4e04dd359a7ff31f286d664fb06f019ec0b58)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6157a4a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6157a4a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6157a4a

Branch: refs/heads/branch-2
Commit: f6157a4a0848398e2ba812f1d5bfe51da0d7f6b7
Parents: 3bf44b8
Author: Jakob Homan <jghoman@gmail.com>
Authored: Mon Jun 22 16:30:45 2015 -0700
Committer: Jakob Homan <jghoman@gmail.com>
Committed: Mon Jun 22 16:38:07 2015 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hdfs/web/JsonUtilClient.java  | 10 ++++
 .../hadoop/hdfs/web/WebHdfsFileSystem.java      | 33 +++++++++--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 +
 .../web/resources/NamenodeWebHdfsMethods.java   | 12 ++--
 .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 59 ++++++++++++++++++++
 .../hdfs/web/TestWebHdfsFileSystemContract.java |  2 +-
 6 files changed, 108 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6157a4a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
index ca94840..e025e31 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
@@ -389,6 +389,16 @@ class JsonUtilClient {
     return aclStatusBuilder.build();
   }
 
+  static String getPath(final Map<?, ?> json)
+      throws IOException {
+    if (json == null) {
+      return null;
+    }
+
+    String path = (String) json.get("Path");
+    return path;
+  }
+
   static byte[] getXAttr(final Map<?, ?> json, final String name)
       throws IOException {
     if (json == null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6157a4a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index ccf7bcf..fb5f029 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -110,6 +110,7 @@ public class WebHdfsFileSystem extends FileSystem
   protected Text tokenServiceName;
   private RetryPolicy retryPolicy = null;
   private Path workingDir;
+  private Path cachedHomeDirectory;
   private InetSocketAddress nnAddrs[];
   private int currentNNAddrIndex;
   private boolean disallowFallbackToInsecureCluster;
@@ -192,7 +193,7 @@ public class WebHdfsFileSystem extends FileSystem
               failoverSleepMaxMillis);
     }
 
-    this.workingDir = getHomeDirectory();
+    this.workingDir = makeQualified(new Path(getHomeDirectoryString(ugi)));
     this.canRefreshDelegationToken = UserGroupInformation.isSecurityEnabled();
     this.disallowFallbackToInsecureCluster = !conf.getBoolean(
         CommonConfigurationKeys.IPC_CLIENT_FALLBACK_TO_SIMPLE_AUTH_ALLOWED_KEY,
@@ -276,14 +277,35 @@ public class WebHdfsFileSystem extends FileSystem
     return NetUtils.getCanonicalUri(uri, getDefaultPort());
   }
 
-  /** @return the home directory. */
+  /** @return the home directory */
+  @Deprecated
   public static String getHomeDirectoryString(final UserGroupInformation ugi) {
     return "/user/" + ugi.getShortUserName();
   }
 
   @Override
   public Path getHomeDirectory() {
-    return makeQualified(new Path(getHomeDirectoryString(ugi)));
+    if (cachedHomeDirectory == null) {
+      final HttpOpParam.Op op = GetOpParam.Op.GETHOMEDIRECTORY;
+      try {
+        String pathFromDelegatedFS = new FsPathResponseRunner<String>(op, null,
+            new UserParam(ugi)) {
+          @Override
+          String decodeResponse(Map<?, ?> json) throws IOException {
+            return JsonUtilClient.getPath(json);
+          }
+        }   .run();
+
+        cachedHomeDirectory = new Path(pathFromDelegatedFS).makeQualified(
+            this.getUri(), null);
+
+      } catch (IOException e) {
+        LOG.error("Unable to get HomeDirectory from original File System", e);
+        cachedHomeDirectory = new Path("/user/" + ugi.getShortUserName())
+            .makeQualified(this.getUri(), null);
+      }
+    }
+    return cachedHomeDirectory;
   }
 
   @Override
@@ -293,12 +315,13 @@ public class WebHdfsFileSystem extends FileSystem
 
   @Override
   public synchronized void setWorkingDirectory(final Path dir) {
-    String result = makeAbsolute(dir).toUri().getPath();
+    Path absolutePath = makeAbsolute(dir);
+    String result = absolutePath.toUri().getPath();
     if (!DFSUtilClient.isValidName(result)) {
       throw new IllegalArgumentException("Invalid DFS directory name " +
                                          result);
     }
-    workingDir = makeAbsolute(dir);
+    workingDir = absolutePath;
   }
 
   private Path makeAbsolute(Path f) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6157a4a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b85015a..14f4f9a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -604,6 +604,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-4366. Block Replication Policy Implementation May Skip Higher-Priority
     Blocks for Lower-Priority Blocks (Derek Dagit via kihwal)
 
+    HDFS-8542. WebHDFS getHomeDirectory behavior does not match specification.
+    (Kanaka Kumar Avvaru via jghoman)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6157a4a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index d33721c..6e880f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -53,6 +53,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Options;
 import org.apache.hadoop.fs.XAttr;
 import org.apache.hadoop.fs.permission.AclStatus;
@@ -826,6 +827,8 @@ public class NamenodeWebHdfsMethods {
       final TokenServiceParam tokenService
       ) throws IOException, URISyntaxException {
     final NameNode namenode = (NameNode)context.getAttribute("name.node");
+    final Configuration conf = (Configuration) context
+        .getAttribute(JspHelper.CURRENT_CONF);
     final NamenodeProtocols np = getRPCServer(namenode);
 
     switch(op.getValue()) {
@@ -892,11 +895,10 @@ public class NamenodeWebHdfsMethods {
       final String js = JsonUtil.toJsonString(token);
       return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
     }
-    case GETHOMEDIRECTORY:
-    {
-      final String js = JsonUtil.toJsonString(
-          org.apache.hadoop.fs.Path.class.getSimpleName(),
-          WebHdfsFileSystem.getHomeDirectoryString(ugi));
+    case GETHOMEDIRECTORY: {
+      final String js = JsonUtil.toJsonString("Path",
+          FileSystem.get(conf != null ? conf : new Configuration())
+              .getHomeDirectory().toUri().getPath());
       return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
     }
     case GETACLSTATUS: {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6157a4a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index 9c06068..f7fc566 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -18,12 +18,15 @@
 
 package org.apache.hadoop.hdfs.web;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
 import static org.junit.Assert.fail;
 
 import java.io.IOException;
 import java.io.OutputStream;
 import java.net.HttpURLConnection;
 import java.net.InetSocketAddress;
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.security.PrivilegedExceptionAction;
@@ -570,4 +573,60 @@ public class TestWebHDFS {
       }
     }
   }
+
+  @Test(timeout = 30000)
+  public void testGetHomeDirectory() throws Exception {
+
+    MiniDFSCluster cluster = null;
+    try {
+      Configuration conf = new Configuration();
+      cluster = new MiniDFSCluster.Builder(conf).build();
+      cluster.waitActive();
+      DistributedFileSystem hdfs = cluster.getFileSystem();
+
+      final URI uri = new URI(WebHdfsConstants.WEBHDFS_SCHEME + "://"
+          + cluster.getHttpUri(0).replace("http://", ""));
+      final Configuration confTemp = new Configuration();
+
+      {
+        WebHdfsFileSystem webhdfs = (WebHdfsFileSystem) FileSystem.get(uri,
+            confTemp);
+
+        assertEquals(hdfs.getHomeDirectory().toUri().getPath(), webhdfs
+            .getHomeDirectory().toUri().getPath());
+
+        webhdfs.close();
+      }
+
+      {
+        WebHdfsFileSystem webhdfs = createWebHDFSAsTestUser(confTemp, uri,
+            "XXX");
+
+        assertNotEquals(hdfs.getHomeDirectory().toUri().getPath(), webhdfs
+            .getHomeDirectory().toUri().getPath());
+
+        webhdfs.close();
+      }
+
+    } finally {
+      if (cluster != null)
+        cluster.shutdown();
+    }
+  }
+
+  private WebHdfsFileSystem createWebHDFSAsTestUser(final Configuration conf,
+      final URI uri, final String userName) throws Exception {
+
+    final UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
+        userName, new String[] { "supergroup" });
+
+    return ugi.doAs(new PrivilegedExceptionAction<WebHdfsFileSystem>() {
+      @Override
+      public WebHdfsFileSystem run() throws IOException {
+        WebHdfsFileSystem webhdfs = (WebHdfsFileSystem) FileSystem.get(uri,
+            conf);
+        return webhdfs;
+      }
+    });
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6157a4a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
index ed2532f..c44c33e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
@@ -401,7 +401,7 @@ public class TestWebHdfsFileSystemContract extends FileSystemContractBaseTest
{
       final HttpURLConnection conn = (HttpURLConnection) url.openConnection();
       final Map<?, ?> m = WebHdfsTestUtil.connectAndGetJson(
           conn, HttpServletResponse.SC_OK);
-      assertEquals(WebHdfsFileSystem.getHomeDirectoryString(ugi),
+      assertEquals(webhdfs.getHomeDirectory().toUri().getPath(),
           m.get(Path.class.getSimpleName()));
       conn.disconnect();
     }


Mime
View raw message