Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E3EDB111FF for ; Thu, 4 Sep 2014 02:30:59 +0000 (UTC) Received: (qmail 71241 invoked by uid 500); 4 Sep 2014 02:30:59 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 71167 invoked by uid 500); 4 Sep 2014 02:30:59 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 71158 invoked by uid 99); 4 Sep 2014 02:30:59 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 04 Sep 2014 02:30:59 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 53787A07024; Thu, 4 Sep 2014 02:30:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: cmccabe@apache.org To: common-commits@hadoop.apache.org Message-Id: <3a7e7e775ea749d9afcb41cda43f4e40@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: HDFS-6959 Make the HDFS home directory location customizable. Contributed by Yongjun Zhang Date: Thu, 4 Sep 2014 02:30:59 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk 3a0142bd7 -> f4caedfcb HDFS-6959 Make the HDFS home directory location customizable. Contributed by Yongjun Zhang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4caedfc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4caedfc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4caedfc Branch: refs/heads/trunk Commit: f4caedfcbfeae7e2fe7c0e812ddbb087608a5ffd Parents: 3a0142b Author: Colin Patrick Mccabe Authored: Wed Sep 3 19:30:30 2014 -0700 Committer: Colin Patrick Mccabe Committed: Wed Sep 3 19:30:30 2014 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 ++ .../hadoop/hdfs/DistributedFileSystem.java | 11 ++++++-- .../src/main/resources/hdfs-default.xml | 8 ++++++ .../org/apache/hadoop/hdfs/TestLocalDFS.java | 29 +++++++++++++++++++- 5 files changed, 49 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4caedfc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 555f294..8498b00 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -435,6 +435,9 @@ Release 2.6.0 - UNRELEASED HDFS-4257. The ReplaceDatanodeOnFailure policies could have a forgiving option (szetszwo via cmccabe) + HDFS-6959. Make the HDFS home directory location customizable. (yzhang via + cmccabe) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4caedfc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index ace2ae9..b5b4f3c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -41,6 +41,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final int DFS_STREAM_BUFFER_SIZE_DEFAULT = 4096; public static final String DFS_BYTES_PER_CHECKSUM_KEY = "dfs.bytes-per-checksum"; public static final int DFS_BYTES_PER_CHECKSUM_DEFAULT = 512; + public static final String DFS_USER_HOME_DIR_PREFIX_KEY = "dfs.user.home.dir.prefix"; + public static final String DFS_USER_HOME_DIR_PREFIX_DEFAULT = "/user"; public static final String DFS_CLIENT_RETRY_POLICY_ENABLED_KEY = "dfs.client.retry.policy.enabled"; public static final boolean DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT = false; public static final String DFS_CLIENT_RETRY_POLICY_SPEC_KEY = "dfs.client.retry.policy.spec"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4caedfc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index fc4bd84..bf7d62e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -102,6 +102,8 @@ import com.google.common.base.Preconditions; public class DistributedFileSystem extends FileSystem { private Path workingDir; private URI uri; + private String homeDirPrefix = + DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT; DFSClient dfs; private boolean verifyChecksum = true; @@ -136,7 +138,10 @@ public class DistributedFileSystem extends FileSystem { if (host == null) { throw new IOException("Incomplete HDFS URI, no host: "+ uri); } - + homeDirPrefix = conf.get( + DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_KEY, + DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT); + this.dfs = new DFSClient(uri, conf, statistics); this.uri = URI.create(uri.getScheme()+"://"+uri.getAuthority()); this.workingDir = getHomeDirectory(); @@ -167,10 +172,10 @@ public class DistributedFileSystem extends FileSystem { workingDir = fixRelativePart(dir); } - @Override public Path getHomeDirectory() { - return makeQualified(new Path("/user/" + dfs.ugi.getShortUserName())); + return makeQualified(new Path(homeDirPrefix + "/" + + dfs.ugi.getShortUserName())); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4caedfc/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index cdb0100..6d1eecc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2098,4 +2098,12 @@ + + dfs.user.home.dir.prefix + /user + The directory to prepend to user name to get the user's + home direcotry. + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4caedfc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocalDFS.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocalDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocalDFS.java index 7a5e3d0..1542962 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocalDFS.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLocalDFS.java @@ -90,7 +90,9 @@ public class TestLocalDFS { // test home directory Path home = - fileSys.makeQualified(new Path("/user/" + getUserName(fileSys))); + fileSys.makeQualified( + new Path(DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_DEFAULT + + "/" + getUserName(fileSys))); Path fsHome = fileSys.getHomeDirectory(); assertEquals(home, fsHome); @@ -99,4 +101,29 @@ public class TestLocalDFS { cluster.shutdown(); } } + + /** + * Tests get/set working directory in DFS. + */ + @Test(timeout=30000) + public void testHomeDirectory() throws IOException { + final String[] homeBases = new String[] {"/home", "/home/user"}; + Configuration conf = new HdfsConfiguration(); + for (final String homeBase : homeBases) { + conf.set(DFSConfigKeys.DFS_USER_HOME_DIR_PREFIX_KEY, homeBase); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build(); + FileSystem fileSys = cluster.getFileSystem(); + try { + // test home directory + Path home = + fileSys.makeQualified( + new Path(homeBase + "/" + getUserName(fileSys))); + Path fsHome = fileSys.getHomeDirectory(); + assertEquals(home, fsHome); + } finally { + fileSys.close(); + cluster.shutdown(); + } + } + } }