Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id EA51717EF5 for ; Mon, 29 Sep 2014 02:29:34 +0000 (UTC) Received: (qmail 64733 invoked by uid 500); 29 Sep 2014 02:29:34 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 64658 invoked by uid 500); 29 Sep 2014 02:29:34 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 64649 invoked by uid 99); 29 Sep 2014 02:29:34 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 29 Sep 2014 02:29:34 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 550B79BB1E2; Mon, 29 Sep 2014 02:29:34 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: arp@apache.org To: common-commits@hadoop.apache.org Message-Id: <097edbe61de64264bc121d49488ee101@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: git commit: HDFS-7093. Add config key to restrict setStoragePolicy. (Arpit Agarwal) Date: Mon, 29 Sep 2014 02:29:34 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk 400e1bb4e -> b38e52b5e HDFS-7093. Add config key to restrict setStoragePolicy. (Arpit Agarwal) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b38e52b5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b38e52b5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b38e52b5 Branch: refs/heads/trunk Commit: b38e52b5e8f2b2ad91c2c8c4eabc232ee4753ca0 Parents: 400e1bb Author: arp Authored: Sun Sep 28 19:28:51 2014 -0700 Committer: arp Committed: Sun Sep 28 19:28:51 2014 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 ++ .../hdfs/server/namenode/FSNamesystem.java | 28 ++++++++++++++-- .../src/main/resources/hdfs-default.xml | 8 +++++ .../hadoop/hdfs/TestBlockStoragePolicy.java | 35 ++++++++++++++++++++ 5 files changed, 74 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38e52b5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index a0e176a..5e1d717 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -514,6 +514,8 @@ Release 2.6.0 - UNRELEASED HDFS-7156. Update fsck documentation. (Masahiro Yamaguch via shv) + HDFS-7093. Add config key to restrict setStoragePolicy. (Arpit Agarwal) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38e52b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java index 3aa9acf..b016750 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java @@ -558,6 +558,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys { public static final String DFS_DOMAIN_SOCKET_PATH_KEY = "dfs.domain.socket.path"; public static final String DFS_DOMAIN_SOCKET_PATH_DEFAULT = ""; + public static final String DFS_STORAGE_POLICY_ENABLED_KEY = "dfs.storage.policy.enabled"; + public static final boolean DFS_STORAGE_POLICY_ENABLED_DEFAULT = true; + // HA related configuration public static final String DFS_HA_NAMENODES_KEY_PREFIX = "dfs.ha.namenodes"; public static final String DFS_HA_NAMENODE_ID_KEY = "dfs.ha.namenode.id"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38e52b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 4ae47b7..9b4380c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -84,6 +84,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROU import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROUP_KEY; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_DEFAULT; import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER; import static org.apache.hadoop.util.Time.now; @@ -420,6 +422,9 @@ public class FSNamesystem implements Namesystem, FSClusterStats, private final CacheManager cacheManager; private final DatanodeStatistics datanodeStatistics; + // whether setStoragePolicy is allowed. + private final boolean isStoragePolicyEnabled; + private String nameserviceId; private RollingUpgradeInfo rollingUpgradeInfo = null; @@ -781,6 +786,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats, this.datanodeStatistics = blockManager.getDatanodeManager().getDatanodeStatistics(); this.blockIdGenerator = new SequentialBlockIdGenerator(this.blockManager); + this.isStoragePolicyEnabled = + conf.getBoolean(DFS_STORAGE_POLICY_ENABLED_KEY, + DFS_STORAGE_POLICY_ENABLED_DEFAULT); + this.fsOwner = UserGroupInformation.getCurrentUser(); this.fsOwnerShortUserName = fsOwner.getShortUserName(); this.supergroup = conf.get(DFS_PERMISSIONS_SUPERUSERGROUP_KEY, @@ -2273,8 +2282,17 @@ public class FSNamesystem implements Namesystem, FSClusterStats, } private void setStoragePolicyInt(String src, final String policyName) - throws IOException { - checkSuperuserPrivilege(); + throws IOException, UnresolvedLinkException, AccessControlException { + + if (!isStoragePolicyEnabled) { + throw new IOException("Failed to set storage policy since " + + DFS_STORAGE_POLICY_ENABLED_KEY + " is set to false."); + } + FSPermissionChecker pc = null; + if (isPermissionEnabled) { + pc = getPermissionChecker(); + } + checkOperation(OperationCategory.WRITE); byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src); waitForLoadingFSImage(); @@ -2283,6 +2301,12 @@ public class FSNamesystem implements Namesystem, FSClusterStats, try { checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot set storage policy for " + src); + + if (pc != null) { + checkPermission(pc, src, false, null, null, FsAction.WRITE, null, + false, true); + } + src = FSDirectory.resolvePath(src, pathComponents, dir); // get the corresponding policy and make sure the policy name is valid http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38e52b5/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml index 2d4109a..23b25fa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml @@ -2132,4 +2132,12 @@ + + dfs.storage.policy.enabled + true + + Allow users to change the storage policy on files and directories. + + + http://git-wip-us.apache.org/repos/asf/hadoop/blob/b38e52b5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java index 39d1439..771b7bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java @@ -21,6 +21,7 @@ import static org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySu import java.io.File; import java.io.FileNotFoundException; +import java.io.IOException; import java.util.*; import com.google.common.collect.Lists; @@ -69,6 +70,40 @@ public class TestBlockStoragePolicy { static final byte WARM = (byte) 8; static final byte HOT = (byte) 12; + + @Test (timeout=300000) + public void testConfigKeyEnabled() throws IOException { + Configuration conf = new HdfsConfiguration(); + conf.setBoolean(DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY, true); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(1).build(); + try { + cluster.waitActive(); + cluster.getFileSystem().setStoragePolicy(new Path("/"), "COLD"); + } finally { + cluster.shutdown(); + } + } + + /** + * Ensure that setStoragePolicy throws IOException when + * dfs.storage.policy.enabled is set to false. + * @throws IOException + */ + @Test (timeout=300000, expected=IOException.class) + public void testConfigKeyDisabled() throws IOException { + Configuration conf = new HdfsConfiguration(); + conf.setBoolean(DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY, false); + MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf) + .numDataNodes(1).build(); + try { + cluster.waitActive(); + cluster.getFileSystem().setStoragePolicy(new Path("/"), "COLD"); + } finally { + cluster.shutdown(); + } + } + @Test public void testDefaultPolicies() { final Map expectedPolicyStrings = new HashMap();