Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9098F11D41 for ; Wed, 2 Jul 2014 03:20:49 +0000 (UTC) Received: (qmail 75945 invoked by uid 500); 2 Jul 2014 03:20:48 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 75807 invoked by uid 500); 2 Jul 2014 03:20:48 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 75796 invoked by uid 99); 2 Jul 2014 03:20:48 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 Jul 2014 03:20:48 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 Jul 2014 03:20:46 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 27003238890D; Wed, 2 Jul 2014 03:20:26 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1607239 - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./ src/test/java/org/apache/hadoop/hdfs/server/namenode/ src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/ src/test/java/org/apache/hadoop/hdfs/tools/offlineIm... Date: Wed, 02 Jul 2014 03:20:25 -0000 To: hdfs-commits@hadoop.apache.org From: cnauroth@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140702032026.27003238890D@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: cnauroth Date: Wed Jul 2 03:20:25 2014 New Revision: 1607239 URL: http://svn.apache.org/r1607239 Log: HDFS-6603. Add XAttr with ACL test. Contributed by Stephen Chu. Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithXAttr.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1607239&r1=1607238&r2=1607239&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Jul 2 03:20:25 2014 @@ -482,6 +482,8 @@ Release 2.5.0 - UNRELEASED HDFS-6572. Add an option to the NameNode that prints the software and on-disk image versions. (Charles Lamb via cnauroth) + HDFS-6603. Add XAttr with ACL test. (Stephen Chu via cnauroth) + OPTIMIZATIONS HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn) Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java?rev=1607239&r1=1607238&r2=1607239&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java Wed Jul 2 03:20:25 2014 @@ -32,12 +32,20 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.XAttrSetFlag; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.test.GenericTestUtils; + +import static org.apache.hadoop.fs.permission.AclEntryScope.ACCESS; +import static org.apache.hadoop.fs.permission.AclEntryType.USER; +import static org.apache.hadoop.fs.permission.FsAction.ALL; +import static org.apache.hadoop.fs.permission.FsAction.READ; +import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.aclEntry; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import org.junit.After; @@ -60,7 +68,7 @@ public class FSXAttrBaseTest { protected static MiniDFSCluster dfsCluster; protected static Configuration conf; private static int pathCount = 0; - private static Path path; + protected static Path path; // XAttrs protected static final String name1 = "user.a1"; @@ -73,10 +81,16 @@ public class FSXAttrBaseTest { protected FileSystem fs; + private static final UserGroupInformation BRUCE = + UserGroupInformation.createUserForTesting("bruce", new String[] { }); + private static final UserGroupInformation DIANA = + UserGroupInformation.createUserForTesting("diana", new String[] { }); + @BeforeClass public static void init() throws Exception { conf = new HdfsConfiguration(); conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY, true); + conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true); conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_XATTRS_PER_INODE_KEY, 3); conf.setInt(DFSConfigKeys.DFS_NAMENODE_MAX_XATTR_SIZE_KEY, MAX_SIZE); initCluster(true); @@ -388,6 +402,21 @@ public class FSXAttrBaseTest { fs.removeXAttr(path, name3); } + @Test(timeout = 120000) + public void testRenameFileWithXAttr() throws Exception { + FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750)); + fs.setXAttr(path, name1, value1, EnumSet.of(XAttrSetFlag.CREATE)); + fs.setXAttr(path, name2, value2, EnumSet.of(XAttrSetFlag.CREATE)); + Path renamePath = new Path(path.toString() + "-rename"); + fs.rename(path, renamePath); + Map xattrs = fs.getXAttrs(renamePath); + Assert.assertEquals(xattrs.size(), 2); + Assert.assertArrayEquals(value1, xattrs.get(name1)); + Assert.assertArrayEquals(value2, xattrs.get(name2)); + fs.removeXAttr(renamePath, name1); + fs.removeXAttr(renamePath, name2); + } + /** * Test the listXAttrs api. * listXAttrs on a path that doesn't exist. @@ -535,6 +564,50 @@ public class FSXAttrBaseTest { Assert.assertArrayEquals(value1, xattrs.get(name1)); Assert.assertArrayEquals(value2, xattrs.get(name2)); } + + @Test(timeout = 120000) + public void testXAttrAcl() throws Exception { + FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short) 0750)); + fs.setOwner(path, BRUCE.getUserName(), null); + FileSystem fsAsBruce = createFileSystem(BRUCE); + FileSystem fsAsDiana = createFileSystem(DIANA); + fsAsBruce.setXAttr(path, name1, value1); + + Map xattrs; + try { + xattrs = fsAsDiana.getXAttrs(path); + Assert.fail("Diana should not have read access to get xattrs"); + } catch (AccessControlException e) { + // Ignore + } + + // Give Diana read permissions to the path + fsAsBruce.modifyAclEntries(path, Lists.newArrayList( + aclEntry(ACCESS, USER, DIANA.getUserName(), READ))); + xattrs = fsAsDiana.getXAttrs(path); + Assert.assertArrayEquals(value1, xattrs.get(name1)); + + try { + fsAsDiana.removeXAttr(path, name1); + Assert.fail("Diana should not have write access to remove xattrs"); + } catch (AccessControlException e) { + // Ignore + } + + try { + fsAsDiana.setXAttr(path, name2, value2); + Assert.fail("Diana should not have write access to set xattrs"); + } catch (AccessControlException e) { + // Ignore + } + + fsAsBruce.modifyAclEntries(path, Lists.newArrayList( + aclEntry(ACCESS, USER, DIANA.getUserName(), ALL))); + fsAsDiana.setXAttr(path, name2, value2); + Assert.assertArrayEquals(value2, fsAsDiana.getXAttrs(path).get(name2)); + fsAsDiana.removeXAttr(path, name1); + fsAsDiana.removeXAttr(path, name2); + } /** * Creates a FileSystem for the super-user. @@ -545,6 +618,18 @@ public class FSXAttrBaseTest { protected FileSystem createFileSystem() throws Exception { return dfsCluster.getFileSystem(); } + + /** + * Creates a FileSystem for a specific user. + * + * @param user UserGroupInformation specific user + * @return FileSystem for specific user + * @throws Exception if creation fails + */ + protected FileSystem createFileSystem(UserGroupInformation user) + throws Exception { + return DFSTestUtil.getFileSystemAs(user, conf); + } /** * Initializes all FileSystem instances used in the tests. Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithXAttr.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithXAttr.java?rev=1607239&r1=1607238&r2=1607239&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithXAttr.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithXAttr.java Wed Jul 2 03:20:25 2014 @@ -49,6 +49,8 @@ public class TestFSImageWithXAttr { private static final byte[] newValue1 = {0x31, 0x31, 0x31}; private static final String name2 = "user.a2"; private static final byte[] value2 = {0x37, 0x38, 0x39}; + private static final String name3 = "user.a3"; + private static final byte[] value3 = {}; @BeforeClass public static void setUp() throws IOException { @@ -70,25 +72,29 @@ public class TestFSImageWithXAttr { fs.setXAttr(path, name1, value1, EnumSet.of(XAttrSetFlag.CREATE)); fs.setXAttr(path, name2, value2, EnumSet.of(XAttrSetFlag.CREATE)); + fs.setXAttr(path, name3, null, EnumSet.of(XAttrSetFlag.CREATE)); restart(fs, persistNamespace); Map xattrs = fs.getXAttrs(path); - Assert.assertEquals(xattrs.size(), 2); + Assert.assertEquals(xattrs.size(), 3); Assert.assertArrayEquals(value1, xattrs.get(name1)); Assert.assertArrayEquals(value2, xattrs.get(name2)); + Assert.assertArrayEquals(value3, xattrs.get(name3)); fs.setXAttr(path, name1, newValue1, EnumSet.of(XAttrSetFlag.REPLACE)); restart(fs, persistNamespace); xattrs = fs.getXAttrs(path); - Assert.assertEquals(xattrs.size(), 2); + Assert.assertEquals(xattrs.size(), 3); Assert.assertArrayEquals(newValue1, xattrs.get(name1)); Assert.assertArrayEquals(value2, xattrs.get(name2)); + Assert.assertArrayEquals(value3, xattrs.get(name3)); fs.removeXAttr(path, name1); fs.removeXAttr(path, name2); + fs.removeXAttr(path, name3); restart(fs, persistNamespace); xattrs = fs.getXAttrs(path); Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java?rev=1607239&r1=1607238&r2=1607239&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java Wed Jul 2 03:20:25 2014 @@ -269,6 +269,18 @@ public class TestXAttrWithSnapshot { } /** + * Assert exception of removing xattr on read-only snapshot. + */ + @Test + public void testRemoveXAttrSnapshotPath() throws Exception { + FileSystem.mkdirs(hdfs, path, FsPermission.createImmutable((short) 0700)); + hdfs.setXAttr(path, name1, value1); + SnapshotTestHelper.createSnapshot(hdfs, path, snapshotName); + exception.expect(SnapshotAccessControlException.class); + hdfs.removeXAttr(snapshotPath, name1); + } + + /** * Assert exception of setting xattr when exceeding quota. */ @Test Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java?rev=1607239&r1=1607238&r2=1607239&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java Wed Jul 2 03:20:25 2014 @@ -143,6 +143,8 @@ public class TestOfflineImageViewer { hdfs.mkdirs(xattr); hdfs.setXAttr(xattr, "user.a1", new byte[]{ 0x31, 0x32, 0x33 }); hdfs.setXAttr(xattr, "user.a2", new byte[]{ 0x37, 0x38, 0x39 }); + // OIV should be able to handle empty value XAttrs + hdfs.setXAttr(xattr, "user.a3", null); writtenFiles.put(xattr.toString(), hdfs.getFileStatus(xattr)); // Write results to the fsimage file