Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 841D111881 for ; Tue, 9 Sep 2014 00:04:44 +0000 (UTC) Received: (qmail 64535 invoked by uid 500); 9 Sep 2014 00:04:44 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 64309 invoked by uid 500); 9 Sep 2014 00:04:44 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 64292 invoked by uid 99); 9 Sep 2014 00:04:44 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 09 Sep 2014 00:04:44 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id B4830A0F545; Tue, 9 Sep 2014 00:04:43 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wang@apache.org To: common-commits@hadoop.apache.org Date: Tue, 09 Sep 2014 00:04:43 -0000 Message-Id: <6a0dea799fc745818c03a668aa2bec14@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/2] git commit: HDFS-6951. Correctly persist raw namespace xattrs to edit log and fsimage. Contributed by Charles Lamb. Repository: hadoop Updated Branches: refs/heads/branch-2 bdcf5e940 -> 176b07b48 HDFS-6951. Correctly persist raw namespace xattrs to edit log and fsimage. Contributed by Charles Lamb. (cherry picked from commit 04915a08141bbe71bdef26e3f539aa8b76f89ac7) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dc4da242 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dc4da242 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dc4da242 Branch: refs/heads/branch-2 Commit: dc4da242f202a51bf1279479b7b887d286fe048c Parents: bdcf5e9 Author: Andrew Wang Authored: Mon Sep 8 16:59:30 2014 -0700 Committer: Andrew Wang Committed: Mon Sep 8 17:01:47 2014 -0700 ---------------------------------------------------------------------- .../server/namenode/EncryptionZoneManager.java | 12 +++ .../hdfs/server/namenode/FSDirectory.java | 2 +- .../server/namenode/FSImageFormatPBINode.java | 18 ++++- .../server/namenode/NameNodeLayoutVersion.java | 3 +- .../hadoop-hdfs/src/main/proto/fsimage.proto | 8 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 8 ++ .../hdfs/server/namenode/FSXAttrBaseTest.java | 75 ++++++++++++------- .../hadoop-hdfs/src/test/resources/editsStored | Bin 5252 -> 5252 bytes .../src/test/resources/editsStored.xml | 2 +- 9 files changed, 93 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc4da242/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java index a0e1f0c..f00f132 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java @@ -111,6 +111,18 @@ public class EncryptionZoneManager { */ void addEncryptionZone(Long inodeId, String keyName) { assert dir.hasWriteLock(); + unprotectedAddEncryptionZone(inodeId, keyName); + } + + /** + * Add a new encryption zone. + *

+ * Does not assume that the FSDirectory lock is held. + * + * @param inodeId of the encryption zone + * @param keyName encryption zone key name + */ + void unprotectedAddEncryptionZone(Long inodeId, String keyName) { final EncryptionZoneInt ez = new EncryptionZoneInt(inodeId, keyName); encryptionZones.put(inodeId, ez); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc4da242/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 7b5f485..cdf4dba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@ -2100,7 +2100,7 @@ public class FSDirectory implements Closeable { for (XAttr xattr : xattrs) { final String xaName = XAttrHelper.getPrefixName(xattr); if (CRYPTO_XATTR_ENCRYPTION_ZONE.equals(xaName)) { - ezManager.addEncryptionZone(inode.getId(), + ezManager.unprotectedAddEncryptionZone(inode.getId(), new String(xattr.getValue())); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc4da242/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java index 600d284..2a46fc4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java @@ -83,7 +83,12 @@ public final class FSImageFormatPBINode { private static final int XATTR_NAMESPACE_OFFSET = 30; private static final int XATTR_NAME_MASK = (1 << 24) - 1; private static final int XATTR_NAME_OFFSET = 6; - private static final XAttr.NameSpace[] XATTR_NAMESPACE_VALUES = + + /* See the comments in fsimage.proto for an explanation of the following. */ + private static final int XATTR_NAMESPACE_EXT_OFFSET = 5; + private static final int XATTR_NAMESPACE_EXT_MASK = 1; + + private static final XAttr.NameSpace[] XATTR_NAMESPACE_VALUES = XAttr.NameSpace.values(); @@ -121,6 +126,8 @@ public final class FSImageFormatPBINode { int v = xAttrCompactProto.getName(); int nid = (v >> XATTR_NAME_OFFSET) & XATTR_NAME_MASK; int ns = (v >> XATTR_NAMESPACE_OFFSET) & XATTR_NAMESPACE_MASK; + ns |= + ((v >> XATTR_NAMESPACE_EXT_OFFSET) & XATTR_NAMESPACE_EXT_MASK) << 2; String name = stringTable[nid]; byte[] value = null; if (xAttrCompactProto.getValue() != null) { @@ -367,10 +374,13 @@ public final class FSImageFormatPBINode { for (XAttr a : f.getXAttrs()) { XAttrCompactProto.Builder xAttrCompactBuilder = XAttrCompactProto. newBuilder(); - int v = ((a.getNameSpace().ordinal() & XATTR_NAMESPACE_MASK) << - XATTR_NAMESPACE_OFFSET) - | ((stringMap.getId(a.getName()) & XATTR_NAME_MASK) << + int nsOrd = a.getNameSpace().ordinal(); + Preconditions.checkArgument(nsOrd < 8, "Too many namespaces."); + int v = ((nsOrd & XATTR_NAMESPACE_MASK) << XATTR_NAMESPACE_OFFSET) + | ((stringMap.getId(a.getName()) & XATTR_NAME_MASK) << XATTR_NAME_OFFSET); + v |= (((nsOrd >> 2) & XATTR_NAMESPACE_EXT_MASK) << + XATTR_NAMESPACE_EXT_OFFSET); xAttrCompactBuilder.setName(v); if (a.getValue() != null) { xAttrCompactBuilder.setValue(PBHelper.getByteString(a.getValue())); http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc4da242/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java index 1df6df4..404e205 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java @@ -67,7 +67,8 @@ public class NameNodeLayoutVersion { EDITLOG_LENGTH(-56, "Add length field to every edit log op"), XATTRS(-57, "Extended attributes"), CREATE_OVERWRITE(-58, "Use single editlog record for " + - "creating file with overwrite"); + "creating file with overwrite"), + XATTRS_NAMESPACE_EXT(-59, "Increase number of xattr namespaces"); private final FeatureInfo info; http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc4da242/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto index 1c8edfa..29fcd36 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto @@ -113,8 +113,12 @@ message INodeSection { * * [0:2) -- the namespace of XAttr (XAttrNamespaceProto) * [2:26) -- the name of the entry, which is an ID that points to a - * string in the StringTableSection. - * [26:32) -- reserved for future uses. + * string in the StringTableSection. + * [26:27) -- namespace extension. Originally there were only 4 namespaces + * so only 2 bits were needed. At that time, this bit was reserved. When a + * 5th namespace was created (raw) this bit became used as a 3rd namespace + * bit. + * [27:32) -- reserved for future uses. */ required fixed32 name = 1; optional bytes value = 2; http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc4da242/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index 0ef538d..d0d29ea 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -49,6 +49,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.client.HdfsAdmin; import org.apache.hadoop.hdfs.protocol.EncryptionZone; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; +import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction; import org.apache.hadoop.hdfs.server.namenode.EncryptionFaultInjector; import org.apache.hadoop.hdfs.server.namenode.EncryptionZoneManager; import org.apache.hadoop.security.AccessControlException; @@ -314,6 +315,13 @@ public class TestEncryptionZones { assertNumZones(numZones); assertZonePresent(null, zonePath.toString()); } + + fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER); + fs.saveNamespace(); + fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE); + cluster.restartNameNode(true); + assertNumZones(numZones); + assertZonePresent(null, zone1.toString()); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc4da242/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java index 0c7b807..9c48400 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSXAttrBaseTest.java @@ -56,6 +56,7 @@ import org.junit.BeforeClass; import org.junit.Test; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; /** * Tests NameNode interaction for all XAttr APIs. @@ -129,51 +130,73 @@ public class FSXAttrBaseTest { */ @Test(timeout = 120000) public void testCreateXAttr() throws Exception { - FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750)); - fs.setXAttr(path, name1, value1, EnumSet.of(XAttrSetFlag.CREATE)); + Map expectedXAttrs = Maps.newHashMap(); + expectedXAttrs.put(name1, value1); + expectedXAttrs.put(name2, null); + doTestCreateXAttr(path, expectedXAttrs); + expectedXAttrs.put(raw1, value1); + doTestCreateXAttr(rawPath, expectedXAttrs); + } + + private void doTestCreateXAttr(Path usePath, Map expectedXAttrs) throws Exception { + FileSystem.mkdirs(fs, usePath, FsPermission.createImmutable((short)0750)); + fs.setXAttr(usePath, name1, value1, EnumSet.of(XAttrSetFlag.CREATE)); - Map xattrs = fs.getXAttrs(path); + Map xattrs = fs.getXAttrs(usePath); Assert.assertEquals(xattrs.size(), 1); Assert.assertArrayEquals(value1, xattrs.get(name1)); - fs.removeXAttr(path, name1); + fs.removeXAttr(usePath, name1); - xattrs = fs.getXAttrs(path); + xattrs = fs.getXAttrs(usePath); Assert.assertEquals(xattrs.size(), 0); // Create xattr which already exists. - fs.setXAttr(path, name1, value1, EnumSet.of(XAttrSetFlag.CREATE)); + fs.setXAttr(usePath, name1, value1, EnumSet.of(XAttrSetFlag.CREATE)); try { - fs.setXAttr(path, name1, value1, EnumSet.of(XAttrSetFlag.CREATE)); + fs.setXAttr(usePath, name1, value1, EnumSet.of(XAttrSetFlag.CREATE)); Assert.fail("Creating xattr which already exists should fail."); } catch (IOException e) { } - fs.removeXAttr(path, name1); + fs.removeXAttr(usePath, name1); - // Create two xattrs - fs.setXAttr(path, name1, value1, EnumSet.of(XAttrSetFlag.CREATE)); - fs.setXAttr(path, name2, null, EnumSet.of(XAttrSetFlag.CREATE)); - xattrs = fs.getXAttrs(path); - Assert.assertEquals(xattrs.size(), 2); - Assert.assertArrayEquals(value1, xattrs.get(name1)); - Assert.assertArrayEquals(new byte[0], xattrs.get(name2)); + // Create the xattrs + for (Map.Entry ent : expectedXAttrs.entrySet()) { + fs.setXAttr(usePath, ent.getKey(), ent.getValue(), + EnumSet.of(XAttrSetFlag.CREATE)); + } + xattrs = fs.getXAttrs(usePath); + Assert.assertEquals(xattrs.size(), expectedXAttrs.size()); + for (Map.Entry ent : expectedXAttrs.entrySet()) { + final byte[] val = + (ent.getValue() == null) ? new byte[0] : ent.getValue(); + Assert.assertArrayEquals(val, xattrs.get(ent.getKey())); + } restart(false); initFileSystem(); - xattrs = fs.getXAttrs(path); - Assert.assertEquals(xattrs.size(), 2); - Assert.assertArrayEquals(value1, xattrs.get(name1)); - Assert.assertArrayEquals(new byte[0], xattrs.get(name2)); + xattrs = fs.getXAttrs(usePath); + Assert.assertEquals(xattrs.size(), expectedXAttrs.size()); + for (Map.Entry ent : expectedXAttrs.entrySet()) { + final byte[] val = + (ent.getValue() == null) ? new byte[0] : ent.getValue(); + Assert.assertArrayEquals(val, xattrs.get(ent.getKey())); + } restart(true); initFileSystem(); - xattrs = fs.getXAttrs(path); - Assert.assertEquals(xattrs.size(), 2); - Assert.assertArrayEquals(value1, xattrs.get(name1)); - Assert.assertArrayEquals(new byte[0], xattrs.get(name2)); - - fs.removeXAttr(path, name1); - fs.removeXAttr(path, name2); + xattrs = fs.getXAttrs(usePath); + Assert.assertEquals(xattrs.size(), expectedXAttrs.size()); + for (Map.Entry ent : expectedXAttrs.entrySet()) { + final byte[] val = + (ent.getValue() == null) ? new byte[0] : ent.getValue(); + Assert.assertArrayEquals(val, xattrs.get(ent.getKey())); + } + + for (Map.Entry ent : expectedXAttrs.entrySet()) { + fs.removeXAttr(usePath, ent.getKey()); + } } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc4da242/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored index 5d93a50..754f690 100644 Binary files a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored differ http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc4da242/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml index 977be98..7cfb689 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml @@ -1,6 +1,6 @@ - -58 + -59 OP_START_LOG_SEGMENT