hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject hadoop git commit: HDFS-10983. OIV tool should make an EC file explicit. Contributed by Manoj Govindassamy.
Date Wed, 08 Mar 2017 23:36:23 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk d7762a551 -> 5ca6ef0c2


HDFS-10983. OIV tool should make an EC file explicit. Contributed by Manoj Govindassamy.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ca6ef0c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ca6ef0c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ca6ef0c

Branch: refs/heads/trunk
Commit: 5ca6ef0c268b1acb3abf12505b9ead6fe7e38a23
Parents: d7762a5
Author: Andrew Wang <wang@apache.org>
Authored: Wed Mar 8 15:36:19 2017 -0800
Committer: Andrew Wang <wang@apache.org>
Committed: Wed Mar 8 15:36:19 2017 -0800

----------------------------------------------------------------------
 .../server/namenode/FSImageFormatPBINode.java   |  1 +
 .../OfflineImageReconstructor.java              |  4 +
 .../offlineImageViewer/PBImageXmlWriter.java    | 15 ++-
 .../hdfs/server/namenode/TestFSImage.java       |  1 +
 .../TestOfflineImageViewer.java                 | 99 +++++++++++++++++++-
 5 files changed, 112 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca6ef0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 17b1da7..ef334f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -332,6 +332,7 @@ public final class FSImageFormatPBINode {
       BlockType blockType = PBHelperClient.convert(f.getBlockType());
       LoaderContext state = parent.getLoaderContext();
       boolean isStriped = f.hasErasureCodingPolicyID();
+      assert ((!isStriped) || (isStriped && !f.hasReplication()));
       Short replication = (!isStriped ? (short) f.getReplication() : null);
       ErasureCodingPolicy ecPolicy = isStriped ?
           ErasureCodingPolicyManager.getPolicyByPolicyID(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca6ef0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
index ed348d3..e80f4d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
@@ -647,6 +647,10 @@ class OfflineImageReconstructor {
         break;
       case "STRIPED":
         bld.setBlockType(HdfsProtos.BlockTypeProto.STRIPED);
+        ival = node.removeChildInt(INODE_SECTION_EC_POLICY_ID);
+        if (ival != null) {
+          bld.setErasureCodingPolicyID(ival);
+        }
         break;
       default:
         throw new IOException("INode XML found with unknown <blocktype> " +

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca6ef0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
index f8734cb..5a42a6b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheD
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
 import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTypeProto;
 import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos;
 import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode;
 import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName;
@@ -59,6 +58,7 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotDiffSection;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotSection;
 import org.apache.hadoop.hdfs.server.namenode.FsImageProto.StringTableSection;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
 import org.apache.hadoop.hdfs.util.XMLUtils;
 import org.apache.hadoop.util.LimitInputStream;
 import com.google.common.collect.ImmutableList;
@@ -132,6 +132,8 @@ public final class PBImageXmlWriter {
   public static final String INODE_SECTION_STORAGE_POLICY_ID =
       "storagePolicyId";
   public static final String INODE_SECTION_BLOCK_TYPE = "blockType";
+  public static final String INODE_SECTION_EC_POLICY_ID =
+      "erasureCodingPolicyId";
   public static final String INODE_SECTION_NS_QUOTA = "nsquota";
   public static final String INODE_SECTION_DS_QUOTA = "dsquota";
   public static final String INODE_SECTION_TYPE_QUOTA = "typeQuota";
@@ -472,8 +474,12 @@ public final class PBImageXmlWriter {
   }
 
   private void dumpINodeFile(INodeSection.INodeFile f) {
-    o(SECTION_REPLICATION, f.getReplication())
-        .o(INODE_SECTION_MTIME, f.getModificationTime())
+    if (f.hasErasureCodingPolicyID()) {
+      o(SECTION_REPLICATION, INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS);
+    } else {
+      o(SECTION_REPLICATION, f.getReplication());
+    }
+    o(INODE_SECTION_MTIME, f.getModificationTime())
         .o(INODE_SECTION_ATIME, f.getAccessTime())
         .o(INODE_SECTION_PREFERRED_BLOCK_SIZE, f.getPreferredBlockSize())
         .o(INODE_SECTION_PERMISSION, dumpPermission(f.getPermission()));
@@ -495,8 +501,9 @@ public final class PBImageXmlWriter {
     if (f.hasStoragePolicyID()) {
       o(INODE_SECTION_STORAGE_POLICY_ID, f.getStoragePolicyID());
     }
-    if (f.getBlockType() != BlockTypeProto.CONTIGUOUS) {
+    if (f.hasErasureCodingPolicyID()) {
       o(INODE_SECTION_BLOCK_TYPE, f.getBlockType().name());
+      o(INODE_SECTION_EC_POLICY_ID, f.getErasureCodingPolicyID());
     }
 
     if (f.hasFileUC()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca6ef0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index ae15491..57416ed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -223,6 +223,7 @@ public class TestFSImage {
     // blocks to/from legacy fsimage
     assertEquals(3, fileByLoaded.getBlocks().length);
     assertEquals(preferredBlockSize, fileByLoaded.getPreferredBlockSize());
+    assertEquals(file.getFileReplication(), fileByLoaded.getFileReplication());
 
     if (isUC) {
       assertEquals(client,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca6ef0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index b587017..e52d536 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -76,11 +76,13 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.BlockType;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
 import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
 import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion;
 import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
 import org.apache.hadoop.io.IOUtils;
@@ -91,9 +93,8 @@ import org.apache.log4j.Level;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
-import org.junit.Rule;
 import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
+import org.xml.sax.Attributes;
 import org.xml.sax.InputSource;
 import org.xml.sax.SAXException;
 import org.xml.sax.helpers.DefaultHandler;
@@ -112,7 +113,6 @@ public class TestOfflineImageViewer {
   // namespace as written to dfs, to be compared with viewer's output
   final static HashMap<String, FileStatus> writtenFiles = Maps.newHashMap();
   static int dirCount = 0;
-
   private static File tempDir;
 
   // Create a populated namespace for later testing. Save its contents to a
@@ -358,6 +358,96 @@ public class TestOfflineImageViewer {
     assertEquals(0, status);
   }
 
+  /**
+   *  SAX handler to verify EC Files and their policies.
+   */
+  class ECXMLHandler extends DefaultHandler {
+
+    private boolean isInode = false;
+    private boolean isAttrRepl = false;
+    private boolean isAttrName = false;
+    private boolean isXAttrs = false;
+    private boolean isAttrECPolicy = false;
+    private boolean isAttrBlockType = false;
+    private String currentInodeName;
+    private String currentECPolicy;
+    private String currentBlockType;
+    private String currentRepl;
+
+    @Override
+    public void startElement(String uri, String localName, String qName,
+        Attributes attributes) throws SAXException {
+      super.startElement(uri, localName, qName, attributes);
+      if (qName.equalsIgnoreCase(PBImageXmlWriter.INODE_SECTION_INODE)) {
+        isInode = true;
+      } else if (isInode && !isXAttrs && qName.equalsIgnoreCase(
+          PBImageXmlWriter.SECTION_NAME)) {
+        isAttrName = true;
+      } else if (isInode && qName.equalsIgnoreCase(
+          PBImageXmlWriter.SECTION_REPLICATION)) {
+        isAttrRepl = true;
+      } else if (isInode &&
+          qName.equalsIgnoreCase(PBImageXmlWriter.INODE_SECTION_EC_POLICY_ID)) {
+        isAttrECPolicy = true;
+      } else if (isInode && qName.equalsIgnoreCase(
+          PBImageXmlWriter.INODE_SECTION_BLOCK_TYPE)) {
+        isAttrBlockType = true;
+      } else if (isInode && qName.equalsIgnoreCase(
+          PBImageXmlWriter.INODE_SECTION_XATTRS)) {
+        isXAttrs = true;
+      }
+    }
+
+    @Override
+    public void endElement(String uri, String localName, String qName)
+        throws SAXException {
+      super.endElement(uri, localName, qName);
+      if (qName.equalsIgnoreCase(PBImageXmlWriter.INODE_SECTION_INODE)) {
+        if (currentInodeName != null && currentInodeName.length() > 0) {
+          if (currentBlockType != null && currentBlockType.equalsIgnoreCase(
+              BlockType.STRIPED.name())) {
+            Assert.assertEquals("INode '"
+                    + currentInodeName + "' has unexpected EC Policy!",
+                Byte.parseByte(currentECPolicy),
+                ErasureCodingPolicyManager.getPolicyByPolicyID(
+                    HdfsConstants.XOR_2_1_POLICY_ID).getId());
+            Assert.assertEquals("INode '"
+                    + currentInodeName + "' has unexpected replication!",
+                currentRepl,
+                Short.toString(INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS));
+          }
+        }
+        isInode = false;
+        currentInodeName = "";
+        currentECPolicy = "";
+        currentRepl = "";
+      } else if (qName.equalsIgnoreCase(
+          PBImageXmlWriter.INODE_SECTION_XATTRS)) {
+        isXAttrs = false;
+      }
+    }
+
+    @Override
+    public void characters(char[] ch, int start, int length)
+        throws SAXException {
+      super.characters(ch, start, length);
+      String value = new String(ch, start, length);
+      if (isAttrName) {
+        currentInodeName = value;
+        isAttrName = false;
+      } else if (isAttrRepl) {
+        currentRepl = value;
+        isAttrRepl = false;
+      } else if (isAttrECPolicy) {
+        currentECPolicy = value;
+        isAttrECPolicy = false;
+      } else if (isAttrBlockType) {
+        currentBlockType = value;
+        isAttrBlockType = false;
+      }
+    }
+  }
+
   @Test
   public void testPBImageXmlWriter() throws IOException, SAXException,
       ParserConfigurationException {
@@ -368,7 +458,8 @@ public class TestOfflineImageViewer {
     SAXParserFactory spf = SAXParserFactory.newInstance();
     SAXParser parser = spf.newSAXParser();
     final String xml = output.toString();
-    parser.parse(new InputSource(new StringReader(xml)), new DefaultHandler());
+    ECXMLHandler ecxmlHandler = new ECXMLHandler();
+    parser.parse(new InputSource(new StringReader(xml)), ecxmlHandler);
   }
 
   @Test


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message