hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From weic...@apache.org
Subject hadoop git commit: HDFS-13813. Exit NameNode if dangling child inode is detected when saving FsImage. Contributed by Siyao Meng.
Date Mon, 13 Aug 2018 23:14:10 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 eecfbaff3 -> 4b606eb9f


HDFS-13813. Exit NameNode if dangling child inode is detected when saving FsImage. Contributed
by Siyao Meng.

(cherry picked from commit 23854443efa62aa70a1c30c32c3816750e5d7a5b)
(cherry picked from commit b38649c59a70b3112811443464b3b3180f4b9873)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b606eb9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b606eb9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b606eb9

Branch: refs/heads/branch-3.0
Commit: 4b606eb9fd5dde5629e791d3338886a70b5aa57c
Parents: eecfbaf
Author: Wei-Chiu Chuang <weichiu@apache.org>
Authored: Mon Aug 13 16:12:37 2018 -0700
Committer: Wei-Chiu Chuang <weichiu@apache.org>
Committed: Mon Aug 13 16:14:01 2018 -0700

----------------------------------------------------------------------
 .../server/namenode/FSImageFormatPBINode.java   | 28 ++++++++++++++++++--
 .../server/namenode/FSImageFormatProtobuf.java  |  9 ++++---
 2 files changed, 32 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b606eb9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 5e60038..a233d26 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -439,6 +439,8 @@ public final class FSImageFormatPBINode {
   }
 
   public final static class Saver {
+    private long numImageErrors;
+
     private static long buildPermissionStatus(INodeAttributes n,
         final SaverContext.DeduplicationMap<String> stringMap) {
       long userId = stringMap.getId(n.getUserName());
@@ -563,11 +565,13 @@ public final class FSImageFormatPBINode {
       this.summary = summary;
       this.context = parent.getContext();
       this.fsn = context.getSourceNamesystem();
+      this.numImageErrors = 0;
     }
 
     void serializeINodeDirectorySection(OutputStream out) throws IOException {
-      Iterator<INodeWithAdditionalFields> iter = fsn.getFSDirectory()
-          .getINodeMap().getMapIterator();
+      FSDirectory dir = fsn.getFSDirectory();
+      Iterator<INodeWithAdditionalFields> iter = dir.getINodeMap()
+          .getMapIterator();
       final ArrayList<INodeReference> refList = parent.getSaverContext()
           .getRefList();
       int i = 0;
@@ -583,6 +587,17 @@ public final class FSImageFormatPBINode {
           INodeDirectorySection.DirEntry.Builder b = INodeDirectorySection.
               DirEntry.newBuilder().setParent(n.getId());
           for (INode inode : children) {
+            // Error if the child inode doesn't exist in inodeMap
+            if (dir.getInode(inode.getId()) == null) {
+              FSImage.LOG.error(
+                  "FSImageFormatPBINode#serializeINodeDirectorySection: " +
+                      "Dangling child pointer found. Missing INode in " +
+                      "inodeMap: id=" + inode.getId() +
+                      "; path=" + inode.getFullPathName() +
+                      "; parent=" + (inode.getParent() == null ? "null" :
+                      inode.getParent().getFullPathName()));
+              ++numImageErrors;
+            }
             if (!inode.isReference()) {
               b.addChildren(inode.getId());
             } else {
@@ -711,6 +726,15 @@ public final class FSImageFormatPBINode {
           .setId(n.getId())
           .setName(ByteString.copyFrom(n.getLocalNameBytes()));
     }
+
+    /**
+     * Number of non-fatal errors detected while writing the
+     * INodeSection and INodeDirectorySection sections.
+     * @return the number of non-fatal errors detected.
+     */
+    public long getNumImageErrors() {
+      return numImageErrors;
+    }
   }
 
   private FSImageFormatPBINode() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b606eb9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
index 4ac20ad..9752733 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
@@ -474,13 +474,15 @@ public final class FSImageFormatProtobuf {
       out.write(lengthBytes);
     }
 
-    private void saveInodes(FileSummary.Builder summary) throws IOException {
+    private long saveInodes(FileSummary.Builder summary) throws IOException {
       FSImageFormatPBINode.Saver saver = new FSImageFormatPBINode.Saver(this,
           summary);
 
       saver.serializeINodeSection(sectionOutputStream);
       saver.serializeINodeDirectorySection(sectionOutputStream);
       saver.serializeFilesUCSection(sectionOutputStream);
+
+      return saver.getNumImageErrors();
     }
 
     /**
@@ -543,8 +545,9 @@ public final class FSImageFormatProtobuf {
 
       step = new Step(StepType.INODES, filePath);
       prog.beginStep(Phase.SAVING_CHECKPOINT, step);
-      saveInodes(b);
-      long numErrors = saveSnapshots(b);
+      // Count number of non-fatal errors when saving inodes and snapshots.
+      long numErrors = saveInodes(b);
+      numErrors += saveSnapshots(b);
       prog.endStep(Phase.SAVING_CHECKPOINT, step);
 
       step = new Step(StepType.DELEGATION_TOKENS, filePath);


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message