hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From z..@apache.org
Subject [19/50] [abbrv] hadoop git commit: HDFS-7819. Log WARN message for the blocks which are not in Block ID based layout (Rakesh R via Colin P. McCabe)
Date Mon, 02 Mar 2015 17:15:35 GMT
HDFS-7819. Log WARN message for the blocks which are not in Block ID based layout (Rakesh R
via Colin P. McCabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/da85e17c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/da85e17c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/da85e17c

Branch: refs/heads/HDFS-7285
Commit: da85e17c772dfd2348b8f0e93583f7e7ac4d4e16
Parents: db0b6e6
Author: Colin Patrick Mccabe <cmccabe@cloudera.com>
Authored: Thu Feb 26 11:58:29 2015 -0800
Committer: Zhe Zhang <zhezhang@cloudera.com>
Committed: Mon Mar 2 09:13:52 2015 -0800

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 +++
 .../hdfs/server/datanode/DirectoryScanner.java  | 26 +++++++++++++++++---
 2 files changed, 25 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/da85e17c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e09714f..54b4057 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -679,6 +679,9 @@ Release 2.7.0 - UNRELEASED
     HDFS-7832. Show 'Last Modified' in Namenode's 'Browse Filesystem'
     (vinayakumarb)
 
+    HDFS-7819. Log WARN message for the blocks which are not in Block ID based
+    layout (Rakesh R via Colin P. McCabe)
+
   OPTIMIZATIONS
 
     HDFS-7454. Reduce memory footprint for AclEntries in NameNode.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/da85e17c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 71f976b..09c2914 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -597,14 +597,15 @@ public class DirectoryScanner implements Runnable {
       for (String bpid : bpList) {
         LinkedList<ScanInfo> report = new LinkedList<ScanInfo>();
         File bpFinalizedDir = volume.getFinalizedDir(bpid);
-        result.put(bpid, compileReport(volume, bpFinalizedDir, report));
+        result.put(bpid,
+            compileReport(volume, bpFinalizedDir, bpFinalizedDir, report));
       }
       return result;
     }
 
     /** Compile list {@link ScanInfo} for the blocks in the directory <dir> */
-    private LinkedList<ScanInfo> compileReport(FsVolumeSpi vol, File dir,
-        LinkedList<ScanInfo> report) {
+    private LinkedList<ScanInfo> compileReport(FsVolumeSpi vol,
+        File bpFinalizedDir, File dir, LinkedList<ScanInfo> report) {
       File[] files;
       try {
         files = FileUtil.listFiles(dir);
@@ -622,12 +623,14 @@ public class DirectoryScanner implements Runnable {
        */
       for (int i = 0; i < files.length; i++) {
         if (files[i].isDirectory()) {
-          compileReport(vol, files[i], report);
+          compileReport(vol, bpFinalizedDir, files[i], report);
           continue;
         }
         if (!Block.isBlockFilename(files[i])) {
           if (isBlockMetaFile("blk_", files[i].getName())) {
             long blockId = Block.getBlockId(files[i].getName());
+            verifyFileLocation(files[i].getParentFile(), bpFinalizedDir,
+                blockId);
             report.add(new ScanInfo(blockId, null, files[i], vol));
           }
           continue;
@@ -646,9 +649,24 @@ public class DirectoryScanner implements Runnable {
             break;
           }
         }
+        verifyFileLocation(blockFile.getParentFile(), bpFinalizedDir,
+            blockId);
         report.add(new ScanInfo(blockId, blockFile, metaFile, vol));
       }
       return report;
     }
+
+    /**
+     * Verify whether the actual directory location of block file has the
+     * expected directory path computed using its block ID.
+     */
+    private void verifyFileLocation(File actualBlockDir,
+        File bpFinalizedDir, long blockId) {
+      File blockDir = DatanodeUtil.idToBlockDir(bpFinalizedDir, blockId);
+      if (actualBlockDir.compareTo(blockDir) != 0) {
+        LOG.warn("Block: " + blockId
+            + " has to be upgraded to block ID-based layout");
+      }
+    }
   }
 }


Mime
View raw message