hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject hadoop git commit: HADOOP-12056. Use DirectoryStream in DiskChecker#checkDirs to detect errors when listing a directory. Contributed by Zhihai Xu.
Date Fri, 05 Jun 2015 20:52:48 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 83a97907c -> 1137fd4c5


HADOOP-12056. Use DirectoryStream in DiskChecker#checkDirs to detect errors when listing a
directory. Contributed by Zhihai Xu.

(cherry picked from commit bc11e158b1726174fae2c7d2e8aa1f5005bf42eb)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1137fd4c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1137fd4c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1137fd4c

Branch: refs/heads/branch-2
Commit: 1137fd4c5b353e396660a629ebca037ca01c0113
Parents: 83a9790
Author: Andrew Wang <wang@apache.org>
Authored: Fri Jun 5 13:52:21 2015 -0700
Committer: Andrew Wang <wang@apache.org>
Committed: Fri Jun 5 13:52:43 2015 -0700

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../org/apache/hadoop/util/DiskChecker.java     | 24 ++++++++++++++++----
 .../org/apache/hadoop/util/TestDiskChecker.java | 22 ++++++++++++++++++
 3 files changed, 45 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1137fd4c/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index d915723..025c761 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -147,6 +147,9 @@ Release 2.8.0 - UNRELEASED
     HADOOP-12059. S3Credentials should support use of CredentialProvider.
     (Sean Busbey via wang)
 
+    HADOOP-12056. Use DirectoryStream in DiskChecker#checkDirs to detect
+    errors when listing a directory. (Zhihai Xu via wang)
+
   OPTIMIZATIONS
 
     HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1137fd4c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
index 6b27ae5..a36a7a0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DiskChecker.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.util;
 
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.DirectoryIteratorException;
+import java.nio.file.Files;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceStability;
@@ -86,13 +89,26 @@ public class DiskChecker {
    */
   public static void checkDirs(File dir) throws DiskErrorException {
     checkDir(dir);
-    for (File child : dir.listFiles()) {
-      if (child.isDirectory()) {
-        checkDirs(child);
+    IOException ex = null;
+    try (DirectoryStream<java.nio.file.Path> stream =
+        Files.newDirectoryStream(dir.toPath())) {
+      for (java.nio.file.Path entry: stream) {
+        File child = entry.toFile();
+        if (child.isDirectory()) {
+          checkDirs(child);
+        }
       }
+    } catch (DirectoryIteratorException de) {
+      ex = de.getCause();
+    } catch (IOException ie) {
+      ex = ie;
+    }
+    if (ex != null) {
+      throw new DiskErrorException("I/O error when open a directory: "
+          + dir.toString(), ex);
     }
   }
-  
+
   /**
    * Create the directory if it doesn't exist and check that dir is readable,
    * writable and executable

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1137fd4c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
index 5ab1313..de54735 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestDiskChecker.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.DiskChecker.DiskErrorException;
 import org.apache.hadoop.util.Shell;
 
@@ -180,4 +181,25 @@ public class TestDiskChecker {
     System.out.println("checkDir success: " + success);
 
   }
+
+  @Test (timeout = 30000)
+  public void testCheckDirsIOException() throws Throwable {
+    Path path = new Path("target", TestDiskChecker.class.getSimpleName());
+    File localDir = new File(path.toUri().getRawPath());
+    localDir.mkdir();
+    File localFile = new File(localDir, "test");
+    localFile.createNewFile();
+    File spyLocalDir = spy(localDir);
+    doReturn(localFile.toPath()).when(spyLocalDir).toPath();
+    try {
+      DiskChecker.checkDirs(spyLocalDir);
+      fail("Expected exception for I/O error");
+    } catch (DiskErrorException e) {
+      GenericTestUtils.assertExceptionContains("I/O error", e);
+      assertTrue(e.getCause() instanceof IOException);
+    } finally {
+      localFile.delete();
+      localDir.delete();
+    }
+  }
 }


Mime
View raw message