hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rang...@apache.org
Subject svn commit: r646942 - in /hadoop/core/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/DFSClient.java src/test/org/apache/hadoop/dfs/DFSTestUtil.java src/test/org/apache/hadoop/dfs/TestFsck.java
Date Thu, 10 Apr 2008 20:14:28 GMT
Author: rangadi
Date: Thu Apr 10 13:14:25 2008
New Revision: 646942

URL: http://svn.apache.org/viewvc?rev=646942&view=rev
Log:
HADOOP-2905. 'fsck -move' triggers NPE in NameNode. (Lohit Vjayarenu via rangadi)

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
    hadoop/core/trunk/src/test/org/apache/hadoop/dfs/DFSTestUtil.java
    hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFsck.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=646942&r1=646941&r2=646942&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Apr 10 13:14:25 2008
@@ -13,6 +13,9 @@
 
   BUG FIXES
 
+   HADOOP-2905. 'fsck -move' triggers NPE in NameNode. 
+   (Lohit Vjayarenu via rangadi)
+
 Release 0.17.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java?rev=646942&r1=646941&r2=646942&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/dfs/DFSClient.java Thu Apr 10 13:14:25 2008
@@ -689,8 +689,7 @@
   /**
    */
   public boolean mkdirs(String src) throws IOException {
-    checkOpen();
-    return namenode.mkdirs(src, null);
+    return mkdirs(src, null);
   }
 
   /**

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/DFSTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/DFSTestUtil.java?rev=646942&r1=646941&r2=646942&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/DFSTestUtil.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/DFSTestUtil.java Thu Apr 10 13:14:25
2008
@@ -185,6 +185,19 @@
       waitReplication(fs, new Path(root, files[idx].getName()), value);
     }
   }
+
+  /** return list of filenames created as part of createFiles */
+  public String[] getFileNames(String topDir) {
+    if (nFiles == 0)
+      return new String[]{};
+    else {
+      String[] fileNames =  new String[nFiles];
+      for (int idx=0; idx < nFiles; idx++) {
+        fileNames[idx] = topDir + "/" + files[idx].getName();
+      }
+      return fileNames;
+    }
+  }
   
   /** wait for the file's replication to be done */
   static void waitReplication(FileSystem fs, Path fileName, 

Modified: hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFsck.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFsck.java?rev=646942&r1=646941&r2=646942&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFsck.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/dfs/TestFsck.java Thu Apr 10 13:14:25 2008
@@ -20,6 +20,8 @@
 
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
+import java.net.InetSocketAddress;
+import java.io.File;
 
 import junit.framework.TestCase;
 
@@ -33,13 +35,13 @@
  * A JUnit test for doing fsck
  */
 public class TestFsck extends TestCase {
-  static String runFsck(Configuration conf, String path) throws Exception {
+  static String runFsck(Configuration conf, String... path) throws Exception {
     PrintStream oldOut = System.out;
     ByteArrayOutputStream bStream = new ByteArrayOutputStream();
     PrintStream newOut = new PrintStream(bStream, true);
     System.setOut(newOut);
     ((Log4JLogger)PermissionChecker.LOG).getLogger().setLevel(Level.ALL);
-    assertEquals(0, ToolRunner.run(new DFSck(conf), new String[] {path}));
+    assertEquals(0, ToolRunner.run(new DFSck(conf), path));
     ((Log4JLogger)PermissionChecker.LOG).getLogger().setLevel(Level.INFO);
     System.setOut(oldOut);
     return bStream.toString();
@@ -98,4 +100,60 @@
       if (cluster != null) { cluster.shutdown(); }
     }
   }
-}
\ No newline at end of file
+
+  public void testFsckMove() throws Exception {
+    DFSTestUtil util = new DFSTestUtil("TestFsck", 5, 3, 8*1024);
+    MiniDFSCluster cluster = null;
+    FileSystem fs = null;
+    try {
+      Configuration conf = new Configuration();
+      conf.set("dfs.blockreport.intervalMsec", Integer.toString(30));
+      cluster = new MiniDFSCluster(conf, 4, true, null);
+      String topDir = "/srcdat";
+      fs = cluster.getFileSystem();
+      cluster.waitActive();
+      util.createFiles(fs, topDir);
+      String outStr = runFsck(conf, "/");
+      assertTrue(outStr.contains("HEALTHY"));
+      
+      // Corrupt a block by deleting it
+      String[] fileNames = util.getFileNames(topDir);
+      DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost",
+                                          cluster.getNameNodePort()), conf);
+      String block = dfsClient.namenode.
+                      getBlockLocations(fileNames[0], 0, Long.MAX_VALUE).
+                      get(0).getBlock().toString();
+      File baseDir = new File(System.getProperty("test.build.data"),"dfs/data");
+      for (int i=0; i<8; i++) {
+        File blockFile = new File(baseDir, "data" +(i+1)+ "/current/" + block);
+        if(blockFile.exists()) {
+          assertTrue(blockFile.delete());
+        }
+      }
+
+      //Sleep for a while until block reports are sent to discover corruption
+      try {
+        Thread.sleep(5000);
+      } catch (InterruptedException e) {
+      }
+      
+      // We excpect the filesystem to be corrupted
+      outStr = runFsck(conf, "/");
+      assertTrue(outStr.contains("CORRUPT"));
+      
+      // Fix the filesystem by moving corrupted files to lost+found
+      outStr = runFsck(conf, "/", "-move");
+      assertTrue(outStr.contains("CORRUPT"));
+      
+      // Check to make sure we have healthy filesystem
+      outStr = runFsck(conf, "/");
+      assertTrue(outStr.contains("HEALTHY")); 
+      util.cleanup(fs, topDir);
+      if (fs != null) {try{fs.close();} catch(Exception e){}}
+      cluster.shutdown();
+    } finally {
+      if (fs != null) {try{fs.close();} catch(Exception e){}}
+      if (cluster != null) { cluster.shutdown(); }
+    }
+  }
+}



Mime
View raw message