hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject svn commit: r1136258 - in /hadoop/common/branches/branch-0.22/hdfs: CHANGES.txt src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
Date Thu, 16 Jun 2011 00:24:12 GMT
Author: mattf
Date: Thu Jun 16 00:24:11 2011
New Revision: 1136258

URL: http://svn.apache.org/viewvc?rev=1136258&view=rev
Log:
HDFS-1952. FSEditLog.open() appears to succeed even if all EDITS directories fail. Contributed
by Andrew Wang.

Modified:
    hadoop/common/branches/branch-0.22/hdfs/CHANGES.txt
    hadoop/common/branches/branch-0.22/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
    hadoop/common/branches/branch-0.22/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java

Modified: hadoop/common/branches/branch-0.22/hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/hdfs/CHANGES.txt?rev=1136258&r1=1136257&r2=1136258&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.22/hdfs/CHANGES.txt Thu Jun 16 00:24:11 2011
@@ -587,6 +587,9 @@ Release 0.22.0 - Unreleased
     HDFS-2071. Use of isConnected() in DataXceiver is invalid. (Kihwal Lee
     via todd)
 
+    HDFS-1952. FSEditLog.open() appears to succeed even if all EDITS
+    directories fail. (Andrew Wang via mattf)
+
 Release 0.21.1 - Unreleased
 
   IMPROVEMENTS

Modified: hadoop/common/branches/branch-0.22/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1136258&r1=1136257&r2=1136258&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
(original)
+++ hadoop/common/branches/branch-0.22/hdfs/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
Thu Jun 16 00:24:11 2011
@@ -198,11 +198,17 @@ public class FSEditLog {
         // Remove the directory from list of storage directories
         if(al == null) al = new ArrayList<StorageDirectory>(1);
         al.add(sd);
-        
       }
     }
     
     if(al != null) fsimage.processIOError(al, false);
+    
+    // If there was an error in every storage dir, each one will have
+    // been removed from the list of storage directories.
+    if (fsimage.getNumStorageDirs(NameNodeDirType.EDITS) == 0) {
+      throw new IOException(
+          "Failed to initialize edits log in any storage directory.");
+    }
   }
   
   

Modified: hadoop/common/branches/branch-0.22/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1136258&r1=1136257&r2=1136258&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
(original)
+++ hadoop/common/branches/branch-0.22/hdfs/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
Thu Jun 16 00:24:11 2011
@@ -24,6 +24,9 @@ import java.util.Iterator;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.permission.*;
@@ -38,11 +41,16 @@ import org.apache.hadoop.hdfs.server.nam
 import org.apache.hadoop.metrics.util.MetricsTimeVaryingInt;
  
 import org.mockito.Mockito;
+import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
 
 /**
  * This class tests the creation and validation of a checkpoint.
  */
 public class TestEditLog extends TestCase {
+  private static final Log LOG = LogFactory.getLog(TestEditLog.class);
+  
   static final int NUM_DATA_NODES = 0;
 
   // This test creates NUM_THREADS threads and each thread does
@@ -303,4 +311,63 @@ public class TestEditLog extends TestCas
       if(cluster != null) cluster.shutdown();
     }
   }
+  
+  public void testFailedOpen() throws Exception {
+    Configuration conf = new HdfsConfiguration();
+    MiniDFSCluster cluster = null;
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build();
+    cluster.waitActive();
+    FSNamesystem fsn = cluster.getNamesystem();
+
+    // Set up spys
+    FSImage originalImage = fsn.getFSImage();
+    originalImage.unlockAll();
+    FSImage spyImage = spy(originalImage);
+    
+    FSEditLog editLog = originalImage.getEditLog();
+    FSEditLog spyLog = spy(editLog);
+
+    fsn.dir.fsImage = spyImage;
+    spyImage.setStorageDirectories(
+        FSNamesystem.getNamespaceDirs(conf), 
+        FSNamesystem.getNamespaceEditsDirs(conf));
+    
+    // Fail every attempt to open a new edit file
+    doThrow(new IOException("Injected fault: open")).
+      when(spyLog).addNewEditLogStream((File)anyObject());
+    
+    try {
+      spyLog.close();
+      spyLog.open();
+      fail("open did not fail even when all directories failed!");
+    } catch(IOException ioe) {
+      LOG.info("Got expected exception", ioe);
+    } finally {
+      spyLog.close();
+    }
+    
+    // Reset and try it with a working open
+    editLog.close();
+    originalImage.close();
+    fsn.close();
+    
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_DATA_NODES).build();
+    cluster.waitActive();
+    fsn = cluster.getNamesystem();
+    originalImage = fsn.getFSImage();
+    originalImage.unlockAll();
+    editLog = originalImage.getEditLog();
+    originalImage.setStorageDirectories(
+        FSNamesystem.getNamespaceDirs(conf), 
+        FSNamesystem.getNamespaceEditsDirs(conf));
+    
+    editLog.close();
+    editLog.open();
+    
+    // Close everything off
+    editLog.close();
+    originalImage.close();
+    fsn.close();
+  }
+
 }



Mime
View raw message