hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1124518 - in /hadoop/hdfs/branches/HDFS-1073: ./ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/
Date Thu, 19 May 2011 02:37:38 GMT
Author: todd
Date: Thu May 19 02:37:37 2011
New Revision: 1124518

URL: http://svn.apache.org/viewvc?rev=1124518&view=rev
Log:
HDFS-1926. Remove references to StorageDirectory from JournalManager interface. Contributed
by Ivan Kelly.

Modified:
    hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt
    hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
    hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
    hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
    hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
    hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
    hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java

Modified: hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt?rev=1124518&r1=1124517&r2=1124518&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt (original)
+++ hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt Thu May 19 02:37:37 2011
@@ -27,3 +27,5 @@ HDFS-1800. Extend image checksumming to 
            per directory. (todd)
 HDFS-1725. Set storage directories only at FSImage construction (Ivan Kelly
            via todd)
+HDFS-1926. Remove references to StorageDirectory from JournalManager interface
+           (Ivan Kelly via todd)

Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java?rev=1124518&r1=1124517&r2=1124518&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
Thu May 19 02:37:37 2011
@@ -92,6 +92,9 @@ class EditLogFileOutputStream extends Ed
   /** {@inheritDoc} */
   @Override
   public void write(int b) throws IOException {
+    if (fp == null) {
+      throw new IOException("Trying to use aborted output stream");
+    }
     bufCurrent.write(b);
   }
 
@@ -105,6 +108,9 @@ class EditLogFileOutputStream extends Ed
    * */
   @Override
   void write(byte op, long txid, Writable... writables) throws IOException {
+    if (fp == null) {
+      throw new IOException("Trying to use aborted output stream");
+    }
     int start = bufCurrent.getLength();
     write(op);
     bufCurrent.writeLong(txid);
@@ -134,6 +140,10 @@ class EditLogFileOutputStream extends Ed
 
   @Override
   public void close() throws IOException {
+    if (fp == null) {
+      throw new IOException("Trying to use aborted output stream");
+    }
+
     setReadyToFlush();
     flush();
     
@@ -157,6 +167,9 @@ class EditLogFileOutputStream extends Ed
   
   @Override
   public void abort() throws IOException {
+    if (fp == null) {
+      return;
+    }
     IOUtils.cleanup(LOG, fp);
     fp = null;
   }
@@ -180,6 +193,10 @@ class EditLogFileOutputStream extends Ed
    */
   @Override
   protected void flushAndSync() throws IOException {
+    if (fp == null) {
+      throw new IOException("Trying to use aborted output stream");
+    }
+    
     preallocate(); // preallocate file if necessary
     bufReady.writeTo(fp); // write data to file
     bufReady.reset(); // erase all data in the buffer

Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1124518&r1=1124517&r2=1124518&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
Thu May 19 02:37:37 2011
@@ -61,7 +61,7 @@ import static org.apache.hadoop.hdfs.ser
  */
 @InterfaceAudience.Private
 @InterfaceStability.Evolving
-public class FSEditLog implements NNStorageListener {
+public class FSEditLog  {
 
   static final String NO_JOURNAL_STREAMS_WARNING = "!!! WARNING !!!" +
       " File system changes are not persistent. No journal streams.";
@@ -135,7 +135,6 @@ public class FSEditLog implements NNStor
   FSEditLog(NNStorage storage) {
     isSyncRunning = false;
     this.storage = storage;
-    this.storage.registerListener(this);
     metrics = NameNode.getNameNodeMetrics();
     lastPrintTime = now();
   }
@@ -1056,70 +1055,20 @@ public class FSEditLog implements NNStor
    * Called when some journals experience an error in some operation.
    * This propagates errors to the storage level.
    */
-  void disableAndReportErrorOnJournals(List<JournalAndStream> badJournals) {
+  private void disableAndReportErrorOnJournals(List<JournalAndStream> badJournals)
{
     if (badJournals == null || badJournals.isEmpty()) {
       return; // nothing to do
     }
-
-    ArrayList<StorageDirectory> errorDirs = new ArrayList<StorageDirectory>();
+ 
     for (JournalAndStream j : badJournals) {
       LOG.error("Disabling journal " + j);
-      StorageDirectory sd = j.getStorageDirectory();
-      if (sd != null) {
-        errorDirs.add(sd);
-        // We will report this error to storage, which will propagate back
-        // to our listener interface, at which point we'll mark it faulty.
-      } else {
-        // Just mark it faulty ourselves, since it's not associated with a
-        // storage directory.
-        markJournalFaulty(j);
-      }
-    }
-  }
-
-  private synchronized void markJournalFaulty(JournalAndStream jas) {
-    try {
-      jas.abort();
-    } catch (IOException e) {
-      LOG.warn("Failed to abort faulty journal " + jas
-          + " before removing it (might be OK)", e);
-    }
-  }
-
-  /**
-   * Error Handling on a storageDirectory
-   *
-   */
-  // NNStorageListener Interface
-  @Override // NNStorageListener
-  public synchronized void errorOccurred(StorageDirectory sd)
-      throws IOException {
-    
-    LOG.debug("Error occurred on " + sd);
-    
-    for (JournalAndStream jas : journals) {
-      if (jas.getStorageDirectory() == sd) {
-        LOG.warn("Marking corresponding journal " + jas + " faulty");
-        markJournalFaulty(jas);
-        return;
+      try {
+        j.abort();
+      } catch (IOException ioe) {
+        LOG.warn("Failed to abort faulty journal " + j
+                 + " before removing it (might be OK)", ioe);
       }
     }
-    
-    LOG.debug("Faulty " + sd + " did not correspond to any live journal manager.");
-  }
-
-  @Override // NNStorageListener
-  public synchronized void formatOccurred(StorageDirectory sd)
-      throws IOException {
-    if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) {
-      createEditLogFile(NNStorage.getStorageFile(sd, NameNodeFile.EDITS));
-    }
-  };
-
-  @Override // NNStorageListener
-  public synchronized void directoryAvailable(StorageDirectory sd)
-      throws IOException {
-    // We'll always just check all of the journals every time we roll.
   }
 
   /**
@@ -1156,10 +1105,6 @@ public class FSEditLog implements NNStor
       stream = manager.createRevertedStream(source);
     }
 
-    public StorageDirectory getStorageDirectory() {
-      return manager.getStorageDirectory();
-    }
-
     public void close() throws IOException {
       if (stream == null) return;
       stream.close();
@@ -1191,8 +1136,13 @@ public class FSEditLog implements NNStor
     }
 
     @VisibleForTesting
-    void setCurrentStreamForTests(EditLogFileOutputStream stream) {
+    void setCurrentStreamForTests(EditLogOutputStream stream) {
       this.stream = stream;
     }
+    
+    @VisibleForTesting
+    JournalManager getManager() {
+      return manager;
+    }
   }
 }

Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1124518&r1=1124517&r2=1124518&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Thu May 19 02:37:37 2011
@@ -1196,6 +1196,10 @@ public class FSImage implements NNStorag
       }
       LOG.info("Storage directory " + sd.getRoot()
                + " has been successfully formatted.");
+    } else if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) {
+      // TODO this goes away with HDFS-1073
+      File eFile = NNStorage.getEditFile(sd);
+      editLog.createEditLogFile(eFile);
     }
   };
 

Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java?rev=1124518&r1=1124517&r2=1124518&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FileJournalManager.java
Thu May 19 02:37:37 2011
@@ -87,7 +87,7 @@ public class FileJournalManager implemen
     return createStream();
   }
 
-  @Override
+  @VisibleForTesting
   public StorageDirectory getStorageDirectory() {
     return sd;
   }

Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java?rev=1124518&r1=1124517&r2=1124518&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/JournalManager.java
Thu May 19 02:37:37 2011
@@ -19,8 +19,6 @@ package org.apache.hadoop.hdfs.server.na
 
 import java.io.IOException;
 
-import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
-
 /**
  * A JournalManager is responsible for managing a single place of storing
  * edit logs. It may correspond to multiple files, a backup node, etc.
@@ -29,13 +27,6 @@ import org.apache.hadoop.hdfs.server.com
  * this class, which is created when the EditLog is first opened.
  */
 public interface JournalManager {
-
-  /**
-   * @return the StorageDirectory associated with this journal,
-   * or null if this is not a disk-based journal 
-   */
-  StorageDirectory getStorageDirectory();
-
   /**
    * Prepare the stream to write edits
    */

Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java?rev=1124518&r1=1124517&r2=1124518&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestStorageRestore.java
Thu May 19 02:37:37 2011
@@ -29,6 +29,10 @@ import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.Random;
 
+import static org.mockito.Matchers.anyByte;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.spy;
+
 import junit.framework.TestCase;
 
 import org.apache.commons.logging.Log;
@@ -127,6 +131,19 @@ public class TestStorageRestore extends 
     }
     // simulate an error
     fi.getStorage().reportErrorsOnDirectories(al);
+    
+    for (FSEditLog.JournalAndStream j : fi.getEditLog().getJournals()) {
+      if (j.getManager() instanceof FileJournalManager) {
+        FileJournalManager fm = (FileJournalManager)j.getManager();
+        if (fm.getStorageDirectory().getRoot().equals(path2)
+            || fm.getStorageDirectory().getRoot().equals(path3)) {
+          EditLogOutputStream mockStream = spy(j.getCurrentStream());
+          j.setCurrentStreamForTests(mockStream);
+          doThrow(new IOException("Injected fault: write")).
+            when(mockStream).write(anyByte());
+        }
+      }
+    }
   }
   
   /**



Mime
View raw message