hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r549210 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/FSEditLog.java src/java/org/apache/hadoop/dfs/FSNamesystem.java
Date Wed, 20 Jun 2007 19:41:20 GMT
Author: cutting
Date: Wed Jun 20 12:41:19 2007
New Revision: 549210

URL: http://svn.apache.org/viewvc?view=rev&rev=549210
Log:
HADOOP-1003.  Remove flushing of namenode edit log from primary namenode lock, increasing
namenode throughput.  Contributed by Dhruba.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=549210&r1=549209&r2=549210
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed Jun 20 12:41:19 2007
@@ -193,6 +193,10 @@
      "webinterface.private.actions" to enable this.
      (Enis Soztutar via cutting)
 
+ 60. HADOOP-1003.  Remove flushing of namenode edit log from primary
+     namenode lock, increasing namenode throughput.
+     (Dhruba Borthakur via cutting)
+
 
 Release 0.13.0 - 2007-06-08
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java?view=diff&rev=549210&r1=549209&r2=549210
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSEditLog.java Wed Jun 20 12:41:19
2007
@@ -48,6 +48,9 @@
 
   private ArrayList<EditLogOutputStream> editStreams = null;
   private FSImage fsimage = null;
+
+  private long lastModificationTime;
+  private long lastSyncTime;
   
   static class EditLogOutputStream extends DataOutputStream {
     private FileDescriptor fd;
@@ -70,6 +73,8 @@
 
   FSEditLog(FSImage image) {
     fsimage = image;
+    lastModificationTime = 0;
+    lastSyncTime = 0;
   }
 
   private File getEditFile(int idx) {
@@ -345,9 +350,10 @@
   }
 
   /**
-   * Write an operation to the edit log
+   * Write an operation to the edit log. Do not sync to persistent
+   * store yet.
    */
-  void logEdit(byte op, Writable w1, Writable w2) {
+  synchronized void logEdit(byte op, Writable w1, Writable w2) {
     assert this.getNumEditStreams() > 0 : "no editlog streams";
     for (int idx = 0; idx < editStreams.size(); idx++) {
       EditLogOutputStream eStream;
@@ -360,7 +366,6 @@
           if (w2 != null) {
             w2.write(eStream);
           }
-          eStream.flushAndSync();
         } catch (IOException ie) {
           try {
             processIOError(idx);         
@@ -368,6 +373,44 @@
             FSNamesystem.LOG.error("Unable to append to edit log. " +
                                    "Fatal Error.");
             throw new RuntimeException("Unable to append to edit log. ");
+          }
+        }
+      }
+    }
+    //
+    // record the time when new data was written to the edits log
+    //
+    lastModificationTime = System.currentTimeMillis();
+  }
+
+  //
+  // flush all data of the Edits log into persistent store
+  //
+  synchronized void logSync() {
+    assert this.getNumEditStreams() > 0 : "no editlog streams";
+
+    //
+    // If data was generated before the beginning of the last sync time
+    // then there is nothing to flush
+    //
+    if (lastModificationTime < lastSyncTime) {
+      return;
+    }
+    lastSyncTime = System.currentTimeMillis();
+
+    for (int idx = 0; idx < editStreams.size(); idx++) {
+      EditLogOutputStream eStream;
+      synchronized (eStream = editStreams.get(idx)) {
+        try {
+          eStream.flushAndSync();
+        } catch (IOException ie) {
+          try {
+            processIOError(idx);         
+          } catch (IOException e) {
+            FSNamesystem.LOG.error("Unable to sync edit log. " +
+                                   "Fatal Error.");
+            throw new RuntimeException("Unable to sync edit log. " +
+                                       "Fatal Error.");
           }
         }
       }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?view=diff&rev=549210&r1=549209&r2=549210
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Wed Jun 20 12:41:19
2007
@@ -518,7 +518,14 @@
    *         false if file does not exist or is a directory
    * @author shv
    */
-  public synchronized boolean setReplication(String src, 
+  public boolean setReplication(String src, short replication) 
+                                throws IOException {
+    boolean status = setReplicationInternal(src, replication);
+    getEditLog().logSync();
+    return status;
+  }
+
+  private synchronized boolean setReplicationInternal(String src, 
                                              short replication
                                              ) throws IOException {
     if (isInSafeMode())
@@ -867,8 +874,14 @@
    * Before we return, we make sure that all the file's blocks have 
    * been reported by datanodes and are replicated correctly.
    */
-  public synchronized int completeFile(UTF8 src, 
-                                       UTF8 holder) throws IOException {
+  public int completeFile(UTF8 src, UTF8 holder) throws IOException {
+    int status = completeFileInternal(src, holder);
+    getEditLog().logSync();
+    return status;
+  }
+
+  private synchronized int completeFileInternal(UTF8 src, 
+                                                UTF8 holder) throws IOException {
     NameNode.stateChangeLog.debug("DIR* NameSystem.completeFile: " + src + " for " + holder);
     if (isInSafeMode())
       throw new SafeModeException("Cannot complete file " + src, safeMode);
@@ -1081,10 +1094,16 @@
   // are made, edit namespace and return to client.
   ////////////////////////////////////////////////////////////////
 
+  public boolean renameTo(UTF8 src, UTF8 dst) throws IOException {
+    boolean status = renameToInternal(src, dst);
+    getEditLog().logSync();
+    return status;
+  }
+
   /**
    * Change the indicated filename.
    */
-  public synchronized boolean renameTo(UTF8 src, UTF8 dst) throws IOException {
+  private synchronized boolean renameToInternal(UTF8 src, UTF8 dst) throws IOException {
     NameNode.stateChangeLog.debug("DIR* NameSystem.renameTo: " + src + " to " + dst);
     if (isInSafeMode())
       throw new SafeModeException("Cannot rename " + src, safeMode);
@@ -1098,7 +1117,17 @@
    * Remove the indicated filename from the namespace.  This may
    * invalidate some blocks that make up the file.
    */
-  public synchronized boolean delete(UTF8 src) throws IOException {
+  public boolean delete(UTF8 src) throws IOException {
+    boolean status = deleteInternal(src);
+    getEditLog().logSync();
+    return status;
+  }
+
+  /**
+   * Remove the indicated filename from the namespace.  This may
+   * invalidate some blocks that make up the file.
+   */
+  private synchronized boolean deleteInternal(UTF8 src) throws IOException {
     NameNode.stateChangeLog.debug("DIR* NameSystem.delete: " + src);
     if (isInSafeMode())
       throw new SafeModeException("Cannot delete " + src, safeMode);
@@ -1163,11 +1192,19 @@
     }
     return true;
   }
+  /**
+   * Create all the necessary directories
+   */
+  public boolean mkdirs(String src) throws IOException {
+    boolean status = mkdirsInternal(src);
+    getEditLog().logSync();
+    return status;
+  }
     
   /**
    * Create all the necessary directories
    */
-  public synchronized boolean mkdirs(String src) throws IOException {
+  private synchronized boolean mkdirsInternal(String src) throws IOException {
     boolean    success;
     NameNode.stateChangeLog.debug("DIR* NameSystem.mkdirs: " + src);
     if (isInSafeMode())
@@ -1447,7 +1484,15 @@
    * @see DataNode#register()
    * @author Konstantin Shvachko
    */
-  public synchronized void registerDatanode(DatanodeRegistration nodeReg,
+  public void registerDatanode(DatanodeRegistration nodeReg,
+                               String networkLocation
+                               ) throws IOException {
+    registerDatanodeInternal(nodeReg, networkLocation);
+    getEditLog().logSync();
+  }
+
+  private synchronized void registerDatanodeInternal(
+                                            DatanodeRegistration nodeReg,
                                             String networkLocation
                                             ) throws IOException {
 



Mime
View raw message