hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1295241 - in /hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/server/namenode/ src/test/java/org/apache/hadoop/hdfs/server/namenode/
Date Wed, 29 Feb 2012 19:57:46 GMT
Author: todd
Date: Wed Feb 29 19:57:46 2012
New Revision: 1295241

URL: http://svn.apache.org/viewvc?rev=1295241&view=rev
Log:
HDFS-3020. Fix editlog to automatically sync when buffer is full. Contributed by Todd Lipcon.

Modified:
    hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
    hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java
    hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
    hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java

Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1295241&r1=1295240&r2=1295241&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Feb
29 19:57:46 2012
@@ -131,6 +131,8 @@ Release 0.23.3 - UNRELEASED
     HDFS-2968. Protocol translator for BlockRecoveryCommand broken when
     multiple blocks need recovery. (todd)
 
+    HDFS-3020. Fix editlog to automatically sync when buffer is full. (todd)
+
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java?rev=1295241&r1=1295240&r2=1295241&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EditsDoubleBuffer.java
Wed Feb 29 19:57:46 2012
@@ -86,7 +86,7 @@ class EditsDoubleBuffer {
   }
   
   boolean shouldForceSync() {
-    return bufReady.size() >= initBufferSize;
+    return bufCurrent.size() >= initBufferSize;
   }
 
   DataOutputBuffer getCurrentBuf() {

Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1295241&r1=1295240&r2=1295241&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
Wed Feb 29 19:57:46 2012
@@ -730,6 +730,14 @@ public class FSEditLog  {
   synchronized void setRuntimeForTesting(Runtime runtime) {
     this.runtime = runtime;
   }
+
+  /**
+   * Used only by tests.
+   */
+  @VisibleForTesting
+  void setMetricsForTests(NameNodeMetrics metrics) {
+    this.metrics = metrics;
+  }
   
   /**
    * Return a manifest of what finalized edit logs are available

Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1295241&r1=1295240&r2=1295241&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
Wed Feb 29 19:57:46 2012
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.server.nam
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
 import org.apache.hadoop.hdfs.server.namenode.NNStorage;
+import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.log4j.Level;
@@ -798,6 +799,40 @@ public class TestEditLog extends TestCas
       log.close();
     }
   }
+  
+  /**
+   * Regression test for HDFS-1112/HDFS-3020. Ensures that, even if
+   * logSync isn't called periodically, the edit log will sync itself.
+   */
+  public void testAutoSync() throws Exception {
+    File logDir = new File(TEST_DIR, "testAutoSync");
+    logDir.mkdirs();
+    FSEditLog log = FSImageTestUtil.createStandaloneEditLog(logDir);
+    
+    String oneKB = StringUtils.byteToHexString(
+        new byte[500]);
+    
+    try {
+      log.open();
+      NameNodeMetrics mockMetrics = Mockito.mock(NameNodeMetrics.class);
+      log.setMetricsForTests(mockMetrics);
+
+      for (int i = 0; i < 400; i++) {
+        log.logDelete(oneKB, 1L);
+      }
+      // After ~400KB, we're still within the 512KB buffer size
+      Mockito.verify(mockMetrics, Mockito.times(0)).addSync(Mockito.anyLong());
+      
+      // After ~400KB more, we should have done an automatic sync
+      for (int i = 0; i < 400; i++) {
+        log.logDelete(oneKB, 1L);
+      }
+      Mockito.verify(mockMetrics, Mockito.times(1)).addSync(Mockito.anyLong());
+
+    } finally {
+      log.close();
+    }
+  }
 
   /**
    * Tests the getEditLogManifest function using mock storage for a number



Mime
View raw message