hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1097329 - in /hadoop/hdfs/trunk: ./ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/
Date Thu, 28 Apr 2011 06:00:56 GMT
Author: todd
Date: Thu Apr 28 06:00:55 2011
New Revision: 1097329

URL: http://svn.apache.org/viewvc?rev=1097329&view=rev
Log:
HDFS-1846. Preallocate edit log with OP_INVALID instead of zero bytes to ensure blocks are
actually allocated. Contributed by Aaron T. Myers.

Added:
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java
Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1097329&r1=1097328&r2=1097329&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Thu Apr 28 06:00:55 2011
@@ -143,6 +143,9 @@ Trunk (unreleased changes)
 
     HDFS-1862. Improve test reliability of HDFS-1594. (Aaron T. Myers via eli)
 
+    HDFS-1846. Preallocate edit log with OP_INVALID instead of zero bytes
+    to ensure blocks are actually allocated. (Aaron T. Myers via todd)
+
   OPTIMIZATIONS
 
     HDFS-1458. Improve checkpoint performance by avoiding unnecessary image

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java?rev=1097329&r1=1097328&r2=1097329&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
(original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
Thu Apr 28 06:00:55 2011
@@ -43,7 +43,14 @@ class EditLogFileOutputStream extends Ed
   private DataOutputBuffer bufCurrent; // current buffer for writing
   private DataOutputBuffer bufReady; // buffer ready for flushing
   final private int initBufferSize; // inital buffer size
-  static ByteBuffer fill = ByteBuffer.allocateDirect(512); // preallocation
+  static ByteBuffer fill = ByteBuffer.allocateDirect(1024 * 1024); // preallocation, 1MB
+
+  static {
+    fill.position(0);
+    for (int i = 0; i < fill.capacity(); i++) {
+      fill.put(FSEditLogOpCodes.OP_INVALID.getOpCode());
+    }
+  }
 
   /**
    * Creates output buffers and file object.
@@ -182,12 +189,11 @@ class EditLogFileOutputStream extends Ed
         FSNamesystem.LOG.debug("Preallocating Edit log, current size "
             + fc.size());
       }
-      long newsize = position + 1024 * 1024; // 1MB
       fill.position(0);
-      int written = fc.write(fill, newsize);
+      int written = fc.write(fill, position);
       if(FSNamesystem.LOG.isDebugEnabled()) {
         FSNamesystem.LOG.debug("Edit log size is now " + fc.size() +
-            " written " + written + " bytes " + " at offset " + newsize);
+            " written " + written + " bytes " + " at offset " + position);
       }
     }
   }

Added: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java?rev=1097329&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java
(added)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLogFileOutputStream.java
Thu Apr 28 06:00:55 2011
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.namenode;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.DU;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.junit.Test;
+
+public class TestEditLogFileOutputStream {
+
+  @Test
+  public void testPreallocation() throws IOException {
+    Configuration conf = new Configuration();
+    FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
+    conf.set("dfs.http.address", "127.0.0.1:0");
+    NameNode.format(conf);
+    NameNode nn = new NameNode(conf);
+
+    File editLog = nn.getFSImage().getEditLog().getFsEditName();
+
+    assertEquals("Edit log should only be 4 bytes long",
+        4, editLog.length());
+    assertEquals("Edit log disk space used should be one block",
+        4096, new DU(editLog, conf).getUsed());
+
+    nn.mkdirs("/tmp", new FsPermission((short)777), false);
+
+    assertEquals("Edit log should be 1MB + 4 bytes long",
+        (1024 * 1024) + 4, editLog.length());
+    // 256 blocks for the 1MB of preallocation space, 1 block for the original
+    // 4 bytes
+    assertTrue("Edit log disk space used should be at least 257 blocks",
+        257 * 4096 <= new DU(editLog, conf).getUsed());
+  }
+
+}

Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java?rev=1097329&r1=1097328&r2=1097329&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
(original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
Thu Apr 28 06:00:55 2011
@@ -238,11 +238,11 @@ public class TestOfflineEditsViewer {
     // compares position to limit
     if(!small.equals(large)) { return false; }
 
-    // everything after limit should be zero 
+    // everything after limit should be 0xFF
     int i = large.limit();
     large.clear();
     for(; i < large.capacity(); i++) {
-      if(large.get(i) != 0) {
+      if(large.get(i) != FSEditLogOpCodes.OP_INVALID.getOpCode()) {
         return false;
       }
     }

Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored?rev=1097329&r1=1097328&r2=1097329&view=diff
==============================================================================
Binary files - no diff available.



Mime
View raw message