hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1099911 - in /hadoop/hdfs/branches/HDFS-1073: CHANGES.HDFS-1073.txt src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
Date Thu, 05 May 2011 18:32:11 GMT
Author: todd
Date: Thu May  5 18:32:10 2011
New Revision: 1099911

URL: http://svn.apache.org/viewvc?rev=1099911&view=rev
Log:
HDFS-1892. Fix EditLogFileInputStream.getValidLength to be aware of OP_INVALID filler. Contributed
by Todd Lipcon.

Modified:
    hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt
    hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
    hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java

Modified: hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt?rev=1099911&r1=1099910&r2=1099911&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt (original)
+++ hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt Thu May  5 18:32:10 2011
@@ -17,3 +17,5 @@ HDFS-1858. Add state management variable
 HDFS-1859. Add some convenience functions to iterate over edit log streams
            (Ivan Kelly and Todd Lipcon via todd)
 HDFS-1894. Add constants for LAYOUT_VERSIONs in edits log branch (todd)
+HDFS-1892. Fix EditLogFileInputStream.getValidLength to be aware of OP_INVALID
+           filler (todd)

Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java?rev=1099911&r1=1099910&r2=1099911&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileInputStream.java
Thu May  5 18:32:10 2011
@@ -85,9 +85,10 @@ class EditLogFileInputStream extends Edi
   }
 
   /**
-   * Return the length of non-zero bytes in the given file.
+   * Return the length of bytes in the given file after subtracting
+   * the trailer of 0xFF (OP_INVALID)s.
    * This seeks to the end of the file and reads chunks backwards until
-   * it finds a non-zero byte.
+   * it finds a non-0xFF byte.
    * @param chunkSizeToRead chunk size for disk reads
    * @throws IOException if the file cannot be read
    */
@@ -107,7 +108,7 @@ class EditLogFileInputStream extends Edi
         int readLen = (int) Math.min(size - pos, chunkSizeToRead);
         IOUtils.readFully(fis, buf, 0, readLen);
         for (int i = readLen - 1; i >= 0; i--) {
-          if (buf[i] != 0) {
+          if (buf[i] != FSEditLogOpCodes.OP_INVALID.getOpCode()) {
             return pos + i + 1; // + 1 since we count this byte!
           }
         }

Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1099911&r1=1099910&r2=1099911&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
Thu May  5 18:32:10 2011
@@ -21,6 +21,7 @@ import junit.framework.TestCase;
 import java.io.*;
 import java.net.URI;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ExecutorService;
@@ -79,6 +80,8 @@ public class TestEditLog extends TestCas
         "a4ff 0000 0000 0000 0000 0000 0000 0000"
     ).replace(" ",""));
 
+  
+  static final byte TRAILER_BYTE = FSEditLogOpCodes.OP_INVALID.getOpCode();
   //
   // an object that does a bunch of transactions
   //
@@ -410,42 +413,43 @@ public class TestEditLog extends TestCas
     assertTrue(TEST_DIR.mkdirs() || TEST_DIR.exists());
 
     // For each of a combination of valid bytes followed by invalid,
-    // make sure we determine the proper non-zero size of the file.
+    // make sure we determine the proper non-trailer size of the file.
     // Sizes designed to find off-by-one errors around 1024 byte chunk size.
     final int VALID_SIZES[] = new int[] {0, 1, 789, 1023, 1024, 1025};
-    final int ZERO_SIZES[] = new int[] {0, 1, 1024-789-1, 1024-789,
+    final int TRAILER_SIZES[] = new int[] {0, 1, 1024-789-1, 1024-789,
         1024-789+1, 1023, 1024, 1025};
     
     for (int validPart : VALID_SIZES) {
-      for (int zeroPart : ZERO_SIZES) {
-        doValidLengthTest(validPart, zeroPart);
-        doValidLengthTest(validPart*3, zeroPart*3);
+      for (int trailerPart : TRAILER_SIZES) {
+        doValidLengthTest(validPart, trailerPart);
+        doValidLengthTest(validPart*3, trailerPart*3);
       }
     }
   }
 
-  private void doValidLengthTest(int validPart, int zeroPart) throws Exception {
+  private void doValidLengthTest(int validPart, int trailerPart) throws Exception {
     File file = new File(TEST_DIR, "validLengthTest");
     FileOutputStream fos = new FileOutputStream(file);
     try {
       try {
         byte[] valid = new byte[validPart];
         for (int i = 0; i < validPart; i++) {
-          valid[i] = (byte)(i % 10); // make sure we cycle through 0 bytes occasionally
+          valid[i] = (byte)(i & 0xFF);
         }
-        // The valid data shouldn't end in a 0, or else we'd think it was one shorter
+        // The valid data shouldn't end in a trailer byte, or else we'd think it was one
shorter
         // than actual length
         if (validPart > 0) valid[validPart - 1] = 1;
         fos.write(valid);
         
-        byte[] zeros = new byte[zeroPart];
-        fos.write(zeros);
+        byte[] trailer = new byte[trailerPart];
+        Arrays.fill(trailer, TRAILER_BYTE);
+        fos.write(trailer);
       } finally {
         fos.close();
       }
       
       long computedValid = EditLogFileInputStream.getValidLength(file, 1024);
-      assertEquals("Testing valid=" + validPart + " zero=" + zeroPart,
+      assertEquals("Testing valid=" + validPart + " trailer=" + trailerPart,
           validPart, computedValid);
     } finally {
       file.delete();



Mime
View raw message