hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r546297 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/fs/ChecksumFileSystem.java
Date Mon, 11 Jun 2007 21:38:40 GMT
Author: cutting
Date: Mon Jun 11 14:38:40 2007
New Revision: 546297

URL: http://svn.apache.org/viewvc?view=rev&rev=546297
Log:
HADOOP-1450.  Move checksumming closer to user code.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=546297&r1=546296&r2=546297
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Mon Jun 11 14:38:40 2007
@@ -99,6 +99,11 @@
  32. HADOOP-1311.  Fix a bug in BytesWritable#set() where start offset
      was ignored.  (Dhruba Borthakur via cutting)
 
+ 33. HADOOP-1450.  Move checksumming closer to user code, so that
+     checksums are created before data is stored in large buffers and
+     verified after data is read from large buffers, to better catch
+     memory errors.  (cutting)
+
 
 Release 0.13.0 - 2007-06-08
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java?view=diff&rev=546297&r1=546296&r2=546297
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java Mon Jun 11 14:38:40
2007
@@ -109,9 +109,7 @@
     
     public FSInputChecker(ChecksumFileSystem fs, Path file, int bufferSize)
       throws IOException {
-      // open with an extremly small buffer size,
-      // so that the buffer could be by-passed by the buffer in FSDataInputStream
-      datas = fs.getRawFileSystem().open(file, 1);
+      datas = fs.getRawFileSystem().open(file, bufferSize);
       this.fs = fs;
       this.file = file;
       Path sumFile = fs.getChecksumFile(file);
@@ -338,7 +336,7 @@
       throw new FileNotFoundException(f.toString());
     }
     return new FSDataInputStream(new FSInputChecker(this, f, bufferSize),
-                                 bufferSize);
+                                 getBytesPerSum());
   }
 
   /** This class provides an output stream for a checksummed file.
@@ -371,7 +369,7 @@
                           long blockSize,
                           Progressable progress)
       throws IOException {
-      super(fs.getRawFileSystem().create(file, overwrite, 1, 
+      super(fs.getRawFileSystem().create(file, overwrite, bufferSize, 
                                          replication, blockSize, progress));
       this.bytesPerSum = fs.getBytesPerSum();
       int sumBufferSize = fs.getSumBufferSize(bytesPerSum, bufferSize);
@@ -447,7 +445,7 @@
       throw new IOException("Mkdirs failed to create " + parent);
     }
     return new FSDataOutputStream(new FSOutputSummer(this, f, overwrite,
-                                                     bufferSize, replication, blockSize,
progress), bufferSize);
+                                                     bufferSize, replication, blockSize,
progress), getBytesPerSum());
   }
 
   /**



Mime
View raw message