hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r558832 - in /lucene/hadoop/branches/branch-0.14: ./ src/java/org/apache/hadoop/dfs/ src/java/org/apache/hadoop/fs/ src/test/org/apache/hadoop/dfs/
Date Mon, 23 Jul 2007 19:00:24 GMT
Author: cutting
Date: Mon Jul 23 12:00:23 2007
New Revision: 558832

URL: http://svn.apache.org/viewvc?view=rev&rev=558832
Log:
Merge -r 558830:558831 from trunk to 0.14 branch.  Fixes: HADOOP-1619.

Modified:
    lucene/hadoop/branches/branch-0.14/CHANGES.txt
    lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/dfs/DFSClient.java
    lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java
    lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/FSInputChecker.java
    lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestFSInputChecker.java

Modified: lucene/hadoop/branches/branch-0.14/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/CHANGES.txt?view=diff&rev=558832&r1=558831&r2=558832
==============================================================================
--- lucene/hadoop/branches/branch-0.14/CHANGES.txt (original)
+++ lucene/hadoop/branches/branch-0.14/CHANGES.txt Mon Jul 23 12:00:23 2007
@@ -401,6 +401,9 @@
 134. HADOOP-1632.  Fix an IllegalArgumentException in fsck.
      (Hairong Kuang via cutting)
 
+135. HADOOP-1619.  Fix FSInputChecker to not attempt to read past EOF.
+     (Hairong Kuang via cutting)
+
 
 Release 0.13.0 - 2007-06-08
 

Modified: lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/dfs/DFSClient.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/dfs/DFSClient.java?view=diff&rev=558832&r1=558831&r2=558832
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/dfs/DFSClient.java (original)
+++ lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/dfs/DFSClient.java Mon Jul
23 12:00:23 2007
@@ -1264,7 +1264,12 @@
      
     public long skip(long n) throws IOException {
       if ( n > 0 ) {
-        seek(getPos()+n);
+        long curPos = getPos();
+        long fileLen = getFileLength();
+        if( n+curPos > fileLen ) {
+          n = fileLen - curPos;
+        }
+        seek(curPos+n);
         return n;
       }
       return n < 0 ? -1 : 0;

Modified: lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java?view=diff&rev=558832&r1=558831&r2=558832
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java
(original)
+++ lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/ChecksumFileSystem.java
Mon Jul 23 12:00:23 2007
@@ -96,6 +96,7 @@
     private static final int HEADER_LENGTH = 8;
     
     private int bytesPerSum = 1;
+    private long fileLen = -1L;
     
     public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file)
       throws IOException {
@@ -202,6 +203,57 @@
       }
       return nread;
     }
+    
+    /* Return the file length */
+    private long getFileLength() throws IOException {
+      if( fileLen==-1L ) {
+        fileLen = fs.getContentLength(file);
+      }
+      return fileLen;
+    }
+    
+    /**
+     * Skips over and discards <code>n</code> bytes of data from the
+     * input stream.
+     *
+     *The <code>skip</code> method skips over some smaller number of bytes
+     * when reaching end of file before <code>n</code> bytes have been skipped.
+     * The actual number of bytes skipped is returned.  If <code>n</code> is
+     * negative, no bytes are skipped.
+     *
+     * @param      n   the number of bytes to be skipped.
+     * @return     the actual number of bytes skipped.
+     * @exception  IOException  if an I/O error occurs.
+     *             ChecksumException if the chunk to skip to is corrupted
+     */
+    public synchronized long skip(long n) throws IOException {
+      long curPos = getPos();
+      long fileLength = getFileLength();
+      if( n+curPos > fileLength ) {
+        n = fileLength - curPos;
+      }
+      return super.skip(n);
+    }
+    
+    /**
+     * Seek to the given position in the stream.
+     * The next read() will be from that position.
+     * 
+     * <p>This method does not allow seek past the end of the file.
+     * This produces IOException.
+     *
+     * @param      pos   the postion to seek to.
+     * @exception  IOException  if an I/O error occurs or seeks after EOF
+     *             ChecksumException if the chunk to seek to is corrupted
+     */
+
+    public synchronized void seek(long pos) throws IOException {
+      if(pos>getFileLength()) {
+        throw new IOException("Cannot seek after EOF");
+      }
+      super.seek(pos);
+    }
+
   }
 
   /**

Modified: lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/FSInputChecker.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/FSInputChecker.java?view=diff&rev=558832&r1=558831&r2=558832
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/FSInputChecker.java (original)
+++ lucene/hadoop/branches/branch-0.14/src/java/org/apache/hadoop/fs/FSInputChecker.java Mon
Jul 23 12:00:23 2007
@@ -297,7 +297,23 @@
     return count-pos;
   }
   
-  @Override
+  /**
+   * Skips over and discards <code>n</code> bytes of data from the
+   * input stream.
+   *
+   * <p>This method may skip more bytes than are remaining in the backing
+   * file. This produces no exception and the number of bytes skipped
+   * may include some number of bytes that were beyond the EOF of the
+   * backing file. Attempting to read from the stream after skipping past
+   * the end will result in -1 indicating the end of the file.
+   *
+   *<p>If <code>n</code> is negative, no bytes are skipped.
+   *
+   * @param      n   the number of bytes to be skipped.
+   * @return     the actual number of bytes skipped.
+   * @exception  IOException  if an I/O error occurs.
+   *             ChecksumException if the chunk to skip to is corrupted
+   */
   public synchronized long skip(long n) throws IOException {
     if (n <= 0) {
       return 0;
@@ -307,7 +323,19 @@
     return n;
   }
 
-  @Override
+  /**
+   * Seek to the given position in the stream.
+   * The next read() will be from that position.
+   * 
+   * <p>This method may seek past the end of the file.
+   * This produces no exception and an attempt to read from
+   * the stream will result in -1 indicating the end of the file.
+   *
+   * @param      pos   the postion to seek to.
+   * @exception  IOException  if an I/O error occurs.
+   *             ChecksumException if the chunk to seek to is corrupted
+   */
+
   public synchronized void seek(long pos) throws IOException {
     if( pos<0 ) {
       return;
@@ -328,10 +356,7 @@
     // scan to the desired position
     int delta = (int)(pos - chunkPos);
     if( delta > 0) {
-      int nread = readFully(this, new byte[delta], 0, delta);
-      if (nread < delta) {
-        throw new IOException("Cannot seek after EOF");
-      }
+      readFully(this, new byte[delta], 0, delta);
     }
   }
 

Modified: lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestFSInputChecker.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestFSInputChecker.java?view=diff&rev=558832&r1=558831&r2=558832
==============================================================================
--- lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestFSInputChecker.java
(original)
+++ lucene/hadoop/branches/branch-0.14/src/test/org/apache/hadoop/dfs/TestFSInputChecker.java
Mon Jul 23 12:00:23 2007
@@ -181,6 +181,12 @@
     
     stm.seek(0);
     assertEquals(stm.skip(FILE_SIZE), FILE_SIZE);
+    assertEquals(stm.skip(10), 0);
+    
+    stm.seek(0);
+    assertEquals(stm.skip(FILE_SIZE+10), FILE_SIZE);
+    stm.seek(10);
+    assertEquals(stm.skip(FILE_SIZE), FILE_SIZE-10);
   }
 
   private void cleanupFile(FileSystem fileSys, Path name) throws IOException {



Mime
View raw message