hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cmcc...@apache.org
Subject svn commit: r1575798 - in /hadoop/common/branches/branch-2.4/hadoop-hdfs-project: ./ hadoop-hdfs/ hadoop-hdfs/src/main/java/ hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/
Date Mon, 10 Mar 2014 02:48:05 GMT
Author: cmccabe
Date: Mon Mar 10 02:48:04 2014
New Revision: 1575798

URL: http://svn.apache.org/r1575798
Log:
HDFS-6071. BlockReaderLocal does not return -1 on EOF when doing a zero-length read on a short
file. (cmccabe)

Added:
    hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java
      - copied unchanged from r1575797, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestRead.java
Modified:
    hadoop/common/branches/branch-2.4/hadoop-hdfs-project/   (props changed)
    hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/   (props changed)
    hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
    hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/   (props
changed)
    hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
    hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java

Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project:r1575797

Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1575797

Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1575798&r1=1575797&r2=1575798&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Mar
10 02:48:04 2014
@@ -320,6 +320,9 @@ Release 2.4.0 - UNRELEASED
 
     HDFS-6078. TestIncrementalBlockReports is flaky. (Arpit Agarwal)
 
+    HDFS-6071. BlockReaderLocal doesn't return -1 on EOF when doing a
+    zero-length read on a short file (cmccabe)
+
   BREAKDOWN OF HDFS-5698 SUBTASKS AND RELATED JIRAS
 
     HDFS-5717. Save FSImage header in protobuf. (Haohui Mai via jing9)

Propchange: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1575797

Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java?rev=1575798&r1=1575797&r2=1575798&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
(original)
+++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/BlockReaderLocal.java
Mon Mar 10 02:48:04 2014
@@ -478,7 +478,7 @@ class BlockReaderLocal implements BlockR
       total += bb;
       if (buf.remaining() == 0) return total;
     }
-    boolean eof = false;
+    boolean eof = true, done = false;
     do {
       if (buf.isDirect() && (buf.remaining() >= maxReadaheadLength)
             && ((dataPos % bytesPerChecksum) == 0)) {
@@ -493,20 +493,24 @@ class BlockReaderLocal implements BlockR
           buf.limit(oldLimit);
         }
         if (nRead < maxReadaheadLength) {
-          eof = true;
+          done = true;
+        }
+        if (nRead > 0) {
+          eof = false;
         }
         total += nRead;
       } else {
         // Slow lane: refill bounce buffer.
         if (fillDataBuf(canSkipChecksum)) {
-          eof = true;
+          done = true;
         }
         bb = drainDataBuf(buf); // drain bounce buffer if possible
         if (bb >= 0) {
+          eof = false;
           total += bb;
         }
       }
-    } while ((!eof) && (buf.remaining() > 0));
+    } while ((!done) && (buf.remaining() > 0));
     return (eof && total == 0) ? -1 : total;
   }
 

Modified: hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java?rev=1575798&r1=1575797&r2=1575798&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
(original)
+++ hadoop/common/branches/branch-2.4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
Mon Mar 10 02:48:04 2014
@@ -57,11 +57,14 @@ import org.apache.hadoop.hdfs.server.pro
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.io.nativeio.NativeIO;
 import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.unix.DomainSocket;
+import org.apache.hadoop.net.unix.TemporarySocketDirectory;
 import org.apache.hadoop.security.ShellBasedUnixGroupsMapping;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.apache.hadoop.util.VersionInfo;
+import org.junit.Assume;
 
 import java.io.*;
 import java.net.*;
@@ -1157,4 +1160,43 @@ public class DFSTestUtil {
     long c = (val + factor - 1) / factor;
     return c * factor;
   }
+  
+  /**
+   * A short-circuit test context which makes it easier to get a short-circuit
+   * configuration and set everything up.
+   */
+  public static class ShortCircuitTestContext implements Closeable {
+    private final String testName;
+    private final TemporarySocketDirectory sockDir;
+    private boolean closed = false;
+    private boolean formerTcpReadsDisabled;
+    
+    public ShortCircuitTestContext(String testName) {
+      this.testName = testName;
+      this.sockDir = new TemporarySocketDirectory();
+      DomainSocket.disableBindPathValidation();
+      formerTcpReadsDisabled = DFSInputStream.tcpReadsDisabledForTesting;
+      Assume.assumeTrue(DomainSocket.getLoadingFailureReason() == null);
+    }
+    
+    public Configuration newConfiguration() {
+      Configuration conf = new Configuration();
+      conf.setBoolean(DFSConfigKeys.DFS_CLIENT_READ_SHORTCIRCUIT_KEY, true);
+      conf.set(DFSConfigKeys.DFS_DOMAIN_SOCKET_PATH_KEY,
+          new File(sockDir.getDir(),
+            testName + "._PORT.sock").getAbsolutePath());
+      return conf;
+    }
+
+    public String getTestName() {
+      return testName;
+    }
+
+    public void close() throws IOException {
+      if (closed) return;
+      closed = true;
+      DFSInputStream.tcpReadsDisabledForTesting = formerTcpReadsDisabled;
+      sockDir.close();
+    }
+  }
 }



Mime
View raw message