hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1349025 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/io/IOUtils.java src/test/java/org/apache/hadoop/io/TestIOUtils.java
Date Mon, 11 Jun 2012 21:26:07 GMT
Author: eli
Date: Mon Jun 11 21:26:06 2012
New Revision: 1349025

URL: http://svn.apache.org/viewvc?rev=1349025&view=rev
Log:
HADOOP-8491. Check for short writes when using FileChannel#write and related methods. Contributed
by Colin Patrick McCabe

Modified:
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
    hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1349025&r1=1349024&r2=1349025&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Mon Jun
11 21:26:06 2012
@@ -80,6 +80,12 @@ Release 2.0.1-alpha - UNRELEASED
 
     HADOOP-8485. Don't hardcode "Apache Hadoop 0.23" in the docs. (eli)
 
+    HADOOP-8488. test-patch.sh gives +1 even if the native build fails.
+    (Colin Patrick McCabe via eli)
+
+    HADOOP-8491. Check for short writes when using FileChannel#write
+    and related methods. (Colin Patrick McCabe via eli)
+
   BREAKDOWN OF HDFS-3042 SUBTASKS
 
     HADOOP-8220. ZKFailoverController doesn't handle failure to become active
@@ -112,9 +118,6 @@ Release 2.0.1-alpha - UNRELEASED
     
     HADOOP-8405. ZKFC tests leak ZK instances. (todd)
 
-    HADOOP-8488. test-patch.sh gives +1 even if the native build
-    fails. (Colin Patrick McCabe via eli)
-
 Release 2.0.0-alpha - 05-23-2012
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java?rev=1349025&r1=1349024&r2=1349025&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/IOUtils.java
Mon Jun 11 21:26:06 2012
@@ -20,6 +20,9 @@ package org.apache.hadoop.io;
 
 import java.io.*;
 import java.net.Socket;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.WritableByteChannel;
 
 import org.apache.commons.logging.Log;
 
@@ -245,4 +248,35 @@ public class IOUtils {
     public void write(int b) throws IOException {
     }
   }  
+  
+  /**
+   * Write a ByteBuffer to a WritableByteChannel, handling short writes.
+   * 
+   * @param bc               The WritableByteChannel to write to.
+   * @param buf              The input buffer
+   * @param offset           The offset in the file to start writing at.
+   * @throws IOException     On I/O error.
+   */
+  public static void writeFully(WritableByteChannel bc, ByteBuffer buf)
+      throws IOException {
+    do {
+      bc.write(buf);
+    } while (buf.remaining() > 0);
+  }
+
+  /**
+   * Write a ByteBuffer to a FileChannel at a given offset, 
+   * handling short writes.
+   * 
+   * @param fc               The FileChannel to write to.
+   * @param buf              The input buffer
+   * @param offset           The offset in the file to start writing at.
+   * @throws IOException     On I/O error.
+   */
+  public static void writeFully(FileChannel fc, ByteBuffer buf,
+      long offset) throws IOException {
+    do {
+      offset += fc.write(buf, offset);
+    } while (buf.remaining() > 0);
+  }
 }

Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java?rev=1349025&r1=1349024&r2=1349025&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestIOUtils.java
Mon Jun 11 21:26:06 2012
@@ -21,9 +21,13 @@ package org.apache.hadoop.io;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
 
 import org.junit.Test;
 import org.mockito.Mockito;
@@ -32,7 +36,8 @@ import org.mockito.Mockito;
  * Test cases for IOUtils.java
  */
 public class TestIOUtils {
-
+  private static final String TEST_FILE_NAME = "test_file";
+  
   @Test
   public void testCopyBytesShouldCloseStreamsWhenCloseIsTrue() throws Exception {
     InputStream inputStream = Mockito.mock(InputStream.class);
@@ -110,4 +115,41 @@ public class TestIOUtils {
     Mockito.verify(outputStream, Mockito.atLeastOnce()).close();
   }
   
+  @Test
+  public void testWriteFully() throws IOException {
+    final int INPUT_BUFFER_LEN = 10000;
+    final int HALFWAY = 1 + (INPUT_BUFFER_LEN / 2);
+    byte[] input = new byte[INPUT_BUFFER_LEN];
+    for (int i = 0; i < input.length; i++) {
+      input[i] = (byte)(i & 0xff);
+    }
+    byte[] output = new byte[input.length];
+    
+    try {
+      RandomAccessFile raf = new RandomAccessFile(TEST_FILE_NAME, "rw");
+      FileChannel fc = raf.getChannel();
+      ByteBuffer buf = ByteBuffer.wrap(input);
+      IOUtils.writeFully(fc, buf);
+      raf.seek(0);
+      raf.read(output);
+      for (int i = 0; i < input.length; i++) {
+        assertEquals(input[i], output[i]);
+      }
+      buf.rewind();
+      IOUtils.writeFully(fc, buf, HALFWAY);
+      for (int i = 0; i < HALFWAY; i++) {
+        assertEquals(input[i], output[i]);
+      }
+      raf.seek(0);
+      raf.read(output);
+      for (int i = HALFWAY; i < input.length; i++) {
+        assertEquals(input[i - HALFWAY], output[i]);
+      }
+    } finally {
+      File f = new File(TEST_FILE_NAME);
+      if (f.exists()) {
+        f.delete();
+      }
+    }
+  }
 }



Mime
View raw message