Author: eli
Date: Fri Jul 20 00:37:09 2012
New Revision: 1363601
URL: http://svn.apache.org/viewvc?rev=1363601&view=rev
Log:
HDFS-3577. In DatanodeWebHdfsMethods, use MessageBodyWriter instead of StreamingOutput, otherwise,
it will fail to transfer large files. Contributed by Tsz Wo (Nicholas), SZE
Added:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/OpenEntity.java
- copied unchanged from r1362976, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/OpenEntity.java
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/ (props changed)
hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
(props changed)
hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
(props changed)
hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
(props changed)
hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
(props changed)
hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
(props changed)
hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
(props changed)
hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
Propchange: hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project:r1362976
Propchange: hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1362976
Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1363601&r1=1363600&r2=1363601&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Fri Jul 20 00:37:09 2012
@@ -1245,6 +1245,10 @@ Release 0.23.3 - UNRELEASED
HDFS-3037. TestMulitipleNNDataBlockScanner#testBlockScannerAfterRestart is
racy. (atm)
+ HDFS-3577. In DatanodeWebHdfsMethods, use MessageBodyWriter instead of
+ StreamingOutput, otherwise, it will fail to transfer large files.
+ (szetszwo)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1362976
Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java?rev=1363601&r1=1363600&r2=1363601&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
(original)
+++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
Fri Jul 20 00:37:09 2012
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.server.da
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
@@ -40,7 +39,6 @@ import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import javax.ws.rs.core.StreamingOutput;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -411,31 +409,10 @@ public class DatanodeWebHdfsMethods {
IOUtils.cleanup(LOG, dfsclient);
throw ioe;
}
- final HdfsDataInputStream dis = in;
- final StreamingOutput streaming = new StreamingOutput() {
- @Override
- public void write(final OutputStream out) throws IOException {
- final Long n = length.getValue();
- HdfsDataInputStream dfsin = dis;
- DFSClient client = dfsclient;
- try {
- if (n == null) {
- IOUtils.copyBytes(dfsin, out, b);
- } else {
- IOUtils.copyBytes(dfsin, out, n, false);
- }
- dfsin.close();
- dfsin = null;
- dfsclient.close();
- client = null;
- } finally {
- IOUtils.cleanup(LOG, dfsin);
- IOUtils.cleanup(LOG, client);
- }
- }
- };
-
- return Response.ok(streaming).type(
+
+ final long n = length.getValue() != null? length.getValue()
+ : in.getVisibleLength();
+ return Response.ok(new OpenEntity(in, n, dfsclient)).type(
MediaType.APPLICATION_OCTET_STREAM).build();
}
case GETFILECHECKSUM:
Propchange: hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1362976
Propchange: hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1362976
Propchange: hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1362976
Propchange: hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1362976
Propchange: hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1362976
Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java?rev=1363601&r1=1363600&r2=1363601&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
(original)
+++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsFileSystemContract.java
Fri Jul 20 00:37:09 2012
@@ -26,6 +26,7 @@ import java.net.HttpURLConnection;
import java.net.URL;
import java.util.Arrays;
import java.util.Map;
+import java.util.Random;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.core.MediaType;
@@ -205,15 +206,20 @@ public class TestWebHdfsFileSystemContra
assertEquals(0, count);
}
+ final byte[] mydata = new byte[1 << 20];
+ new Random().nextBytes(mydata);
+
final Path p = new Path(dir, "file");
- createFile(p);
+ FSDataOutputStream out = fs.create(p, false, 4096, (short)3, 1L << 17);
+ out.write(mydata, 0, mydata.length);
+ out.close();
- final int one_third = data.length/3;
+ final int one_third = mydata.length/3;
final int two_third = one_third*2;
{ //test seek
final int offset = one_third;
- final int len = data.length - offset;
+ final int len = mydata.length - offset;
final byte[] buf = new byte[len];
final FSDataInputStream in = fs.open(p);
@@ -225,13 +231,13 @@ public class TestWebHdfsFileSystemContra
for (int i = 0; i < buf.length; i++) {
assertEquals("Position " + i + ", offset=" + offset + ", length=" + len,
- data[i + offset], buf[i]);
+ mydata[i + offset], buf[i]);
}
}
{ //test position read (read the data after the two_third location)
final int offset = two_third;
- final int len = data.length - offset;
+ final int len = mydata.length - offset;
final byte[] buf = new byte[len];
final FSDataInputStream in = fs.open(p);
@@ -240,7 +246,7 @@ public class TestWebHdfsFileSystemContra
for (int i = 0; i < buf.length; i++) {
assertEquals("Position " + i + ", offset=" + offset + ", length=" + len,
- data[i + offset], buf[i]);
+ mydata[i + offset], buf[i]);
}
}
}
|