Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 62446178A7 for ; Mon, 23 Mar 2015 21:26:22 +0000 (UTC) Received: (qmail 19424 invoked by uid 500); 23 Mar 2015 21:26:12 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 19357 invoked by uid 500); 23 Mar 2015 21:26:11 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 19346 invoked by uid 99); 23 Mar 2015 21:26:11 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 23 Mar 2015 21:26:11 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A6F5CE1823; Mon, 23 Mar 2015 21:26:11 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aajisaka@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-7881. TestHftpFileSystem#testSeek fails in branch-2. Contributed by Brahma Reddy Battula. Date: Mon, 23 Mar 2015 21:26:11 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2.7 57e297208 -> 2742f12b5 HDFS-7881. TestHftpFileSystem#testSeek fails in branch-2. Contributed by Brahma Reddy Battula. (cherry picked from commit fad8c78173c4b7c55324033720f04a09943deac7) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2742f12b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2742f12b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2742f12b Branch: refs/heads/branch-2.7 Commit: 2742f12b58f27892374c4fcf9dfb60772365da1e Parents: 57e2972 Author: Akira Ajisaka Authored: Tue Mar 24 06:21:14 2015 +0900 Committer: Akira Ajisaka Committed: Tue Mar 24 06:25:21 2015 +0900 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/web/ByteRangeInputStream.java | 38 ++++++++++++++++---- 2 files changed, 35 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2742f12b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b83c9a6..9155772 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -906,6 +906,9 @@ Release 2.7.0 - UNRELEASED HDFS-7942. NFS: support regexp grouping in nfs.exports.allowed.hosts (brandonli) + HDFS-7881. TestHftpFileSystem#testSeek fails in branch-2. + (Brahma Reddy Battula via aajisaka) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode http://git-wip-us.apache.org/repos/asf/hadoop/blob/2742f12b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java index 395c9f6..9e3b29a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java @@ -28,6 +28,7 @@ import java.util.StringTokenizer; import org.apache.commons.io.input.BoundedInputStream; import org.apache.hadoop.fs.FSInputStream; +import org.apache.http.HttpStatus; import com.google.common.annotations.VisibleForTesting; import com.google.common.net.HttpHeaders; @@ -127,12 +128,7 @@ public abstract class ByteRangeInputStream extends FSInputStream { fileLength = null; } else { // for non-chunked transfer-encoding, get content-length - final String cl = connection.getHeaderField(HttpHeaders.CONTENT_LENGTH); - if (cl == null) { - throw new IOException(HttpHeaders.CONTENT_LENGTH + " is missing: " - + headers); - } - final long streamlength = Long.parseLong(cl); + long streamlength = getStreamLength(connection, headers); fileLength = startPos + streamlength; // Java has a bug with >2GB request streams. It won't bounds check @@ -143,6 +139,36 @@ public abstract class ByteRangeInputStream extends FSInputStream { return in; } + private static long getStreamLength(HttpURLConnection connection, + Map> headers) throws IOException { + String cl = connection.getHeaderField(HttpHeaders.CONTENT_LENGTH); + if (cl == null) { + // Try to get the content length by parsing the content range + // because HftpFileSystem does not return the content length + // if the content is partial. + if (connection.getResponseCode() == HttpStatus.SC_PARTIAL_CONTENT) { + cl = connection.getHeaderField(HttpHeaders.CONTENT_RANGE); + return getLengthFromRange(cl); + } else { + throw new IOException(HttpHeaders.CONTENT_LENGTH + " is missing: " + + headers); + } + } + return Long.parseLong(cl); + } + + private static long getLengthFromRange(String cl) throws IOException { + try { + + String[] str = cl.substring(6).split("[-/]"); + return Long.parseLong(str[1]) - Long.parseLong(str[0]) + 1; + } catch (Exception e) { + throw new IOException( + "failed to get content length by parsing the content range: " + cl + + " " + e.getMessage()); + } + } + private static boolean isChunkedTransferEncoding( final Map> headers) { return contains(headers, HttpHeaders.TRANSFER_ENCODING, "chunked")