Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id F3EA918F61 for ; Fri, 14 Aug 2015 10:54:29 +0000 (UTC) Received: (qmail 95949 invoked by uid 500); 14 Aug 2015 10:54:17 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 95826 invoked by uid 500); 14 Aug 2015 10:54:17 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 94048 invoked by uid 99); 14 Aug 2015 10:54:16 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 14 Aug 2015 10:54:16 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0B523E0449; Fri, 14 Aug 2015 10:54:16 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vinayakumarb@apache.org To: common-commits@hadoop.apache.org Date: Fri, 14 Aug 2015 10:54:43 -0000 Message-Id: <7929fa066b584e7798c4f3d8a8e00837@git.apache.org> In-Reply-To: <0e8188f9427546bfa5c0a42a5f7f5505@git.apache.org> References: <0e8188f9427546bfa5c0a42a5f7f5505@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [29/50] hadoop git commit: HDFS-8719. Erasure Coding: client generates too many small packets when writing parity data. Contributed by Li Bo. HDFS-8719. Erasure Coding: client generates too many small packets when writing parity data. Contributed by Li Bo. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/91d0873c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/91d0873c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/91d0873c Branch: refs/heads/HDFS-7285-REBASE Commit: 91d0873cba1950fd45a7004499129b9c2cae8c71 Parents: 3a24001 Author: Walter Su Authored: Thu Jul 9 16:18:05 2015 +0800 Committer: Vinayakumar B Committed: Thu Aug 13 17:18:40 2015 +0530 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++ .../src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java | 4 +--- .../main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/91d0873c/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 58b91b6..cdbef05 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -332,3 +332,6 @@ HDFS-8461. Erasure coding: fix priority level of UnderReplicatedBlocks for striped block. (Walter Su via jing9) + + HDFS-8719. Erasure Coding: client generates too many small packets when + writing parity data. (Li Bo via waltersu4549) http://git-wip-us.apache.org/repos/asf/hadoop/blob/91d0873c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java index 7e8dd21..088f6dd 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java @@ -24,8 +24,6 @@ import java.nio.channels.ClosedChannelException; import java.util.EnumSet; import java.util.concurrent.atomic.AtomicReference; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.crypto.CryptoProtocolVersion; @@ -457,7 +455,7 @@ public class DFSOutputStream extends FSOutputSummer * write filled up its partial chunk. Tell the summer to generate full * crc chunks from now on. */ - private void adjustChunkBoundary() { + protected void adjustChunkBoundary() { if (getStreamer().getAppendChunk() && getStreamer().getBytesCurBlock() % bytesPerChecksum == 0) { getStreamer().setAppendChunk(false); http://git-wip-us.apache.org/repos/asf/hadoop/blob/91d0873c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index a18a30b..5e80b46 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -318,6 +318,7 @@ public class DFSStripedOutputStream extends DFSOutputStream { private synchronized StripedDataStreamer setCurrentStreamer(int i) { streamer = streamers.get(i); + adjustChunkBoundary(); return getCurrentStreamer(); }