Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 1D49917F7B for ; Mon, 27 Apr 2015 18:15:06 +0000 (UTC) Received: (qmail 49292 invoked by uid 500); 27 Apr 2015 18:14:44 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 49096 invoked by uid 500); 27 Apr 2015 18:14:44 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 46321 invoked by uid 99); 27 Apr 2015 18:14:43 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 27 Apr 2015 18:14:43 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 67209E17DB; Mon, 27 Apr 2015 18:14:43 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zhz@apache.org To: common-commits@hadoop.apache.org Date: Mon, 27 Apr 2015 18:15:29 -0000 Message-Id: <90d2e940b6524ec5b76bc3b3d2d29ccf@git.apache.org> In-Reply-To: <823a77c46fe54bb0981bcff0a915ce9a@git.apache.org> References: <823a77c46fe54bb0981bcff0a915ce9a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [48/50] hadoop git commit: HDFS-8223. Should calculate checksum for parity blocks in DFSStripedOutputStream. Contributed by Yi Liu. HDFS-8223. Should calculate checksum for parity blocks in DFSStripedOutputStream. Contributed by Yi Liu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/17c5ed6e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/17c5ed6e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/17c5ed6e Branch: refs/heads/HDFS-7285 Commit: 17c5ed6e27d5c964c05a3244e6076c0e116622c0 Parents: 2fe4f7a Author: Jing Zhao Authored: Thu Apr 23 15:48:21 2015 -0700 Committer: Zhe Zhang Committed: Mon Apr 27 10:42:55 2015 -0700 ---------------------------------------------------------------------- .../main/java/org/apache/hadoop/fs/FSOutputSummer.java | 4 ++++ hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++ .../org/apache/hadoop/hdfs/DFSStripedOutputStream.java | 10 ++++++++++ 3 files changed, 17 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/17c5ed6e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java index bdc5585..a8a7494 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSOutputSummer.java @@ -196,6 +196,10 @@ abstract public class FSOutputSummer extends OutputStream { return sum.getChecksumSize(); } + protected DataChecksum getDataChecksum() { + return sum; + } + protected TraceScope createWriteTraceScope() { return NullScope.INSTANCE; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/17c5ed6e/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 48791b1..9357e23 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -125,3 +125,6 @@ HDFS-8233. Fix DFSStripedOutputStream#getCurrentBlockGroupBytes when the last stripe is at the block group boundary. (jing9) + + HDFS-8223. Should calculate checksum for parity blocks in DFSStripedOutputStream. + (Yi Liu via jing9) http://git-wip-us.apache.org/repos/asf/hadoop/blob/17c5ed6e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java index 245dfc1..6842267 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedOutputStream.java @@ -62,6 +62,8 @@ public class DFSStripedOutputStream extends DFSOutputStream { */ private final ECInfo ecInfo; private final int cellSize; + // checksum buffer, we only need to calculate checksum for parity blocks + private byte[] checksumBuf; private ByteBuffer[] cellBuffers; private final short numAllBlocks; @@ -99,6 +101,7 @@ public class DFSStripedOutputStream extends DFSOutputStream { checkConfiguration(); + checksumBuf = new byte[getChecksumSize() * (cellSize / bytesPerChecksum)]; cellBuffers = new ByteBuffer[numAllBlocks]; List> stripeBlocks = new ArrayList<>(); @@ -179,6 +182,10 @@ public class DFSStripedOutputStream extends DFSOutputStream { private List generatePackets(ByteBuffer byteBuffer) throws IOException{ List packets = new ArrayList<>(); + assert byteBuffer.hasArray(); + getDataChecksum().calculateChunkedSums(byteBuffer.array(), 0, + byteBuffer.remaining(), checksumBuf, 0); + int ckOff = 0; while (byteBuffer.remaining() > 0) { DFSPacket p = createPacket(packetSize, chunksPerPacket, streamer.getBytesCurBlock(), @@ -186,6 +193,9 @@ public class DFSStripedOutputStream extends DFSOutputStream { int maxBytesToPacket = p.getMaxChunks() * bytesPerChecksum; int toWrite = byteBuffer.remaining() > maxBytesToPacket ? maxBytesToPacket: byteBuffer.remaining(); + int ckLen = ((toWrite - 1) / bytesPerChecksum + 1) * getChecksumSize(); + p.writeChecksum(checksumBuf, ckOff, ckLen); + ckOff += ckLen; p.writeData(byteBuffer, toWrite); streamer.incBytesCurBlock(toWrite); packets.add(p);