From hdfs-commits-return-4843-apmail-hadoop-hdfs-commits-archive=hadoop.apache.org@hadoop.apache.org Tue Nov 6 22:34:54 2012 Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 3C5ADD1B2 for ; Tue, 6 Nov 2012 22:34:54 +0000 (UTC) Received: (qmail 27482 invoked by uid 500); 6 Nov 2012 22:34:54 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 27447 invoked by uid 500); 6 Nov 2012 22:34:54 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 27437 invoked by uid 99); 6 Nov 2012 22:34:54 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 06 Nov 2012 22:34:54 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 06 Nov 2012 22:34:51 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id E7F1A2388980; Tue, 6 Nov 2012 22:34:29 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1406382 - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java Date: Tue, 06 Nov 2012 22:34:29 -0000 To: hdfs-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20121106223429.E7F1A2388980@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: szetszwo Date: Tue Nov 6 22:34:29 2012 New Revision: 1406382 URL: http://svn.apache.org/viewvc?rev=1406382&view=rev Log: HDFS-3979. For hsync, datanode should wait for the local sync to complete before sending ack. Contributed by Lars Hofhansl Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1406382&r1=1406381&r2=1406382&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Nov 6 22:34:29 2012 @@ -558,6 +558,9 @@ Release 2.0.3-alpha - Unreleased HDFS-1331. dfs -test should work like /bin/test (Andy Isaacson via daryn) + HDFS-3979. For hsync, datanode should wait for the local sync to complete + before sending ack. (Lars Hofhansl via szetszwo) + Release 2.0.2-alpha - 2012-09-07 INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java?rev=1406382&r1=1406381&r2=1406382&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java (original) +++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java Tue Nov 6 22:34:29 2012 @@ -319,9 +319,6 @@ class BlockReceiver implements Closeable * @throws IOException */ void flushOrSync(boolean isSync) throws IOException { - if (isSync && (out != null || checksumOut != null)) { - datanode.metrics.incrFsyncCount(); - } long flushTotalNanos = 0; if (checksumOut != null) { long flushStartNanos = System.nanoTime(); @@ -347,6 +344,9 @@ class BlockReceiver implements Closeable } if (checksumOut != null || out != null) { datanode.metrics.addFlushNanos(flushTotalNanos); + if (isSync) { + datanode.metrics.incrFsyncCount(); + } } } @@ -438,8 +438,10 @@ class BlockReceiver implements Closeable int len = header.getDataLen(); boolean syncBlock = header.getSyncBlock(); - // make sure the block gets sync'ed upon close - this.syncOnClose |= syncBlock && lastPacketInBlock; + // avoid double sync'ing on close + if (syncBlock && lastPacketInBlock) { + this.syncOnClose = false; + } // update received bytes long firstByteInBlock = offsetInBlock; @@ -448,11 +450,11 @@ class BlockReceiver implements Closeable replicaInfo.setNumBytes(offsetInBlock); } - // put in queue for pending acks - if (responder != null) { - ((PacketResponder)responder.getRunnable()).enqueue(seqno, - lastPacketInBlock, offsetInBlock); - } + // put in queue for pending acks, unless sync was requested + if (responder != null && !syncBlock) { + ((PacketResponder) responder.getRunnable()).enqueue(seqno, + lastPacketInBlock, offsetInBlock); + } //First write the packet to the mirror: if (mirrorOut != null && !mirrorError) { @@ -471,8 +473,8 @@ class BlockReceiver implements Closeable if(LOG.isDebugEnabled()) { LOG.debug("Receiving an empty packet or the end of the block " + block); } - // flush unless close() would flush anyway - if (syncBlock && !lastPacketInBlock) { + // sync block if requested + if (syncBlock) { flushOrSync(true); } } else { @@ -563,8 +565,8 @@ class BlockReceiver implements Closeable checksumBuf.arrayOffset() + checksumBuf.position(), checksumLen); } - /// flush entire packet, sync unless close() will sync - flushOrSync(syncBlock && !lastPacketInBlock); + /// flush entire packet, sync if requested + flushOrSync(syncBlock); replicaInfo.setLastChecksumAndDataLen( offsetInBlock, lastChunkChecksum @@ -580,6 +582,13 @@ class BlockReceiver implements Closeable } } + // if sync was requested, put in queue for pending acks here + // (after the fsync finished) + if (responder != null && syncBlock) { + ((PacketResponder) responder.getRunnable()).enqueue(seqno, + lastPacketInBlock, offsetInBlock); + } + if (throttler != null) { // throttle I/O throttler.throttle(len); }