Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D78DAD780 for ; Wed, 27 Feb 2013 10:40:25 +0000 (UTC) Received: (qmail 71858 invoked by uid 500); 27 Feb 2013 10:40:25 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 71620 invoked by uid 500); 27 Feb 2013 10:40:24 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 71582 invoked by uid 99); 27 Feb 2013 10:40:23 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 27 Feb 2013 10:40:23 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 27 Feb 2013 10:40:20 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 9D82423888CD; Wed, 27 Feb 2013 10:40:00 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1450723 - in /hadoop/common/trunk/hadoop-mapreduce-project: CHANGES.txt hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java Date: Wed, 27 Feb 2013 10:40:00 -0000 To: mapreduce-commits@hadoop.apache.org From: tomwhite@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130227104000.9D82423888CD@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: tomwhite Date: Wed Feb 27 10:40:00 2013 New Revision: 1450723 URL: http://svn.apache.org/r1450723 Log: MAPREDUCE-5008. Merger progress miscounts with respect to EOF_MARKER. Contributed by Sandy Ryza. Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1450723&r1=1450722&r2=1450723&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Feb 27 10:40:00 2013 @@ -189,6 +189,9 @@ Release 2.0.4-beta - UNRELEASED MAPREDUCE-4951. Container preemption interpreted as task failure. (Sandy Ryza via tomwhite) + MAPREDUCE-5008. Merger progress miscounts with respect to EOF_MARKER. + (Sandy Ryza via tomwhite) + Release 2.0.3-alpha - 2013-02-06 INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java?rev=1450723&r1=1450722&r2=1450723&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java Wed Feb 27 10:40:00 2013 @@ -475,9 +475,9 @@ public class MergeManagerImpl impl combineCollector.setWriter(writer); combineAndSpill(rIter, reduceCombineInputCounter); } + writer.close(); compressAwarePath = new CompressAwarePath(outputPath, writer.getRawLength()); - writer.close(); LOG.info(reduceId + " Merge of the " + noInMemorySegments + @@ -552,9 +552,9 @@ public class MergeManagerImpl impl mergedMapOutputsCounter, null); Merger.writeFile(iter, writer, reporter, jobConf); + writer.close(); compressAwarePath = new CompressAwarePath(outputPath, writer.getRawLength()); - writer.close(); } catch (IOException e) { localFS.delete(outputPath, true); throw e; @@ -713,13 +713,15 @@ public class MergeManagerImpl impl keyClass, valueClass, memDiskSegments, numMemDiskSegments, tmpDir, comparator, reporter, spilledRecordsCounter, null, mergePhase); - final Writer writer = new Writer(job, fs, outputPath, + Writer writer = new Writer(job, fs, outputPath, keyClass, valueClass, codec, null); try { Merger.writeFile(rIter, writer, reporter, job); - // add to list of final disk outputs. + writer.close(); onDiskMapOutputs.add(new CompressAwarePath(outputPath, writer.getRawLength())); + writer = null; + // add to list of final disk outputs. } catch (IOException e) { if (null != outputPath) { try {