Return-Path: Delivered-To: apmail-hadoop-core-commits-archive@www.apache.org Received: (qmail 65624 invoked from network); 16 Dec 2008 10:17:25 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.2) by minotaur.apache.org with SMTP; 16 Dec 2008 10:17:25 -0000 Received: (qmail 52068 invoked by uid 500); 16 Dec 2008 10:17:37 -0000 Delivered-To: apmail-hadoop-core-commits-archive@hadoop.apache.org Received: (qmail 52036 invoked by uid 500); 16 Dec 2008 10:17:37 -0000 Mailing-List: contact core-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: core-dev@hadoop.apache.org Delivered-To: mailing list core-commits@hadoop.apache.org Received: (qmail 52027 invoked by uid 99); 16 Dec 2008 10:17:37 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 Dec 2008 02:17:37 -0800 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 16 Dec 2008 10:17:23 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 4FC5E238895D; Tue, 16 Dec 2008 02:17:03 -0800 (PST) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r727006 - in /hadoop/core/branches/branch-0.20: CHANGES.txt src/mapred/org/apache/hadoop/mapred/ReduceTask.java src/mapred/org/apache/hadoop/mapred/Task.java src/mapred/org/apache/hadoop/mapred/Task_Counter.properties Date: Tue, 16 Dec 2008 10:17:02 -0000 To: core-commits@hadoop.apache.org From: cdouglas@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20081216101703.4FC5E238895D@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: cdouglas Date: Tue Dec 16 02:17:02 2008 New Revision: 727006 URL: http://svn.apache.org/viewvc?rev=727006&view=rev Log: HADOOP-4845. Modify the reduce input byte counter to record only the compressed size and add a human-readable label. Contributed by Yongqiang He Modified: hadoop/core/branches/branch-0.20/CHANGES.txt hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ReduceTask.java hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task_Counter.properties Modified: hadoop/core/branches/branch-0.20/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=727006&r1=727005&r2=727006&view=diff ============================================================================== --- hadoop/core/branches/branch-0.20/CHANGES.txt (original) +++ hadoop/core/branches/branch-0.20/CHANGES.txt Tue Dec 16 02:17:02 2008 @@ -428,6 +428,9 @@ HADOOP-3921. Fixed clover (code coverage) target to work with JDK 6. (tomwhite via nigel) + HADOOP-4845. Modify the reduce input byte counter to record only the + compressed size and add a human-readable label. (Yongqiang He via cdouglas) + Release 0.19.1 - Unreleased IMPROVEMENTS Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ReduceTask.java URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ReduceTask.java?rev=727006&r1=727005&r2=727006&view=diff ============================================================================== --- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ReduceTask.java (original) +++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ReduceTask.java Tue Dec 16 02:17:02 2008 @@ -111,8 +111,8 @@ private Progress copyPhase; private Progress sortPhase; private Progress reducePhase; - private Counters.Counter reduceInputBytes = - getCounters().findCounter(Counter.REDUCE_INPUT_BYTES); + private Counters.Counter reduceShuffleBytes = + getCounters().findCounter(Counter.REDUCE_SHUFFLE_BYTES); private Counters.Counter reduceInputKeyCounter = getCounters().findCounter(Counter.REDUCE_INPUT_GROUPS); private Counters.Counter reduceInputValueCounter = @@ -380,7 +380,6 @@ throw new IOException("Task: " + getTaskID() + " - The reduce copier failed", reduceCopier.mergeThrowable); } - reduceInputBytes.increment(reduceCopier.reducerInputBytes); } copyPhase.complete(); // copy is already complete setPhase(TaskStatus.Phase.SORT); @@ -919,7 +918,7 @@ byte[] data; final boolean inMemory; - long size; + long compressedSize; public MapOutput(TaskID mapId, TaskAttemptID mapAttemptId, Configuration conf, Path file, long size) { @@ -928,14 +927,14 @@ this.conf = conf; this.file = file; - this.size = size; + this.compressedSize = size; this.data = null; this.inMemory = false; } - public MapOutput(TaskID mapId, TaskAttemptID mapAttemptId, byte[] data) { + public MapOutput(TaskID mapId, TaskAttemptID mapAttemptId, byte[] data, int compressedLength) { this.mapId = mapId; this.mapAttemptId = mapAttemptId; @@ -943,7 +942,7 @@ this.conf = null; this.data = data; - this.size = data.length; + this.compressedSize = compressedLength; this.inMemory = true; } @@ -1262,7 +1261,7 @@ } // The size of the map-output - long bytes = mapOutput.size; + long bytes = mapOutput.compressedSize; // lock the ReduceTask while we do the rename synchronized (ReduceTask.this) { @@ -1467,7 +1466,7 @@ byte[] shuffleData = new byte[mapOutputLength]; MapOutput mapOutput = new MapOutput(mapOutputLoc.getTaskId(), - mapOutputLoc.getTaskAttemptId(), shuffleData); + mapOutputLoc.getTaskAttemptId(), shuffleData, compressedLength); int bytesRead = 0; try { @@ -1741,12 +1740,10 @@ return numInFlight > maxInFlight; } - long reducerInputBytes = 0; public boolean fetchOutputs() throws IOException { int totalFailures = 0; int numInFlight = 0, numCopied = 0; - long bytesTransferred = 0; DecimalFormat mbpsFormat = new DecimalFormat("0.00"); final Progress copyPhase = reduceTask.getProgress().phase(); @@ -1938,12 +1935,11 @@ if (cr.getSuccess()) { // a successful copy numCopied++; lastProgressTime = System.currentTimeMillis(); - bytesTransferred += cr.getSize(); - reducerInputBytes += cr.getSize(); + reduceShuffleBytes.increment(cr.getSize()); long secsSinceStart = (System.currentTimeMillis()-startTime)/1000+1; - float mbs = ((float)bytesTransferred)/(1024*1024); + float mbs = ((float)reduceShuffleBytes.getCounter())/(1024*1024); float transferRate = mbs/secsSinceStart; copyPhase.startNextPhase(); Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java?rev=727006&r1=727005&r2=727006&view=diff ============================================================================== --- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java (original) +++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task.java Tue Dec 16 02:17:02 2008 @@ -63,7 +63,7 @@ COMBINE_INPUT_RECORDS, COMBINE_OUTPUT_RECORDS, REDUCE_INPUT_GROUPS, - REDUCE_INPUT_BYTES, + REDUCE_SHUFFLE_BYTES, REDUCE_INPUT_RECORDS, REDUCE_OUTPUT_RECORDS, REDUCE_SKIPPED_GROUPS, Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task_Counter.properties URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task_Counter.properties?rev=727006&r1=727005&r2=727006&view=diff ============================================================================== --- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task_Counter.properties (original) +++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/Task_Counter.properties Tue Dec 16 02:17:02 2008 @@ -10,6 +10,7 @@ COMBINE_INPUT_RECORDS.name= Combine input records COMBINE_OUTPUT_RECORDS.name= Combine output records REDUCE_INPUT_GROUPS.name= Reduce input groups +REDUCE_SHUFFLE_BYTES.name= Reduce shuffle bytes REDUCE_INPUT_RECORDS.name= Reduce input records REDUCE_OUTPUT_RECORDS.name= Reduce output records REDUCE_SKIPPED_RECORDS.name= Reduce skipped records