Return-Path: Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: (qmail 66956 invoked from network); 4 Mar 2011 04:46:51 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 4 Mar 2011 04:46:51 -0000 Received: (qmail 14743 invoked by uid 500); 4 Mar 2011 04:46:51 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 14714 invoked by uid 500); 4 Mar 2011 04:46:51 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 14706 invoked by uid 99); 4 Mar 2011 04:46:51 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 04 Mar 2011 04:46:51 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 04 Mar 2011 04:46:48 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id B627C2388C1C; Fri, 4 Mar 2011 04:46:28 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1077705 - /hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java Date: Fri, 04 Mar 2011 04:46:28 -0000 To: common-commits@hadoop.apache.org From: omalley@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110304044628.B627C2388C1C@eris.apache.org> Author: omalley Date: Fri Mar 4 04:46:28 2011 New Revision: 1077705 URL: http://svn.apache.org/viewvc?rev=1077705&view=rev Log: commit bd2da1fa328e4c0fec0ba0eabba6fead891b5eb0 Author: Arun C Murthy Date: Sun Sep 19 14:59:37 2010 -0700 Reduce logging during shuffle. Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java?rev=1077705&r1=1077704&r2=1077705&view=diff ============================================================================== --- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java (original) +++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java Fri Mar 4 04:46:28 2011 @@ -1428,8 +1428,10 @@ class ReduceTask extends Task { " arrived to reduce task " + reduce); return null; } - LOG.info("header: " + mapId + ", compressed len: " + compressedLength + - ", decompressed len: " + decompressedLength); + if (LOG.isDebugEnabled()) { + LOG.debug("header: " + mapId + ", compressed len: " + compressedLength + + ", decompressed len: " + decompressedLength); + } //We will put a file in memory if it meets certain criteria: //1. The size of the (decompressed) file should be less than 25% of @@ -1441,19 +1443,23 @@ class ReduceTask extends Task { // Shuffle MapOutput mapOutput = null; - if (shuffleInMemory) { - LOG.info("Shuffling " + decompressedLength + " bytes (" + - compressedLength + " raw bytes) " + - "into RAM from " + mapOutputLoc.getTaskAttemptId()); + if (shuffleInMemory) { + if (LOG.isDebugEnabled()) { + LOG.debug("Shuffling " + decompressedLength + " bytes (" + + compressedLength + " raw bytes) " + + "into RAM from " + mapOutputLoc.getTaskAttemptId()); + } mapOutput = shuffleInMemory(mapOutputLoc, connection, input, (int)decompressedLength, (int)compressedLength); } else { - LOG.info("Shuffling " + decompressedLength + " bytes (" + - compressedLength + " raw bytes) " + - "into Local-FS from " + mapOutputLoc.getTaskAttemptId()); - + if (LOG.isDebugEnabled()) { + LOG.debug("Shuffling " + decompressedLength + " bytes (" + + compressedLength + " raw bytes) " + + "into Local-FS from " + mapOutputLoc.getTaskAttemptId()); + } + mapOutput = shuffleToDisk(mapOutputLoc, input, filename, compressedLength); } @@ -1602,8 +1608,10 @@ class ReduceTask extends Task { (shuffleData.length-bytesRead)); } - LOG.info("Read " + bytesRead + " bytes from map-output for " + - mapOutputLoc.getTaskAttemptId()); + if (LOG.isDebugEnabled()) { + LOG.debug("Read " + bytesRead + " bytes from map-output for " + + mapOutputLoc.getTaskAttemptId()); + } input.close(); } catch (IOException ioe) { @@ -1658,13 +1666,15 @@ class ReduceTask extends Task { } // TODO: Remove this after a 'fix' for HADOOP-3647 - if (mapOutputLength > 0) { - DataInputBuffer dib = new DataInputBuffer(); - dib.reset(shuffleData, 0, shuffleData.length); - LOG.info("Rec #1 from " + mapOutputLoc.getTaskAttemptId() + " -> (" + - WritableUtils.readVInt(dib) + ", " + - WritableUtils.readVInt(dib) + ") from " + - mapOutputLoc.getHost()); + if (LOG.isDebugEnabled()) { + if (mapOutputLength > 0) { + DataInputBuffer dib = new DataInputBuffer(); + dib.reset(shuffleData, 0, shuffleData.length); + LOG.debug("Rec #1 from " + mapOutputLoc.getTaskAttemptId() + + " -> (" + WritableUtils.readVInt(dib) + ", " + + WritableUtils.readVInt(dib) + ") from " + + mapOutputLoc.getHost()); + } } return mapOutput; @@ -2732,9 +2742,11 @@ class ReduceTask extends Task { do { try { int numNewMaps = getMapCompletionEvents(); - if (numNewMaps > 0) { - LOG.info(reduceTask.getTaskID() + ": " + - "Got " + numNewMaps + " new map-outputs"); + if (LOG.isDebugEnabled()) { + if (numNewMaps > 0) { + LOG.debug(reduceTask.getTaskID() + ": " + + "Got " + numNewMaps + " new map-outputs"); + } } Thread.sleep(SLEEP_TIME); }