Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E2A2D106B1 for ; Tue, 23 Jul 2013 15:33:06 +0000 (UTC) Received: (qmail 1577 invoked by uid 500); 23 Jul 2013 15:33:06 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 1390 invoked by uid 500); 23 Jul 2013 15:33:01 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 1328 invoked by uid 99); 23 Jul 2013 15:32:59 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 23 Jul 2013 15:32:59 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 23 Jul 2013 15:32:56 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id BF5D023888E7 for ; Tue, 23 Jul 2013 15:32:34 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1506114 - in /hadoop/common/branches/branch-1: ./ src/mapred/ src/mapred/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/mapreduce/ Date: Tue, 23 Jul 2013 15:32:34 -0000 To: common-commits@hadoop.apache.org From: acmurthy@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20130723153234.BF5D023888E7@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: acmurthy Date: Tue Jul 23 15:32:33 2013 New Revision: 1506114 URL: http://svn.apache.org/r1506114 Log: MAPREDUCE-5408. Backport MAPREDUCE-336 to branch-1. Contributed by Arun C. Murthy. Modified: hadoop/common/branches/branch-1/CHANGES.txt hadoop/common/branches/branch-1/src/mapred/mapred-default.xml hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobConf.java hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/MapTaskRunner.java hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTaskRunner.java hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskRunner.java hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestChild.java Modified: hadoop/common/branches/branch-1/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1506114&r1=1506113&r2=1506114&view=diff ============================================================================== --- hadoop/common/branches/branch-1/CHANGES.txt (original) +++ hadoop/common/branches/branch-1/CHANGES.txt Tue Jul 23 15:32:33 2013 @@ -27,6 +27,8 @@ Release 1.3.0 - unreleased MAPREDUCE-5406. Improve logging around Task Tracker exiting with JVM manager inconsistent state. (Chelsey Chang via cnauroth) + MAPREDUCE-5408. Backport MAPREDUCE-336 to branch-1. (acmurthy) + BUG FIXES MAPREDUCE-5047. keep.failed.task.files=true causes job failure on Modified: hadoop/common/branches/branch-1/src/mapred/mapred-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/mapred-default.xml?rev=1506114&r1=1506113&r2=1506114&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/mapred/mapred-default.xml (original) +++ hadoop/common/branches/branch-1/src/mapred/mapred-default.xml Tue Jul 23 15:32:33 2013 @@ -589,6 +589,22 @@ + mapred.map.child.log.level + INFO + The logging level for the map task. The allowed levels are: + OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL. + + + + + mapred.reduce.child.log.level + INFO + The logging level for the reduce task. The allowed levels are: + OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL. + + + + mapred.inmem.merge.threshold 1000 The threshold, in terms of the number of files Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobConf.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobConf.java?rev=1506114&r1=1506113&r2=1506114&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobConf.java (original) +++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobConf.java Tue Jul 23 15:32:33 2013 @@ -45,6 +45,7 @@ import org.apache.hadoop.security.Creden import org.apache.hadoop.util.ClassUtil; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.Tool; +import org.apache.log4j.Level; /** * A map/reduce job configuration. @@ -337,6 +338,29 @@ public class JobConf extends Configurati private Credentials credentials = new Credentials(); /** + * Configuration key to set the logging {@link Level} for the map task. + * + * The allowed logging levels are: + * OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL. + */ + public static final String MAPRED_MAP_TASK_LOG_LEVEL = + "mapred.map.child.log.level"; + + /** + * Configuration key to set the logging {@link Level} for the reduce task. + * + * The allowed logging levels are: + * OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL. + */ + public static final String MAPRED_REDUCE_TASK_LOG_LEVEL = + "mapred.reduce.child.log.level"; + + /** + * Default logging level for map/reduce tasks. + */ + public static final Level DEFAULT_LOG_LEVEL = Level.INFO; + + /** * Construct a map/reduce job configuration. */ public JobConf() { Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/MapTaskRunner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/MapTaskRunner.java?rev=1506114&r1=1506113&r2=1506114&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/MapTaskRunner.java (original) +++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/MapTaskRunner.java Tue Jul 23 15:32:33 2013 @@ -20,6 +20,7 @@ package org.apache.hadoop.mapred; import java.io.*; import org.apache.hadoop.mapred.TaskTracker.TaskInProgress; +import org.apache.log4j.Level; /** Runs a map task. */ class MapTaskRunner extends TaskRunner { @@ -69,4 +70,11 @@ class MapTaskRunner extends TaskRunner { return jobConf.get(JobConf.MAPRED_MAP_TASK_ENV, super.getChildEnv(jobConf)); } + @Override + public Level getLogLevel(JobConf jobConf) { + return Level.toLevel(jobConf.get(JobConf.MAPRED_MAP_TASK_LOG_LEVEL, + JobConf.DEFAULT_LOG_LEVEL.toString()), + Level.INFO); + } + } Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTaskRunner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTaskRunner.java?rev=1506114&r1=1506113&r2=1506114&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTaskRunner.java (original) +++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTaskRunner.java Tue Jul 23 15:32:33 2013 @@ -20,6 +20,7 @@ package org.apache.hadoop.mapred; import java.io.*; import org.apache.hadoop.mapred.TaskTracker.TaskInProgress; +import org.apache.log4j.Level; /** Runs a reduce task. */ class ReduceTaskRunner extends TaskRunner { @@ -73,4 +74,11 @@ class ReduceTaskRunner extends TaskRunne super.getChildEnv(jobConf)); } + @Override + public Level getLogLevel(JobConf jobConf) { + return Level.toLevel(jobConf.get(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL, + JobConf.DEFAULT_LOG_LEVEL.toString()), + Level.INFO); + } + } Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskRunner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskRunner.java?rev=1506114&r1=1506113&r2=1506114&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskRunner.java (original) +++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskRunner.java Tue Jul 23 15:32:33 2013 @@ -46,6 +46,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; +import org.apache.log4j.Level; /** Base class that runs a task in a separate process. Tasks are run in a * separate process in order to isolate the map/reduce system code from bugs in @@ -183,6 +184,13 @@ abstract class TaskRunner extends Thread return jobConf.get(JobConf.MAPRED_TASK_ENV); } + /** + * Get the log {@link Level} for the child map/reduce tasks. + * @param jobConf + * @return the log-level for the child map/reduce tasks + */ + public abstract Level getLogLevel(JobConf jobConf); + @Override public final void run() { String errorInfo = "Child Error"; @@ -477,7 +485,7 @@ abstract class TaskRunner extends Thread vargs.add("-Dlog4j.configuration=task-log4j.properties"); vargs.add("-Dhadoop.log.dir=" + new File(System.getProperty("hadoop.log.dir")).getAbsolutePath()); - vargs.add("-Dhadoop.root.logger=INFO,TLA"); + vargs.add("-Dhadoop.root.logger=" + getLogLevel(conf).toString() + ",TLA"); vargs.add("-Dhadoop.tasklog.taskid=" + taskid); vargs.add("-Dhadoop.tasklog.iscleanup=" + t.isTaskCleanupTask()); vargs.add("-Dhadoop.tasklog.totalLogFileSize=" + logSize); Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestChild.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestChild.java?rev=1506114&r1=1506113&r2=1506114&view=diff ============================================================================== --- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestChild.java (original) +++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestChild.java Tue Jul 23 15:32:33 2013 @@ -29,6 +29,7 @@ import org.apache.hadoop.io.LongWritable import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.HadoopTestCase; import org.apache.hadoop.mapred.JobConf; +import org.apache.log4j.Level; public class TestChild extends HadoopTestCase { private static String TEST_ROOT_DIR = @@ -68,6 +69,12 @@ public class TestChild extends HadoopTes mapJavaOpts, mapJavaOpts, MAP_OPTS_VAL); } + + Level logLevel = + Level.toLevel(conf.get(JobConf.MAPRED_MAP_TASK_LOG_LEVEL, + Level.INFO.toString())); + assertEquals(JobConf.MAPRED_MAP_TASK_LOG_LEVEL + "has value of " + + logLevel, logLevel, Level.OFF); } } @@ -94,6 +101,12 @@ public class TestChild extends HadoopTes reduceJavaOpts, reduceJavaOpts, REDUCE_OPTS_VAL); } + + Level logLevel = + Level.toLevel(conf.get(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL, + Level.INFO.toString())); + assertEquals(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL + "has value of " + + logLevel, logLevel, Level.OFF); } } @@ -108,6 +121,9 @@ public class TestChild extends HadoopTes conf.set(JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS, REDUCE_OPTS_VAL); } + conf.set(JobConf.MAPRED_MAP_TASK_LOG_LEVEL, Level.OFF.toString()); + conf.set(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL, Level.OFF.toString()); + Job job = MapReduceTestUtil.createJob(conf, inDir, outDir, numMaps, numReds); job.setMapperClass(MyMapper.class);