hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r1506114 - in /hadoop/common/branches/branch-1: ./ src/mapred/ src/mapred/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/mapreduce/
Date Tue, 23 Jul 2013 15:32:34 GMT
Author: acmurthy
Date: Tue Jul 23 15:32:33 2013
New Revision: 1506114

URL: http://svn.apache.org/r1506114
Log:
MAPREDUCE-5408. Backport MAPREDUCE-336 to branch-1. Contributed by Arun C. Murthy.

Modified:
    hadoop/common/branches/branch-1/CHANGES.txt
    hadoop/common/branches/branch-1/src/mapred/mapred-default.xml
    hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobConf.java
    hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/MapTaskRunner.java
    hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTaskRunner.java
    hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
    hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestChild.java

Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1506114&r1=1506113&r2=1506114&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Tue Jul 23 15:32:33 2013
@@ -27,6 +27,8 @@ Release 1.3.0 - unreleased
     MAPREDUCE-5406. Improve logging around Task Tracker exiting with JVM manager
     inconsistent state. (Chelsey Chang via cnauroth)
 
+    MAPREDUCE-5408. Backport MAPREDUCE-336 to branch-1. (acmurthy)
+
   BUG FIXES
 
     MAPREDUCE-5047. keep.failed.task.files=true causes job failure on 

Modified: hadoop/common/branches/branch-1/src/mapred/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/mapred-default.xml?rev=1506114&r1=1506113&r2=1506114&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/mapred-default.xml (original)
+++ hadoop/common/branches/branch-1/src/mapred/mapred-default.xml Tue Jul 23 15:32:33 2013
@@ -589,6 +589,22 @@
 </property>
 
 <property>
+  <name>mapred.map.child.log.level</name>
+  <value>INFO</value>
+  <description>The logging level for the map task. The allowed levels are:
+  OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+  </description>
+</property>
+
+<property>
+  <name>mapred.reduce.child.log.level</name>
+  <value>INFO</value>
+  <description>The logging level for the reduce task. The allowed levels are:
+  OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+  </description>
+</property>
+
+<property>
   <name>mapred.inmem.merge.threshold</name>
   <value>1000</value>
   <description>The threshold, in terms of the number of files 

Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobConf.java?rev=1506114&r1=1506113&r2=1506114&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobConf.java (original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobConf.java Tue Jul
23 15:32:33 2013
@@ -45,6 +45,7 @@ import org.apache.hadoop.security.Creden
 import org.apache.hadoop.util.ClassUtil;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Tool;
+import org.apache.log4j.Level;
 
 /** 
  * A map/reduce job configuration.
@@ -337,6 +338,29 @@ public class JobConf extends Configurati
   private Credentials credentials = new Credentials();
   
   /**
+   * Configuration key to set the logging {@link Level} for the map task.
+   *
+   * The allowed logging levels are:
+   * OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+   */
+  public static final String MAPRED_MAP_TASK_LOG_LEVEL = 
+    "mapred.map.child.log.level";
+  
+  /**
+   * Configuration key to set the logging {@link Level} for the reduce task.
+   *
+   * The allowed logging levels are:
+   * OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+   */
+  public static final String MAPRED_REDUCE_TASK_LOG_LEVEL = 
+    "mapred.reduce.child.log.level";
+  
+  /**
+   * Default logging level for map/reduce tasks.
+   */
+  public static final Level DEFAULT_LOG_LEVEL = Level.INFO;
+  
+  /**
    * Construct a map/reduce job configuration.
    */
   public JobConf() {

Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/MapTaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/MapTaskRunner.java?rev=1506114&r1=1506113&r2=1506114&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/MapTaskRunner.java
(original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/MapTaskRunner.java
Tue Jul 23 15:32:33 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapred;
 import java.io.*;
 
 import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
+import org.apache.log4j.Level;
 
 /** Runs a map task. */
 class MapTaskRunner extends TaskRunner {
@@ -69,4 +70,11 @@ class MapTaskRunner extends TaskRunner {
     return jobConf.get(JobConf.MAPRED_MAP_TASK_ENV, super.getChildEnv(jobConf));
   }
 
+  @Override
+  public Level getLogLevel(JobConf jobConf) {
+    return Level.toLevel(jobConf.get(JobConf.MAPRED_MAP_TASK_LOG_LEVEL, 
+                                     JobConf.DEFAULT_LOG_LEVEL.toString()), 
+                         Level.INFO);
+  }
+
 }

Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTaskRunner.java?rev=1506114&r1=1506113&r2=1506114&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTaskRunner.java
(original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/ReduceTaskRunner.java
Tue Jul 23 15:32:33 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapred;
 import java.io.*;
 
 import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
+import org.apache.log4j.Level;
 
 /** Runs a reduce task. */
 class ReduceTaskRunner extends TaskRunner {
@@ -73,4 +74,11 @@ class ReduceTaskRunner extends TaskRunne
                        super.getChildEnv(jobConf));
   }
 
+  @Override
+  public Level getLogLevel(JobConf jobConf) {
+    return Level.toLevel(jobConf.get(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL, 
+                                     JobConf.DEFAULT_LOG_LEVEL.toString()),
+                         Level.INFO);
+  }
+
 }

Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskRunner.java?rev=1506114&r1=1506113&r2=1506114&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/TaskRunner.java Tue
Jul 23 15:32:33 2013
@@ -46,6 +46,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.log4j.Level;
 
 /** Base class that runs a task in a separate process.  Tasks are run in a
  * separate process in order to isolate the map/reduce system code from bugs in
@@ -183,6 +184,13 @@ abstract class TaskRunner extends Thread
     return jobConf.get(JobConf.MAPRED_TASK_ENV);
   }
   
+  /**
+   * Get the log {@link Level} for the child map/reduce tasks.
+   * @param jobConf
+   * @return the log-level for the child map/reduce tasks
+   */
+  public abstract Level getLogLevel(JobConf jobConf);
+  
   @Override
   public final void run() {
     String errorInfo = "Child Error";
@@ -477,7 +485,7 @@ abstract class TaskRunner extends Thread
     vargs.add("-Dlog4j.configuration=task-log4j.properties");
     vargs.add("-Dhadoop.log.dir=" + 
         new File(System.getProperty("hadoop.log.dir")).getAbsolutePath());
-    vargs.add("-Dhadoop.root.logger=INFO,TLA");
+    vargs.add("-Dhadoop.root.logger=" + getLogLevel(conf).toString() + ",TLA");
     vargs.add("-Dhadoop.tasklog.taskid=" + taskid);
     vargs.add("-Dhadoop.tasklog.iscleanup=" + t.isTaskCleanupTask());
     vargs.add("-Dhadoop.tasklog.totalLogFileSize=" + logSize);

Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestChild.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestChild.java?rev=1506114&r1=1506113&r2=1506114&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestChild.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/TestChild.java Tue
Jul 23 15:32:33 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.io.LongWritable
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.HadoopTestCase;
 import org.apache.hadoop.mapred.JobConf;
+import org.apache.log4j.Level;
 
 public class TestChild extends HadoopTestCase {
   private static String TEST_ROOT_DIR =
@@ -68,6 +69,12 @@ public class TestChild extends HadoopTes
                      mapJavaOpts, 
                      mapJavaOpts, MAP_OPTS_VAL);
       }
+      
+      Level logLevel = 
+        Level.toLevel(conf.get(JobConf.MAPRED_MAP_TASK_LOG_LEVEL, 
+                               Level.INFO.toString()));  
+      assertEquals(JobConf.MAPRED_MAP_TASK_LOG_LEVEL + "has value of " + 
+                   logLevel, logLevel, Level.OFF);
     }
   }
   
@@ -94,6 +101,12 @@ public class TestChild extends HadoopTes
                      reduceJavaOpts, 
                      reduceJavaOpts, REDUCE_OPTS_VAL);
       }
+      
+      Level logLevel = 
+        Level.toLevel(conf.get(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL, 
+                               Level.INFO.toString()));  
+      assertEquals(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL + "has value of " + 
+                   logLevel, logLevel, Level.OFF);
     }
   }
   
@@ -108,6 +121,9 @@ public class TestChild extends HadoopTes
       conf.set(JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS, REDUCE_OPTS_VAL);
     }
     
+    conf.set(JobConf.MAPRED_MAP_TASK_LOG_LEVEL, Level.OFF.toString());
+    conf.set(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL, Level.OFF.toString());
+    
     Job job = MapReduceTestUtil.createJob(conf, inDir, outDir, 
                 numMaps, numReds);
     job.setMapperClass(MyMapper.class);



Mime
View raw message