Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 193D110650 for ; Sat, 22 Mar 2014 00:33:22 +0000 (UTC) Received: (qmail 98770 invoked by uid 500); 22 Mar 2014 00:33:20 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 98656 invoked by uid 500); 22 Mar 2014 00:33:19 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 98648 invoked by uid 99); 22 Mar 2014 00:33:19 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 22 Mar 2014 00:33:19 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 22 Mar 2014 00:33:18 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id EB43423888FE; Sat, 22 Mar 2014 00:32:57 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1580100 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/... Date: Sat, 22 Mar 2014 00:32:57 -0000 To: mapreduce-commits@hadoop.apache.org From: vinodkv@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140322003257.EB43423888FE@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: vinodkv Date: Sat Mar 22 00:32:57 2014 New Revision: 1580100 URL: http://svn.apache.org/r1580100 Log: MAPREDUCE-5806. Fixed a bug in MRAppMaster so as to enable users to properly override HADOOP_ROOT_LOGGER or HADOOP_CLIENT_OPTS. Contributed by Varun Vasudev. Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1580100&r1=1580099&r2=1580100&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Sat Mar 22 00:32:57 2014 @@ -255,6 +255,10 @@ Release 2.4.0 - UNRELEASED MAPREDUCE-5570. Map task attempt with fetch failure has incorrect attempt finish time (Rushabh S Shah via jlowe) + MAPREDUCE-5806. Fixed a bug in MRAppMaster so as to enable users to properly + override HADOOP_ROOT_LOGGER or HADOOP_CLIENT_OPTS. (Varun Vasudev via + vinodkv) + Release 2.3.1 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java?rev=1580100&r1=1580099&r2=1580100&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/MapReduceChildJVM.java Sat Mar 22 00:32:57 2014 @@ -19,6 +19,7 @@ package org.apache.hadoop.mapred; import java.net.InetSocketAddress; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -77,7 +78,7 @@ public class MapReduceChildJVM { // streaming) it will have the correct loglevel. environment.put( "HADOOP_ROOT_LOGGER", - getChildLogLevel(conf, task.isMapTask()) + ",CLA"); + getChildLogLevel(conf, task.isMapTask()) + ",console"); // TODO: The following is useful for instance in streaming tasks. Should be // set in ApplicationMaster's env by the RM. @@ -87,18 +88,19 @@ public class MapReduceChildJVM { } else { hadoopClientOpts = hadoopClientOpts + " "; } - // FIXME: don't think this is also needed given we already set java - // properties. - long logSize = TaskLog.getTaskLogLength(conf); - Vector logProps = new Vector(4); - setupLog4jProperties(task, logProps, logSize); - Iterator it = logProps.iterator(); - StringBuffer buffer = new StringBuffer(); - while (it.hasNext()) { - buffer.append(" " + it.next()); - } - hadoopClientOpts = hadoopClientOpts + buffer.toString(); environment.put("HADOOP_CLIENT_OPTS", hadoopClientOpts); + + // setEnvFromInputString above will add env variable values from + // mapredChildEnv to existing variables. We want to overwrite + // HADOOP_ROOT_LOGGER and HADOOP_CLIENT_OPTS if the user set it explicitly. + Map tmpEnv = new HashMap(); + MRApps.setEnvFromInputString(tmpEnv, mapredChildEnv, conf); + String[] keys = { "HADOOP_ROOT_LOGGER", "HADOOP_CLIENT_OPTS" }; + for (String key : keys) { + if (tmpEnv.containsKey(key)) { + environment.put(key, tmpEnv.get(key)); + } + } // Add stdout/stderr env environment.put( Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java?rev=1580100&r1=1580099&r2=1580100&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestMapReduceChildJVM.java Sat Mar 22 00:32:57 2014 @@ -18,11 +18,14 @@ package org.apache.hadoop.mapreduce.v2.app.job.impl; +import java.util.Map; + import junit.framework.Assert; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.v2.api.records.JobState; @@ -65,11 +68,20 @@ public class TestMapReduceChildJVM { " 0" + " 1>/stdout" + " 2>/stderr ]", app.myCommandLine); + + Assert.assertTrue("HADOOP_ROOT_LOGGER not set for job", + app.cmdEnvironment.containsKey("HADOOP_ROOT_LOGGER")); + Assert.assertEquals("INFO,console", + app.cmdEnvironment.get("HADOOP_ROOT_LOGGER")); + Assert.assertTrue("HADOOP_CLIENT_OPTS not set for job", + app.cmdEnvironment.containsKey("HADOOP_CLIENT_OPTS")); + Assert.assertEquals("", app.cmdEnvironment.get("HADOOP_CLIENT_OPTS")); } private static final class MyMRApp extends MRApp { private String myCommandLine; + private Map cmdEnvironment; public MyMRApp(int maps, int reduces, boolean autoComplete, String testName, boolean cleanOnStart) { @@ -88,10 +100,44 @@ public class TestMapReduceChildJVM { String cmdString = launchContext.getCommands().toString(); LOG.info("launchContext " + cmdString); myCommandLine = cmdString; + cmdEnvironment = launchContext.getEnvironment(); } super.handle(event); } }; } } + + @Test + public void testEnvironmentVariables() throws Exception { + MyMRApp app = new MyMRApp(1, 0, true, this.getClass().getName(), true); + Configuration conf = new Configuration(); + conf.set(JobConf.MAPRED_MAP_TASK_ENV, "HADOOP_CLIENT_OPTS=test"); + conf.setStrings(MRJobConfig.MAP_LOG_LEVEL, "WARN"); + conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, false); + Job job = app.submit(conf); + app.waitForState(job, JobState.SUCCEEDED); + app.verifyCompleted(); + + Assert.assertTrue("HADOOP_ROOT_LOGGER not set for job", + app.cmdEnvironment.containsKey("HADOOP_ROOT_LOGGER")); + Assert.assertEquals("WARN,console", + app.cmdEnvironment.get("HADOOP_ROOT_LOGGER")); + Assert.assertTrue("HADOOP_CLIENT_OPTS not set for job", + app.cmdEnvironment.containsKey("HADOOP_CLIENT_OPTS")); + Assert.assertEquals("test", app.cmdEnvironment.get("HADOOP_CLIENT_OPTS")); + + // Try one more. + app = new MyMRApp(1, 0, true, this.getClass().getName(), true); + conf = new Configuration(); + conf.set(JobConf.MAPRED_MAP_TASK_ENV, "HADOOP_ROOT_LOGGER=trace"); + job = app.submit(conf); + app.waitForState(job, JobState.SUCCEEDED); + app.verifyCompleted(); + + Assert.assertTrue("HADOOP_ROOT_LOGGER not set for job", + app.cmdEnvironment.containsKey("HADOOP_ROOT_LOGGER")); + Assert.assertEquals("trace", + app.cmdEnvironment.get("HADOOP_ROOT_LOGGER")); + } }