Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DDEFBE4FE for ; Wed, 21 Nov 2012 21:09:18 +0000 (UTC) Received: (qmail 69283 invoked by uid 500); 21 Nov 2012 21:09:18 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 68571 invoked by uid 500); 21 Nov 2012 21:09:17 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 68511 invoked by uid 99); 21 Nov 2012 21:09:17 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 21 Nov 2012 21:09:17 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 21 Nov 2012 21:09:13 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 412432388A66; Wed, 21 Nov 2012 21:08:52 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1412297 - in /hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project: ./ conf/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory... Date: Wed, 21 Nov 2012 21:08:51 -0000 To: mapreduce-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20121121210852.412432388A66@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: szetszwo Date: Wed Nov 21 21:08:45 2012 New Revision: 1412297 URL: http://svn.apache.org/viewvc?rev=1412297&view=rev Log: Merge r1410998 through r1412282 from trunk. Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/conf/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (props changed) hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1410998-1412282 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/CHANGES.txt ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1410998-1412282 Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/conf/ ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1410998-1412282 Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml?rev=1412297&r1=1412296&r2=1412297&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml Wed Nov 21 21:08:45 2012 @@ -89,11 +89,6 @@ test-compile - - - mrapp-generated-classpath - - maven-antrun-plugin Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1412297&r1=1412296&r2=1412297&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java Wed Nov 21 21:08:45 2012 @@ -601,7 +601,7 @@ public class JobHistoryEventHandler exte setSummarySlotSeconds(summary, context.getJob(jobId).getAllCounters()); break; default: - throw new YarnException("Invalid event type"); + break; } } Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1412297&r1=1412296&r2=1412297&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java Wed Nov 21 21:08:45 2012 @@ -18,13 +18,8 @@ package org.apache.hadoop.mapreduce.v2.util; -import java.io.BufferedReader; -import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.net.URI; -import java.net.URL; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -134,62 +129,24 @@ public class MRApps extends Apps { private static void setMRFrameworkClasspath( Map environment, Configuration conf) throws IOException { - InputStream classpathFileStream = null; - BufferedReader reader = null; - try { - // Get yarn mapreduce-app classpath from generated classpath - // Works if compile time env is same as runtime. Mainly tests. - ClassLoader thisClassLoader = - Thread.currentThread().getContextClassLoader(); - String mrAppGeneratedClasspathFile = "mrapp-generated-classpath"; - classpathFileStream = - thisClassLoader.getResourceAsStream(mrAppGeneratedClasspathFile); - - // Put the file itself on classpath for tasks. - URL classpathResource = thisClassLoader - .getResource(mrAppGeneratedClasspathFile); - if (classpathResource != null) { - String classpathElement = classpathResource.getFile(); - if (classpathElement.contains("!")) { - classpathElement = classpathElement.substring(0, - classpathElement.indexOf("!")); - } else { - classpathElement = new File(classpathElement).getParent(); - } - Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), - classpathElement); - } - - if (classpathFileStream != null) { - reader = new BufferedReader(new InputStreamReader(classpathFileStream, - Charsets.UTF_8)); - String cp = reader.readLine(); - if (cp != null) { - Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), - cp.trim()); - } - } - - // Add standard Hadoop classes - for (String c : conf.getStrings( - YarnConfiguration.YARN_APPLICATION_CLASSPATH, - YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { - Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c - .trim()); - } - for (String c : conf.getStrings( - MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, - MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH)) { - Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c - .trim()); - } - } finally { - if (classpathFileStream != null) { - classpathFileStream.close(); - } - if (reader != null) { - reader.close(); - } + // Propagate the system classpath when using the mini cluster + if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { + Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), + System.getProperty("java.class.path")); + } + + // Add standard Hadoop classes + for (String c : conf.getStrings( + YarnConfiguration.YARN_APPLICATION_CLASSPATH, + YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { + Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c + .trim()); + } + for (String c : conf.getStrings( + MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH, + MRJobConfig.DEFAULT_MAPREDUCE_APPLICATION_CLASSPATH)) { + Apps.addToEnvironment(environment, Environment.CLASSPATH.name(), c + .trim()); } // TODO: Remove duplicates. } Propchange: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml ------------------------------------------------------------------------------ Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1410998-1412282 Modified: hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java?rev=1412297&r1=1412296&r2=1412297&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/security/ssl/TestEncryptedShuffle.java Wed Nov 21 21:08:45 2012 @@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.RunningJ import org.apache.hadoop.mapreduce.MRConfig; import org.apache.hadoop.security.ssl.KeyStoreTestUtil; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -52,6 +53,8 @@ public class TestEncryptedShuffle { private static final String BASEDIR = System.getProperty("test.build.dir", "target/test-dir") + "/" + TestEncryptedShuffle.class.getSimpleName(); + + private String classpathDir; @BeforeClass public static void setUp() throws Exception { @@ -62,27 +65,12 @@ public class TestEncryptedShuffle { @Before public void createCustomYarnClasspath() throws Exception { - String classpathDir = - KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); - - URL url = Thread.currentThread().getContextClassLoader(). - getResource("mrapp-generated-classpath"); - File f = new File(url.getPath()); - BufferedReader reader = new BufferedReader(new FileReader(f)); - String cp = reader.readLine(); - cp = cp + ":" + classpathDir; - f = new File(classpathDir, "mrapp-generated-classpath"); - Writer writer = new FileWriter(f); - writer.write(cp); - writer.close(); + classpathDir = KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); new File(classpathDir, "core-site.xml").delete(); } @After public void cleanUpMiniClusterSpecialConfig() throws Exception { - String classpathDir = - KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); - new File(classpathDir, "mrapp-generated-classpath").delete(); new File(classpathDir, "core-site.xml").delete(); String keystoresDir = new File(BASEDIR).getAbsolutePath(); KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, classpathDir); @@ -98,6 +86,9 @@ public class TestEncryptedShuffle { conf.set("dfs.block.access.token.enable", "false"); conf.set("dfs.permissions", "true"); conf.set("hadoop.security.authentication", "simple"); + String cp = conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH) + + File.pathSeparator + classpathDir; + conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, cp); dfsCluster = new MiniDFSCluster(conf, 1, true, null); FileSystem fileSystem = dfsCluster.getFileSystem(); fileSystem.mkdirs(new Path("/tmp")); @@ -113,8 +104,6 @@ public class TestEncryptedShuffle { mrCluster = MiniMRClientClusterFactory.create(this.getClass(), 1, conf); // so the minicluster conf is avail to the containers. - String classpathDir = - KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class); Writer writer = new FileWriter(classpathDir + "/core-site.xml"); mrCluster.getConfig().writeXml(writer); writer.close();