Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 1ABA69E06 for ; Thu, 19 Apr 2012 20:33:36 +0000 (UTC) Received: (qmail 61159 invoked by uid 500); 19 Apr 2012 20:33:35 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 61049 invoked by uid 500); 19 Apr 2012 20:33:35 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 60850 invoked by uid 99); 19 Apr 2012 20:33:35 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 19 Apr 2012 20:33:35 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 19 Apr 2012 20:33:33 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 1BC072388B6C for ; Thu, 19 Apr 2012 20:33:13 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1328109 - in /hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming: TestFileArgs.java TestMultipleArchiveFiles.java TestMultipleCachefiles.java TestStreamingTaskLog.java TestSymLink.java Date: Thu, 19 Apr 2012 20:33:12 -0000 To: common-commits@hadoop.apache.org From: tucu@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120419203313.1BC072388B6C@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: tucu Date: Thu Apr 19 20:33:11 2012 New Revision: 1328109 URL: http://svn.apache.org/viewvc?rev=1328109&view=rev Log: MAPREDUCE-3867. MiniMRYarn/MiniYarn uses fixed ports (tucu) Modified: hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java Modified: hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java (original) +++ hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java Thu Apr 19 20:33:11 2012 @@ -20,12 +20,13 @@ package org.apache.hadoop.streaming; import java.io.DataOutputStream; import java.io.IOException; +import java.util.Map; + import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.MiniMRCluster; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.junit.After; import org.junit.Before; @@ -38,8 +39,6 @@ public class TestFileArgs extends TestSt private MiniDFSCluster dfs = null; private MiniMRCluster mr = null; private FileSystem fileSys = null; - private String strJobTracker = null; - private String strNamenode = null; private String namenode = null; private Configuration conf = null; @@ -56,8 +55,6 @@ public class TestFileArgs extends TestSt fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().getAuthority(); mr = new MiniMRCluster(1, namenode, 1); - strJobTracker = JTConfig.JT_IPC_ADDRESS + "=localhost:" + mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS); - strNamenode = "fs.default.name=" + mr.createJobConf().get("fs.default.name"); map = LS_PATH; FileSystem.setDefaultUri(conf, "hdfs://" + namenode); @@ -100,18 +97,16 @@ public class TestFileArgs extends TestSt @Override protected String[] genArgs() { + for (Map.Entry entry : mr.createJobConf()) { + args.add("-jobconf"); + args.add(entry.getKey() + "=" + entry.getValue()); + } args.add("-file"); args.add(new java.io.File("target/sidefile").getAbsolutePath()); args.add("-numReduceTasks"); args.add("0"); args.add("-jobconf"); - args.add(strNamenode); - args.add("-jobconf"); - args.add(strJobTracker); - args.add("-jobconf"); args.add("mapred.jar=" + STREAMING_JAR); - args.add("-jobconf"); - args.add("mapreduce.framework.name=yarn"); args.add("-verbose"); return super.genArgs(); } Modified: hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java (original) +++ hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java Thu Apr 19 20:33:11 2012 @@ -19,14 +19,10 @@ package org.apache.hadoop.streaming; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.io.DataOutputStream; -import java.io.InputStreamReader; -import java.io.BufferedReader; -import java.util.Arrays; +import java.util.Map; import java.util.zip.ZipEntry; -import java.util.jar.JarOutputStream; import java.util.zip.ZipOutputStream; import org.apache.commons.logging.Log; @@ -37,12 +33,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.*; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.util.StringUtils; - -import org.junit.Test; -import static org.junit.Assert.*; /** * This class tests cacheArchive option of streaming @@ -66,8 +57,6 @@ public class TestMultipleArchiveFiles ex private MiniDFSCluster dfs = null; private MiniMRCluster mr = null; private FileSystem fileSys = null; - private String strJobTracker = null; - private String strNamenode = null; private String namenode = null; public TestMultipleArchiveFiles() throws Exception { @@ -80,8 +69,6 @@ public class TestMultipleArchiveFiles ex fileSys = dfs.getFileSystem(); namenode = fileSys.getUri().getAuthority(); mr = new MiniMRCluster(1, namenode, 1); - strJobTracker = JTConfig.JT_IPC_ADDRESS + "=localhost:" + mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS); - strNamenode = "fs.default.name=" + mr.createJobConf().get("fs.default.name"); map = "xargs cat"; reduce = "cat"; @@ -123,6 +110,10 @@ public class TestMultipleArchiveFiles ex String cache1 = workDir + CACHE_ARCHIVE_1 + "#symlink1"; String cache2 = workDir + CACHE_ARCHIVE_2 + "#symlink2"; + for (Map.Entry entry : mr.createJobConf()) { + args.add("-jobconf"); + args.add(entry.getKey() + "=" + entry.getValue()); + } args.add("-jobconf"); args.add("mapreduce.job.reduces=1"); args.add("-cacheArchive"); @@ -130,13 +121,7 @@ public class TestMultipleArchiveFiles ex args.add("-cacheArchive"); args.add(cache2); args.add("-jobconf"); - args.add(strNamenode); - args.add("-jobconf"); - args.add(strJobTracker); - args.add("-jobconf"); args.add("mapred.jar=" + STREAMING_JAR); - args.add("-jobconf"); - args.add("mapreduce.framework.name=yarn"); return super.genArgs(); } Modified: hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java (original) +++ hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java Thu Apr 19 20:33:11 2012 @@ -22,8 +22,9 @@ import java.io.BufferedReader; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.io.PrintWriter; -import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import org.junit.Test; import static org.junit.Assert.*; @@ -36,7 +37,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.MiniMRCluster; import org.apache.hadoop.mapred.Utils; -import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; + /** * This test case tests the symlink creation * utility provided by distributed caching @@ -73,15 +74,18 @@ public class TestMultipleCachefiles String namenode = fileSys.getUri().toString(); mr = new MiniMRCluster(1, namenode, 3); - String strJobtracker = JTConfig.JT_IPC_ADDRESS + "=localhost:" + mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS); - String strNamenode = "fs.default.name=" + mr.createJobConf().get("fs.default.name"); + + List args = new ArrayList(); + for (Map.Entry entry : mr.createJobConf()) { + args.add("-jobconf"); + args.add(entry.getKey() + "=" + entry.getValue()); + } + String argv[] = new String[] { "-input", INPUT_FILE, "-output", OUTPUT_DIR, "-mapper", map, "-reducer", reduce, - "-jobconf", strNamenode, - "-jobconf", strJobtracker, "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp"), "-jobconf", JobConf.MAPRED_MAP_TASK_JAVA_OPTS + "=" + @@ -98,9 +102,13 @@ public class TestMultipleCachefiles "-cacheFile", fileSys.getUri() + CACHE_FILE + "#" + mapString, "-cacheFile", fileSys.getUri() + CACHE_FILE_2 + "#" + mapString2, "-jobconf", "mapred.jar=" + TestStreaming.STREAMING_JAR, - "-jobconf", "mapreduce.framework.name=yarn" }; + for (String arg : argv) { + args.add(arg); + } + argv = args.toArray(new String[args.size()]); + fileSys.delete(new Path(OUTPUT_DIR), true); DataOutputStream file = fileSys.create(new Path(INPUT_FILE)); Modified: hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java (original) +++ hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java Thu Apr 19 20:33:11 2012 @@ -19,6 +19,9 @@ package org.apache.hadoop.streaming; import java.io.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -47,20 +50,30 @@ public class TestStreamingTaskLog { final long USERLOG_LIMIT_KB = 5;//consider 5kb as logSize String[] genArgs() { - return new String[] { + + List args = new ArrayList(); + for (Map.Entry entry : mr.createJobConf()) { + args.add("-jobconf"); + args.add(entry.getKey() + "=" + entry.getValue()); + } + + String[] argv = new String[] { "-input", inputPath.toString(), "-output", outputPath.toString(), "-mapper", map, "-reducer", StreamJob.REDUCE_NONE, - "-jobconf", "mapred.job.tracker=" + mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS), - "-jobconf", "fs.default.name=" + fs.getUri().toString(), "-jobconf", "mapred.map.tasks=1", "-jobconf", "keep.failed.task.files=true", "-jobconf", "mapreduce.task.userlog.limit.kb=" + USERLOG_LIMIT_KB, "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp"), "-jobconf", "mapred.jar=" + TestStreaming.STREAMING_JAR, - "-jobconf", "mapreduce.framework.name=yarn" }; + + for (String arg : argv) { + args.add(arg); + } + argv = args.toArray(new String[args.size()]); + return argv; } /** Modified: hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java?rev=1328109&r1=1328108&r2=1328109&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java (original) +++ hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java Thu Apr 19 20:33:11 2012 @@ -21,6 +21,9 @@ package org.apache.hadoop.streaming; import java.io.BufferedReader; import java.io.DataOutputStream; import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import org.junit.Test; import static org.junit.Assert.*; @@ -62,17 +65,20 @@ public class TestSymLink FileSystem fileSys = dfs.getFileSystem(); String namenode = fileSys.getUri().toString(); mr = new MiniMRCluster(1, namenode, 3); + + List args = new ArrayList(); + for (Map.Entry entry : mr.createJobConf()) { + args.add("-jobconf"); + args.add(entry.getKey() + "=" + entry.getValue()); + } + // During tests, the default Configuration will use a local mapred // So don't specify -config or -cluster - String strJobtracker = JTConfig.JT_IPC_ADDRESS + "=localhost:" + mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS); - String strNamenode = "fs.default.name=" + mr.createJobConf().get("fs.default.name"); String argv[] = new String[] { "-input", INPUT_FILE, "-output", OUTPUT_DIR, "-mapper", map, "-reducer", reduce, - "-jobconf", strNamenode, - "-jobconf", strJobtracker, "-jobconf", "stream.tmpdir="+System.getProperty("test.build.data","/tmp"), "-jobconf", JobConf.MAPRED_MAP_TASK_JAVA_OPTS+ "=" + @@ -88,9 +94,13 @@ public class TestSymLink conf.get(JobConf.MAPRED_TASK_JAVA_OPTS, "")), "-cacheFile", fileSys.getUri() + CACHE_FILE + "#testlink", "-jobconf", "mapred.jar=" + TestStreaming.STREAMING_JAR, - "-jobconf", "mapreduce.framework.name=yarn" }; + for (String arg : argv) { + args.add(arg); + } + argv = args.toArray(new String[args.size()]); + fileSys.delete(new Path(OUTPUT_DIR), true); DataOutputStream file = fileSys.create(new Path(INPUT_FILE));