Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 896A6FFC5 for ; Fri, 5 Apr 2013 02:44:03 +0000 (UTC) Received: (qmail 28018 invoked by uid 500); 5 Apr 2013 02:44:03 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 27953 invoked by uid 500); 5 Apr 2013 02:44:03 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 27944 invoked by uid 99); 5 Apr 2013 02:44:03 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 05 Apr 2013 02:44:03 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 05 Apr 2013 02:43:54 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 65C5523889EB for ; Fri, 5 Apr 2013 02:43:32 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1464815 - in /hadoop/common/branches/HDFS-2802: ./ hadoop-tools/hadoop-gridmix/ hadoop-tools/hadoop-gridmix/dev-support/ hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/ hadoop-tools/hadoop-gridmix/src/main/java/... Date: Fri, 05 Apr 2013 02:43:31 -0000 To: common-commits@hadoop.apache.org From: szetszwo@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130405024332.65C5523889EB@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: szetszwo Date: Fri Apr 5 02:43:29 2013 New Revision: 1464815 URL: http://svn.apache.org/r1464815 Log: Merge r1462698 through r1464807 from trunk. Added: hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/dev-support/ - copied from r1464807, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/dev-support/ hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml - copied unchanged from r1464807, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/dev-support/findbugs-exclude.xml hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java - copied unchanged from r1464807, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/CommonJobTest.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java - copied unchanged from r1464807, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestDistCacheEmulation.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridMixClasses.java - copied unchanged from r1464807, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridMixClasses.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java - copied unchanged from r1464807, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSubmission.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestLoadJob.java - copied unchanged from r1464807, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestLoadJob.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestSleepJob.java - copied unchanged from r1464807, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestSleepJob.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount.json - copied unchanged from r1464807, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount.json hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount2.json - copied unchanged from r1464807, hadoop/common/trunk/hadoop-tools/hadoop-gridmix/src/test/resources/data/wordcount2.json Modified: hadoop/common/branches/HDFS-2802/ (props changed) hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/pom.xml hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobMonitor.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobFactory.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobProducer.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java Propchange: hadoop/common/branches/HDFS-2802/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/branch-2.0.4-alpha:r1463804 Merged /hadoop/common/trunk:r1462698-1464807 Modified: hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/pom.xml?rev=1464815&r1=1464814&r2=1464815&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/pom.xml (original) +++ hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/pom.xml Fri Apr 5 02:43:29 2013 @@ -91,10 +91,25 @@ test-jar test + + org.mockito + mockito-all + test + + + org.codehaus.mojo + findbugs-maven-plugin + + true + true + ${basedir}/dev-support/findbugs-exclude.xml + Max + + org.apache.maven.plugins maven-antrun-plugin @@ -116,6 +131,15 @@ + org.apache.rat + apache-rat-plugin + + + src/test/resources/data/* + + + + org.apache.maven.plugins maven-jar-plugin Modified: hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java?rev=1464815&r1=1464814&r2=1464815&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/Gridmix.java Fri Apr 5 02:43:29 2013 @@ -36,7 +36,9 @@ import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.mapred.gridmix.GenerateData.DataStatistics; import org.apache.hadoop.mapred.gridmix.Statistics.JobStats; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.Tool; @@ -693,7 +695,7 @@ public class Gridmix extends Configured try { res = ToolRunner.run(new Configuration(), new Gridmix(argv), argv); } finally { - System.exit(res); + ExitUtil.terminate(res); } } @@ -800,6 +802,10 @@ public class Gridmix extends Configured */ void abort(); } - + // it is need for tests + protected Summarizer getSummarizer() { + return summarizer; + } + } Modified: hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobMonitor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobMonitor.java?rev=1464815&r1=1464814&r2=1464815&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobMonitor.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/JobMonitor.java Fri Apr 5 02:43:29 2013 @@ -86,9 +86,7 @@ class JobMonitor implements Gridmix.Comp * Add a running job's status to the polling queue. */ public void add(JobStats job) throws InterruptedException { - synchronized (runningJobs) { runningJobs.put(job); - } } /** @@ -147,12 +145,10 @@ class JobMonitor implements Gridmix.Comp boolean shutdown; while (true) { try { - synchronized (runningJobs) { - synchronized (mJobs) { - graceful = JobMonitor.this.graceful; - shutdown = JobMonitor.this.shutdown; - runningJobs.drainTo(mJobs); - } + synchronized (mJobs) { + graceful = JobMonitor.this.graceful; + shutdown = JobMonitor.this.shutdown; + runningJobs.drainTo(mJobs); } // shutdown conditions; either shutdown requested and all jobs @@ -160,11 +156,9 @@ class JobMonitor implements Gridmix.Comp // submitted jobs not in the monitored set if (shutdown) { if (!graceful) { - synchronized (runningJobs) { - while (!runningJobs.isEmpty()) { - synchronized (mJobs) { - runningJobs.drainTo(mJobs); - } + while (!runningJobs.isEmpty()) { + synchronized (mJobs) { + runningJobs.drainTo(mJobs); } } break; Modified: hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java?rev=1464815&r1=1464814&r2=1464815&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SerialJobFactory.java Fri Apr 5 02:43:29 2013 @@ -175,4 +175,8 @@ public class SerialJobFactory extends Jo LOG.info(" Starting Serial submission "); this.rThread.start(); } + // it is need for test + void setDistCacheEmulator(DistributedCacheEmulator e) { + jobCreator.setDistCacheEmulator(e); + } } Modified: hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java?rev=1464815&r1=1464814&r2=1464815&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/SleepJob.java Fri Apr 5 02:43:29 2013 @@ -84,7 +84,7 @@ public class SleepJob extends GridmixJob String[] hosts) throws IOException { super(conf, submissionMillis, jobdesc, outRoot, ugi, seq); this.fakeLocations = numLocations; - this.hosts = hosts; + this.hosts = hosts.clone(); this.selector = (fakeLocations > 0)? new Selector(hosts.length, (float) fakeLocations / hosts.length, rand.get()) : null; this.mapTasksOnly = conf.getBoolean(SLEEPJOB_MAPTASK_ONLY, false); @@ -289,9 +289,9 @@ public class SleepJob extends GridmixJob this.id = id; this.sleepDuration = sleepDuration; nSpec = reduceDurations.length; - this.reduceDurations = reduceDurations; + this.reduceDurations = reduceDurations.clone(); this.nMaps = nMaps; - this.locations = locations; + this.locations = locations.clone(); } @Override @@ -349,7 +349,7 @@ public class SleepJob extends GridmixJob @Override public String[] getLocations() { - return locations; + return locations.clone(); } } Modified: hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java?rev=1464815&r1=1464814&r2=1464815&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/emulators/resourceusage/TotalHeapUsageEmulatorPlugin.java Fri Apr 5 02:43:29 2013 @@ -93,7 +93,7 @@ implements ResourceUsageEmulatorPlugin { private static final float DEFAULT_HEAP_LOAD_RATIO = 0.1F; - public static int ONE_MB = 1024 * 1024; + public static final int ONE_MB = 1024 * 1024; /** * Defines the core heap usage emulation algorithm. This engine is expected @@ -129,7 +129,8 @@ implements ResourceUsageEmulatorPlugin { public static class DefaultHeapUsageEmulator implements HeapUsageEmulatorCore { // store the unit loads in a list - protected static ArrayList heapSpace = new ArrayList(); + protected static final ArrayList heapSpace = + new ArrayList(); /** * Increase heap usage by current process by the given amount. Modified: hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobFactory.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobFactory.java?rev=1464815&r1=1464814&r2=1464815&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobFactory.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobFactory.java Fri Apr 5 02:43:29 2013 @@ -29,13 +29,13 @@ import java.util.concurrent.CountDownLat /** * Component generating random job traces for testing on a single node. */ -class DebugJobFactory { +public class DebugJobFactory { interface Debuggable { ArrayList getSubmitted(); } - public static JobFactory getFactory( + public static JobFactory getFactory( JobSubmitter submitter, Path scratch, int numJobs, Configuration conf, CountDownLatch startFlag, UserResolver resolver) throws IOException { GridmixJobSubmissionPolicy policy = GridmixJobSubmissionPolicy.getPolicy( Modified: hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobProducer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobProducer.java?rev=1464815&r1=1464814&r2=1464815&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobProducer.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/DebugJobProducer.java Fri Apr 5 02:43:29 2013 @@ -216,7 +216,7 @@ public class DebugJobProducer implements if (user == null) { user = String.format("foobar%d", id); } - GridmixTestUtils.createHomeAndStagingDirectory(user, (JobConf)conf); + GridmixTestUtils.createHomeAndStagingDirectory(user, conf); return user; } @@ -300,7 +300,7 @@ public class DebugJobProducer implements @Override public String getQueueName() { - String qName = "q" + ((id % 2) + 1); + String qName = "default"; return qName; } Modified: hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java?rev=1464815&r1=1464814&r2=1464815&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/GridmixTestUtils.java Fri Apr 5 02:43:29 2013 @@ -4,55 +4,76 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.mapred.MiniMRCluster; -import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MiniMRClientCluster; +import org.apache.hadoop.mapred.MiniMRClientClusterFactory; +import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.ShellBasedUnixGroupsMapping; -import org.apache.hadoop.security.Groups; import java.io.IOException; /** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at *

* http://www.apache.org/licenses/LICENSE-2.0 *

* Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. */ public class GridmixTestUtils { private static final Log LOG = LogFactory.getLog(GridmixTestUtils.class); static final Path DEST = new Path("/gridmix"); static FileSystem dfs = null; static MiniDFSCluster dfsCluster = null; - static MiniMRCluster mrCluster = null; + static MiniMRClientCluster mrvl = null; + protected static final String GRIDMIX_USE_QUEUE_IN_TRACE = + "gridmix.job-submission.use-queue-in-trace"; + protected static final String GRIDMIX_DEFAULT_QUEUE = + "gridmix.job-submission.default-queue"; - public static void initCluster() throws IOException { + public static void initCluster(Class caller) throws IOException { Configuration conf = new Configuration(); - conf.set("mapred.queue.names", "default,q1,q2"); - dfsCluster = new MiniDFSCluster(conf, 3, true, null); +// conf.set("mapred.queue.names", "default,q1,q2"); + conf.set("mapred.queue.names", "default"); + conf.set("yarn.scheduler.capacity.root.queues", "default"); + conf.set("yarn.scheduler.capacity.root.default.capacity", "100.0"); + + + conf.setBoolean(GRIDMIX_USE_QUEUE_IN_TRACE, false); + conf.set(GRIDMIX_DEFAULT_QUEUE, "default"); + + + dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).format(true) + .build();// MiniDFSCluster(conf, 3, true, null); dfs = dfsCluster.getFileSystem(); conf.set(JTConfig.JT_RETIREJOBS, "false"); - mrCluster = new MiniMRCluster(3, dfs.getUri().toString(), 1, null, null, - new JobConf(conf)); + mrvl = MiniMRClientClusterFactory.create(caller, 2, conf); + + conf = mrvl.getConfig(); + String[] files = conf.getStrings(MRJobConfig.CACHE_FILES); + if (files != null) { + String[] timestamps = new String[files.length]; + for (int i = 0; i < files.length; i++) { + timestamps[i] = Long.toString(System.currentTimeMillis()); + } + conf.setStrings(MRJobConfig.CACHE_FILE_TIMESTAMPS, timestamps); + } + } public static void shutdownCluster() throws IOException { - if (mrCluster != null) { - mrCluster.shutdown(); + if (mrvl != null) { + mrvl.stop(); } if (dfsCluster != null) { dfsCluster.shutdown(); @@ -61,23 +82,25 @@ public class GridmixTestUtils { /** * Methods to generate the home directory for dummy users. - * + * * @param conf */ - public static void createHomeAndStagingDirectory(String user, JobConf conf) { + public static void createHomeAndStagingDirectory(String user, + Configuration conf) { try { FileSystem fs = dfsCluster.getFileSystem(); String path = "/user/" + user; Path homeDirectory = new Path(path); - if(fs.exists(homeDirectory)) { - fs.delete(homeDirectory,true); - } - LOG.info("Creating Home directory : " + homeDirectory); - fs.mkdirs(homeDirectory); - changePermission(user,homeDirectory, fs); - Path stagingArea = - new Path(conf.get("mapreduce.jobtracker.staging.root.dir", - "/tmp/hadoop/mapred/staging")); + if (!fs.exists(homeDirectory)) { + LOG.info("Creating Home directory : " + homeDirectory); + fs.mkdirs(homeDirectory); + changePermission(user, homeDirectory, fs); + + } + changePermission(user, homeDirectory, fs); + Path stagingArea = new Path( + conf.get("mapreduce.jobtracker.staging.root.dir", + "/tmp/hadoop/mapred/staging")); LOG.info("Creating Staging root directory : " + stagingArea); fs.mkdirs(stagingArea); fs.setPermission(stagingArea, new FsPermission((short) 0777)); @@ -87,7 +110,7 @@ public class GridmixTestUtils { } static void changePermission(String user, Path homeDirectory, FileSystem fs) - throws IOException { + throws IOException { fs.setOwner(homeDirectory, user, ""); } } Modified: hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java?rev=1464815&r1=1464814&r2=1464815&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestGridmixSummary.java Fri Apr 5 02:43:29 2013 @@ -133,7 +133,7 @@ public class TestGridmixSummary { /** * A fake {@link JobFactory}. */ - @SuppressWarnings("unchecked") + @SuppressWarnings("rawtypes") private static class FakeJobFactory extends JobFactory { /** * A fake {@link JobStoryProducer} for {@link FakeJobFactory}. @@ -167,7 +167,7 @@ public class TestGridmixSummary { * Test {@link ExecutionSummarizer}. */ @Test - @SuppressWarnings("unchecked") + @SuppressWarnings({ "unchecked", "rawtypes" }) public void testExecutionSummarizer() throws IOException { Configuration conf = new Configuration(); @@ -360,7 +360,6 @@ public class TestGridmixSummary { * Test {@link ClusterSummarizer}. */ @Test - @SuppressWarnings("deprecation") public void testClusterSummarizer() throws IOException { ClusterSummarizer cs = new ClusterSummarizer(); Configuration conf = new Configuration(); @@ -374,13 +373,13 @@ public class TestGridmixSummary { assertEquals("JT name mismatch", jt, cs.getJobTrackerInfo()); assertEquals("NN name mismatch", nn, cs.getNamenodeInfo()); - ClusterStats cstats = ClusterStats.getClusterStats(); + ClusterStats cStats = ClusterStats.getClusterStats(); conf.set(JTConfig.JT_IPC_ADDRESS, "local"); conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "local"); JobClient jc = new JobClient(conf); - cstats.setClusterMetric(jc.getClusterStatus()); + cStats.setClusterMetric(jc.getClusterStatus()); - cs.update(cstats); + cs.update(cStats); // test assertEquals("Cluster summary test failed!", 1, cs.getMaxMapTasks()); Modified: hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1464815&r1=1464814&r2=1464815&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java Fri Apr 5 02:43:29 2013 @@ -875,7 +875,7 @@ public class StreamJob implements Tool { IdentifierResolver.TEXT_ID)); jobConf_.setClass("stream.map.output.reader.class", idResolver.getOutputReaderClass(), OutputReader.class); - if (isMapperACommand) { + if (isMapperACommand || jobConf_.get("stream.map.output") != null) { // if mapper is a command, then map output key/value classes come from the // idResolver jobConf_.setMapOutputKeyClass(idResolver.getOutputKeyClass()); @@ -891,7 +891,7 @@ public class StreamJob implements Tool { IdentifierResolver.TEXT_ID)); jobConf_.setClass("stream.reduce.output.reader.class", idResolver.getOutputReaderClass(), OutputReader.class); - if (isReducerACommand) { + if (isReducerACommand || jobConf_.get("stream.reduce.output") != null) { // if reducer is a command, then output key/value classes come from the // idResolver jobConf_.setOutputKeyClass(idResolver.getOutputKeyClass()); Modified: hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java?rev=1464815&r1=1464814&r2=1464815&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingOutputKeyValueTypes.java Fri Apr 5 02:43:29 2013 @@ -24,6 +24,7 @@ import org.apache.hadoop.mapred.MapReduc import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reducer; import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hadoop.mapreduce.MRJobConfig; import org.junit.Before; import org.junit.Test; @@ -177,6 +178,17 @@ public class TestStreamingOutputKeyValue args.add("0"); super.testCommandLine(); } + + @Test + public void testDefaultToIdentityReducer() throws Exception { + args.add("-mapper");args.add(map); + args.add("-jobconf"); + args.add("mapreduce.task.files.preserve.failedtasks=true"); + args.add("-jobconf"); + args.add("stream.tmpdir="+System.getProperty("test.build.data","/tmp")); + args.add("-inputformat");args.add(TextInputFormat.class.getName()); + super.testCommandLine(); + } @Override @Test Modified: hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java?rev=1464815&r1=1464814&r2=1464815&view=diff ============================================================================== --- hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java (original) +++ hadoop/common/branches/HDFS-2802/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TrApp.java Fri Apr 5 02:43:29 2013 @@ -43,7 +43,6 @@ public class TrApp // Note the dots translated to underscore: // property names have been escaped in PipeMapRed.safeEnvVarName() expectDefined("mapreduce_cluster_local_dir"); - expect("mapred_output_format_class", "org.apache.hadoop.mapred.TextOutputFormat"); expect("mapreduce_map_output_key_class", "org.apache.hadoop.io.Text"); expect("mapreduce_map_output_value_class", "org.apache.hadoop.io.Text");