Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E847CD438 for ; Mon, 10 Dec 2012 18:07:48 +0000 (UTC) Received: (qmail 62769 invoked by uid 500); 10 Dec 2012 18:07:48 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 62687 invoked by uid 500); 10 Dec 2012 18:07:48 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 62678 invoked by uid 99); 10 Dec 2012 18:07:47 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 10 Dec 2012 18:07:47 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 10 Dec 2012 18:07:44 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id F00B02388A91; Mon, 10 Dec 2012 18:07:22 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1419618 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ Date: Mon, 10 Dec 2012 18:07:22 -0000 To: mapreduce-commits@hadoop.apache.org From: tucu@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20121210180722.F00B02388A91@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: tucu Date: Mon Dec 10 18:07:20 2012 New Revision: 1419618 URL: http://svn.apache.org/viewvc?rev=1419618&view=rev Log: MAPREDUCE-4703. Add the ability to start the MiniMRClientCluster using the configurations used before it is being stopped. (ahmed.radwan via tucu) Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1419618&r1=1419617&r2=1419618&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Mon Dec 10 18:07:20 2012 @@ -171,6 +171,9 @@ Release 2.0.3-alpha - Unreleased MAPREDUCE-4723. Fix warnings found by findbugs 2. (Sandy Ryza via eli) + MAPREDUCE-4703. Add the ability to start the MiniMRClientCluster using + the configurations used before it is being stopped. (ahmed.radwan via tucu) + OPTIMIZATIONS BUG FIXES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java?rev=1419618&r1=1419617&r2=1419618&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientCluster.java Mon Dec 10 18:07:20 2012 @@ -31,6 +31,11 @@ public interface MiniMRClientCluster { public void start() throws IOException; + /** + * Stop and start back the cluster using the same configuration. + */ + public void restart() throws IOException; + public void stop() throws IOException; public Configuration getConfig() throws IOException; Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java?rev=1419618&r1=1419617&r2=1419618&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRClientClusterFactory.java Mon Dec 10 18:07:20 2012 @@ -67,6 +67,10 @@ public class MiniMRClientClusterFactory MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(caller .getName(), noOfNMs); + job.getConfiguration().set("minimrclientcluster.caller.name", + caller.getName()); + job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number", + noOfNMs); miniMRYarnCluster.init(job.getConfiguration()); miniMRYarnCluster.start(); Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java?rev=1419618&r1=1419617&r2=1419618&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MiniMRYarnClusterAdapter.java Mon Dec 10 18:07:20 2012 @@ -18,8 +18,13 @@ package org.apache.hadoop.mapred; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.service.Service.STATE; /** * An adapter for MiniMRYarnCluster providing a MiniMRClientCluster interface. @@ -29,6 +34,8 @@ public class MiniMRYarnClusterAdapter im private MiniMRYarnCluster miniMRYarnCluster; + private static final Log LOG = LogFactory.getLog(MiniMRYarnClusterAdapter.class); + public MiniMRYarnClusterAdapter(MiniMRYarnCluster miniMRYarnCluster) { this.miniMRYarnCluster = miniMRYarnCluster; } @@ -48,4 +55,22 @@ public class MiniMRYarnClusterAdapter im miniMRYarnCluster.stop(); } + @Override + public void restart() { + if (!miniMRYarnCluster.getServiceState().equals(STATE.STARTED)){ + LOG.warn("Cannot restart the mini cluster, start it first"); + return; + } + Configuration oldConf = new Configuration(getConfig()); + String callerName = oldConf.get("minimrclientcluster.caller.name", + this.getClass().getName()); + int noOfNMs = oldConf.getInt("minimrclientcluster.nodemanagers.number", 1); + oldConf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true); + oldConf.setBoolean(JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS, true); + stop(); + miniMRYarnCluster = new MiniMRYarnCluster(callerName, noOfNMs); + miniMRYarnCluster.init(oldConf); + miniMRYarnCluster.start(); + } + } Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java?rev=1419618&r1=1419617&r2=1419618&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRClientCluster.java Mon Dec 10 18:07:20 2012 @@ -32,6 +32,8 @@ import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -92,6 +94,65 @@ public class TestMiniMRClientCluster { } @Test + public void testRestart() throws Exception { + + String rmAddress1 = mrCluster.getConfig().get(YarnConfiguration.RM_ADDRESS); + String rmAdminAddress1 = mrCluster.getConfig().get( + YarnConfiguration.RM_ADMIN_ADDRESS); + String rmSchedAddress1 = mrCluster.getConfig().get( + YarnConfiguration.RM_SCHEDULER_ADDRESS); + String rmRstrackerAddress1 = mrCluster.getConfig().get( + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS); + String rmWebAppAddress1 = mrCluster.getConfig().get( + YarnConfiguration.RM_WEBAPP_ADDRESS); + + String mrHistAddress1 = mrCluster.getConfig().get( + JHAdminConfig.MR_HISTORY_ADDRESS); + String mrHistWebAppAddress1 = mrCluster.getConfig().get( + JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS); + + mrCluster.restart(); + + String rmAddress2 = mrCluster.getConfig().get(YarnConfiguration.RM_ADDRESS); + String rmAdminAddress2 = mrCluster.getConfig().get( + YarnConfiguration.RM_ADMIN_ADDRESS); + String rmSchedAddress2 = mrCluster.getConfig().get( + YarnConfiguration.RM_SCHEDULER_ADDRESS); + String rmRstrackerAddress2 = mrCluster.getConfig().get( + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS); + String rmWebAppAddress2 = mrCluster.getConfig().get( + YarnConfiguration.RM_WEBAPP_ADDRESS); + + String mrHistAddress2 = mrCluster.getConfig().get( + JHAdminConfig.MR_HISTORY_ADDRESS); + String mrHistWebAppAddress2 = mrCluster.getConfig().get( + JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS); + + assertEquals("Address before restart: " + rmAddress1 + + " is different from new address: " + rmAddress2, rmAddress1, + rmAddress2); + assertEquals("Address before restart: " + rmAdminAddress1 + + " is different from new address: " + rmAdminAddress2, + rmAdminAddress1, rmAdminAddress2); + assertEquals("Address before restart: " + rmSchedAddress1 + + " is different from new address: " + rmSchedAddress2, + rmSchedAddress1, rmSchedAddress2); + assertEquals("Address before restart: " + rmRstrackerAddress1 + + " is different from new address: " + rmRstrackerAddress2, + rmRstrackerAddress1, rmRstrackerAddress2); + assertEquals("Address before restart: " + rmWebAppAddress1 + + " is different from new address: " + rmWebAppAddress2, + rmWebAppAddress1, rmWebAppAddress2); + assertEquals("Address before restart: " + mrHistAddress1 + + " is different from new address: " + mrHistAddress2, mrHistAddress1, + mrHistAddress2); + assertEquals("Address before restart: " + mrHistWebAppAddress1 + + " is different from new address: " + mrHistWebAppAddress2, + mrHistWebAppAddress1, mrHistWebAppAddress2); + + } + + @Test public void testJob() throws Exception { final Job job = createJob(); org.apache.hadoop.mapreduce.lib.input.FileInputFormat.setInputPaths(job,