hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1423426 - in /hadoop/common/branches/branch-1: CHANGES.txt src/test/org/apache/hadoop/mapred/MiniMRClientCluster.java src/test/org/apache/hadoop/mapred/MiniMRClusterAdapter.java src/test/org/apache/hadoop/mapred/TestMiniMRClientCluster.java
Date Tue, 18 Dec 2012 13:25:22 GMT
Author: tucu
Date: Tue Dec 18 13:25:20 2012
New Revision: 1423426

URL: http://svn.apache.org/viewvc?rev=1423426&view=rev
Log:
MAPREDUCE-4703. Add the ability to start the MiniMRClientCluster using the configurations
used before it is being stopped. (ahmed.radwan via tucu)

Modified:
    hadoop/common/branches/branch-1/CHANGES.txt
    hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClientCluster.java
    hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClusterAdapter.java
    hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestMiniMRClientCluster.java

Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1423426&r1=1423425&r2=1423426&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Tue Dec 18 13:25:20 2012
@@ -139,6 +139,9 @@ Release 1.2.0 - unreleased
 
     HADOOP-9098. Add missing license headers. (Arpit Agarwal via suresh)
 
+    MAPREDUCE-4703. Add the ability to start the MiniMRClientCluster using the 
+    configurations used before it is being stopped. (ahmed.radwan via tucu)
+
   OPTIMIZATIONS
 
     HDFS-2533. Backport: Remove needless synchronization on some FSDataSet

Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClientCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClientCluster.java?rev=1423426&r1=1423425&r2=1423426&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClientCluster.java
(original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClientCluster.java
Tue Dec 18 13:25:20 2012
@@ -31,6 +31,11 @@ public interface MiniMRClientCluster {
 
   public void start() throws IOException;
 
+  /**
+   * Stop and start back the cluster using the same configuration.
+   */
+  public void restart() throws IOException;
+
   public void stop() throws IOException;
 
   public Configuration getConfig() throws IOException;

Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClusterAdapter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClusterAdapter.java?rev=1423426&r1=1423425&r2=1423426&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClusterAdapter.java
(original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/MiniMRClusterAdapter.java
Tue Dec 18 13:25:20 2012
@@ -20,7 +20,11 @@ package org.apache.hadoop.mapred;
 
 import java.io.IOException;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 
 /**
  * An adapter for MiniMRCluster providing a MiniMRClientCluster interface. This
@@ -30,6 +34,8 @@ public class MiniMRClusterAdapter implem
 
   private MiniMRCluster miniMRCluster;
 
+  private static final Log LOG = LogFactory.getLog(MiniMRClusterAdapter.class);
+
   public MiniMRClusterAdapter(MiniMRCluster miniMRCluster) {
     this.miniMRCluster = miniMRCluster;
   }
@@ -50,4 +56,37 @@ public class MiniMRClusterAdapter implem
     miniMRCluster.shutdown();
   }
 
+  @Override
+  public void restart() throws IOException {
+    if (!miniMRCluster.getJobTrackerRunner().isActive()) {
+      LOG.warn("Cannot restart the mini cluster, start it first");
+      return;
+    }
+
+    int jobTrackerPort = miniMRCluster.getJobTrackerPort();
+    int taskTrackerPort = getConfig().getInt(
+        "mapred.task.tracker.report.address", 0);
+    int numtaskTrackers = miniMRCluster.getNumTaskTrackers();
+    String namenode = getConfig().get(FileSystem.FS_DEFAULT_NAME_KEY);
+
+    stop();
+    
+    // keep retrying to start the cluster for a max. of 30 sec.
+    for (int i = 0; i < 30; i++) {
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException e) {
+        throw new IOException(e);
+      }
+      try {
+        miniMRCluster = new MiniMRCluster(jobTrackerPort, taskTrackerPort,
+            numtaskTrackers, namenode, 1);
+        break;
+      } catch (Exception e) {
+        LOG.info("Retrying to start the cluster");
+      }
+    }
+
+  }
+
 }

Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestMiniMRClientCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestMiniMRClientCluster.java?rev=1423426&r1=1423425&r2=1423426&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestMiniMRClientCluster.java
(original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestMiniMRClientCluster.java
Tue Dec 18 13:25:20 2012
@@ -99,6 +99,29 @@ public class TestMiniMRClientCluster {
     validateCounters(job.getCounters(), 5, 25, 5, 5);
   }
 
+  @Test
+  public void testRestart() throws Exception {
+
+    String jobTrackerAddr1 = mrCluster.getConfig().get("mapred.job.tracker");
+    String taskTrackerAddr1 = mrCluster.getConfig().get(
+        "mapred.task.tracker.report.address");
+
+    mrCluster.restart();
+
+    String jobTrackerAddr2 = mrCluster.getConfig().get("mapred.job.tracker");
+    String taskTrackerAddr2 = mrCluster.getConfig().get(
+        "mapred.task.tracker.report.address");
+
+    assertEquals("Address before restart: " + jobTrackerAddr1
+        + " is different from new address: " + jobTrackerAddr2,
+        jobTrackerAddr1, jobTrackerAddr2);
+
+    assertEquals("Address before restart: " + taskTrackerAddr1
+        + " is different from new address: " + taskTrackerAddr2,
+        taskTrackerAddr1, taskTrackerAddr2);
+
+  }
+
   private void validateCounters(Counters counters, long mapInputRecords,
       long mapOutputRecords, long reduceInputGroups, long reduceOutputRecords) {
     assertEquals("MapInputRecords", mapInputRecords, counters.findCounter(



Mime
View raw message