hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r736802 - in /hadoop/core/branches/branch-0.20: ./ src/core/org/apache/hadoop/util/ src/mapred/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/mapred/ src/test/org/apache/hadoop/test/
Date Thu, 22 Jan 2009 21:31:20 GMT
Author: ddas
Date: Thu Jan 22 13:31:19 2009
New Revision: 736802

URL: http://svn.apache.org/viewvc?rev=736802&view=rev
Log:
Merge -r 736800:736801 from trunk onto 0.20 branch. Fixes HADOOP-4939.

Added:
    hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/ReliabilityTest.java
      - copied unchanged from r736801, hadoop/core/trunk/src/test/org/apache/hadoop/mapred/ReliabilityTest.java
Modified:
    hadoop/core/branches/branch-0.20/CHANGES.txt
    hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/util/Shell.java
    hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java
    hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobClient.java
    hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobSubmissionProtocol.java
    hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
    hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
    hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
    hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/test/AllTestDriver.java

Modified: hadoop/core/branches/branch-0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/CHANGES.txt?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/CHANGES.txt (original)
+++ hadoop/core/branches/branch-0.20/CHANGES.txt Thu Jan 22 13:31:19 2009
@@ -311,6 +311,9 @@
     HADOOP-4828. Updates documents to do with configuration (HADOOP-4631).
     (Sharad Agarwal via ddas)
 
+    HADOOP-4939. Adds a test that would inject random failures for tasks in 
+    large jobs and would also inject TaskTracker failures. (ddas)
+
   OPTIMIZATIONS
 
     HADOOP-3293. Fixes FileInputFormat to do provide locations for splits

Modified: hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/util/Shell.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/util/Shell.java?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/util/Shell.java (original)
+++ hadoop/core/branches/branch-0.20/src/core/org/apache/hadoop/util/Shell.java Thu Jan 22
13:31:19 2009
@@ -334,7 +334,23 @@
    * @return the output of the executed command.
    */
   public static String execCommand(String ... cmd) throws IOException {
+    return execCommand(null, cmd);
+  }
+  
+  /** 
+   * Static method to execute a shell command. 
+   * Covers most of the simple cases without requiring the user to implement  
+   * the <code>Shell</code> interface.
+   * @param env the map of environment key=value
+   * @param cmd shell command to execute.
+   * @return the output of the executed command.
+   */
+  public static String execCommand(Map<String,String> env, String ... cmd) 
+  throws IOException {
     ShellCommandExecutor exec = new ShellCommandExecutor(cmd);
+    if (env != null) {
+      exec.setEnvironment(env);
+    }
     exec.execute();
     return exec.getOutput();
   }

Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java
(original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java
Thu Jan 22 13:31:19 2009
@@ -61,6 +61,7 @@
   private Collection<String> activeTrackers = new ArrayList<String>();
   private Collection<String> blacklistedTrackers = new ArrayList<String>();
   private int numBlacklistedTrackers;
+  private long ttExpiryInterval;
   private int map_tasks;
   private int reduce_tasks;
   private int max_map_tasks;
@@ -85,7 +86,8 @@
   @Deprecated
   ClusterStatus(int trackers, int maps, int reduces, int maxMaps,
                 int maxReduces, JobTracker.State state) {
-    this(trackers, 0, maps, reduces, maxMaps, maxReduces, state);
+    this(trackers, 0, JobTracker.TASKTRACKER_EXPIRY_INTERVAL, maps, reduces,
+        maxMaps, maxReduces, state);
   }
   
   /**
@@ -93,16 +95,19 @@
    * 
    * @param trackers no. of tasktrackers in the cluster
    * @param blacklists no of blacklisted task trackers in the cluster
+   * @param ttExpiryInterval the tasktracker expiry interval
    * @param maps no. of currently running map-tasks in the cluster
    * @param reduces no. of currently running reduce-tasks in the cluster
    * @param maxMaps the maximum no. of map tasks in the cluster
    * @param maxReduces the maximum no. of reduce tasks in the cluster
    * @param state the {@link JobTracker.State} of the <code>JobTracker</code>
    */
-  ClusterStatus(int trackers, int blacklists, int maps, int reduces,
+  ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, 
+                int maps, int reduces,
                 int maxMaps, int maxReduces, JobTracker.State state) {
     numActiveTrackers = trackers;
     numBlacklistedTrackers = blacklists;
+    this.ttExpiryInterval = ttExpiryInterval;
     map_tasks = maps;
     reduce_tasks = reduces;
     max_map_tasks = maxMaps;
@@ -117,6 +122,7 @@
    * 
    * @param activeTrackers active tasktrackers in the cluster
    * @param blacklistedTrackers blacklisted tasktrackers in the cluster
+   * @param ttExpiryInterval the tasktracker expiry interval
    * @param maps no. of currently running map-tasks in the cluster
    * @param reduces no. of currently running reduce-tasks in the cluster
    * @param maxMaps the maximum no. of map tasks in the cluster
@@ -125,10 +131,11 @@
    */
   ClusterStatus(Collection<String> activeTrackers, 
       Collection<String> blacklistedTrackers,
+      long ttExpiryInterval,
       int maps, int reduces, int maxMaps, int maxReduces, 
       JobTracker.State state) {
-    this(activeTrackers.size(), blacklistedTrackers.size(), maps, reduces,
-        maxMaps, maxReduces, state);
+    this(activeTrackers.size(), blacklistedTrackers.size(), ttExpiryInterval, 
+        maps, reduces, maxMaps, maxReduces, state);
     this.activeTrackers = activeTrackers;
     this.blacklistedTrackers = blacklistedTrackers;
   }
@@ -171,6 +178,14 @@
   }
   
   /**
+   * Get the tasktracker expiry interval for the cluster
+   * @return the expiry interval in msec
+   */
+  public long getTTExpiryInterval() {
+    return ttExpiryInterval;
+  }
+  
+  /**
    * Get the number of currently running map tasks in the cluster.
    * 
    * @return the number of currently running map tasks in the cluster.
@@ -255,6 +270,7 @@
         Text.writeString(out, tracker);
       }
     }
+    out.writeLong(ttExpiryInterval);
     out.writeInt(map_tasks);
     out.writeInt(reduce_tasks);
     out.writeInt(max_map_tasks);
@@ -281,6 +297,7 @@
         blacklistedTrackers.add(name);
       }
     }
+    ttExpiryInterval = in.readLong();
     map_tasks = in.readInt();
     reduce_tasks = in.readInt();
     max_map_tasks = in.readInt();

Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobClient.java?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobClient.java (original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobClient.java Thu
Jan 22 13:31:19 2009
@@ -1193,7 +1193,20 @@
    * @throws IOException
    */
   public ClusterStatus getClusterStatus() throws IOException {
-    return jobSubmitClient.getClusterStatus(false);
+    return getClusterStatus(false);
+  }
+
+  /**
+   * Get status information about the Map-Reduce cluster.
+   *  
+   * @param  detailed if true then get a detailed status including the
+   *         tracker names
+   * @return the status information about the Map-Reduce cluster as an object
+   *         of {@link ClusterStatus}.
+   * @throws IOException
+   */
+  public ClusterStatus getClusterStatus(boolean detailed) throws IOException {
+    return jobSubmitClient.getClusterStatus(detailed);
   }
     
 

Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobSubmissionProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobSubmissionProtocol.java?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobSubmissionProtocol.java
(original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobSubmissionProtocol.java
Thu Jan 22 13:31:19 2009
@@ -55,9 +55,11 @@
    *             for HADOOP-4305
    * Version 19: Modified TaskReport to have TIP status and modified the
    *             method getClusterStatus() to take a boolean argument
-   *             for HADOOP-4807                     
+   *             for HADOOP-4807
+   * Version 20: Modified ClusterStatus to have the tasktracker expiry
+   *             interval for HADOOP-4939                     
    */
-  public static final long versionID = 19L;
+  public static final long versionID = 20L;
 
   /**
    * Allocate a name for the job.

Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/JobTracker.java Thu
Jan 22 13:31:19 2009
@@ -2711,6 +2711,7 @@
         List<List<String>> trackerNames = taskTrackerNames();
         return new ClusterStatus(trackerNames.get(0),
             trackerNames.get(1),
+            TASKTRACKER_EXPIRY_INTERVAL,
             totalMaps,
             totalReduces,
             totalMapTaskCapacity,
@@ -2720,6 +2721,7 @@
         return new ClusterStatus(taskTrackers.size() - 
             getBlacklistedTrackerCount(),
             getBlacklistedTrackerCount(),
+            TASKTRACKER_EXPIRY_INTERVAL,
             totalMaps,
             totalReduces,
             totalMapTaskCapacity,

Modified: hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
(original)
+++ hadoop/core/branches/branch-0.20/src/mapred/org/apache/hadoop/mapred/LocalJobRunner.java
Thu Jan 22 13:31:19 2009
@@ -386,7 +386,7 @@
   }
   
   public ClusterStatus getClusterStatus(boolean detailed) {
-    return new ClusterStatus(1, 0, map_tasks, reduce_tasks, 1, 1, 
+    return new ClusterStatus(1, 0, 0, map_tasks, reduce_tasks, 1, 1, 
                              JobTracker.State.RUNNING);
   }
 

Modified: hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
(original)
+++ hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/mapred/TestJobQueueTaskScheduler.java
Thu Jan 22 13:31:19 2009
@@ -135,7 +135,9 @@
     @Override
     public ClusterStatus getClusterStatus() {
       int numTrackers = trackers.size();
-      return new ClusterStatus(numTrackers, 0, maps, reduces,
+      return new ClusterStatus(numTrackers, 0, 
+                               JobTracker.TASKTRACKER_EXPIRY_INTERVAL,
+                               maps, reduces,
                                numTrackers * maxMapTasksPerTracker,
                                numTrackers * maxReduceTasksPerTracker,
                                JobTracker.State.RUNNING);

Modified: hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/test/AllTestDriver.java
URL: http://svn.apache.org/viewvc/hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/test/AllTestDriver.java?rev=736802&r1=736801&r2=736802&view=diff
==============================================================================
--- hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/test/AllTestDriver.java (original)
+++ hadoop/core/branches/branch-0.20/src/test/org/apache/hadoop/test/AllTestDriver.java Thu
Jan 22 13:31:19 2009
@@ -22,6 +22,7 @@
 import org.apache.hadoop.mapred.BigMapOutput;
 import org.apache.hadoop.mapred.GenericMRLoadGenerator;
 import org.apache.hadoop.mapred.MRBench;
+import org.apache.hadoop.mapred.ReliabilityTest;
 import org.apache.hadoop.mapred.SortValidator;
 import org.apache.hadoop.mapred.TestMapRed;
 import org.apache.hadoop.mapred.TestSequenceFileInputFormat;
@@ -74,6 +75,9 @@
       pgd.addClass("filebench", FileBench.class, "Benchmark SequenceFile(Input|Output)Format
(block,record compressed and uncompressed), Text(Input|Output)Format (compressed and uncompressed)");
       pgd.addClass("dfsthroughput", BenchmarkThroughput.class, 
                    "measure hdfs throughput");
+      pgd.addClass("MRReliabilityTest", ReliabilityTest.class,
+          "A program that tests the reliability of the MR framework by " +
+          "injecting faults/failures");
       pgd.driver(argv);
     } catch(Throwable e) {
       e.printStackTrace();



Mime
View raw message