hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1079250 - in /hadoop/mapreduce/branches/yahoo-merge/src: java/org/apache/hadoop/mapred/ java/org/apache/hadoop/mapreduce/server/tasktracker/ test/mapred/org/apache/hadoop/mapred/ test/mapred/org/apache/hadoop/mapreduce/lib/chain/
Date Tue, 08 Mar 2011 06:00:23 GMT
Author: omalley
Date: Tue Mar  8 06:00:22 2011
New Revision: 1079250

URL: http://svn.apache.org/viewvc?rev=1079250&view=rev
Log:
commit 4813f70f71ac02ad153d977716519c6cab0f3663
Author: Greg Roelofs <roelofs@yahoo-inc.com>
Date:   Wed Jan 19 17:12:36 2011 -0800

    Next round of uber-fixes (5 unit tests and typo-fixes only), corresponding to
    changes in v10b patch that were not already in v10.

Modified:
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskMemoryManagerThread.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
    hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJvmReuse.java
    hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java
    hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskPerformanceSplitTranscription.java
    hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
    hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java
    hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/lib/chain/TestChainErrors.java

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskMemoryManagerThread.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskMemoryManagerThread.java?rev=1079250&r1=1079249&r2=1079250&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskMemoryManagerThread.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/TaskMemoryManagerThread.java
Tue Mar  8 06:00:22 2011
@@ -54,7 +54,7 @@ class TaskMemoryManagerThread extends Th
   private final List<TaskAttemptID> tasksToBeRemoved;
 
   private static final String MEMORY_USAGE_STRING =
-    "Memory usage of ProcessTree %s for task-id %s : Virutal %d bytes, " +
+    "Memory usage of ProcessTree %s for task-id %s : Virtual %d bytes, " +
       "limit : %d bytes; Physical %d bytes, limit %d bytes";
   
   public TaskMemoryManagerThread(TaskTracker taskTracker) {
@@ -299,7 +299,7 @@ class TaskMemoryManagerThread extends Th
       if (doCheckVirtualMemory() &&
           memoryStillInUsage > maxMemoryAllowedForAllTasks) {
         LOG.warn("The total memory in usage " + memoryStillInUsage
-            + " is still overflowing TTs limits "
+            + " is still overflowing TT's limits "
             + maxMemoryAllowedForAllTasks
             + ". Trying to kill a few tasks with the least progress.");
         killTasksWithLeastProgress(memoryStillInUsage);
@@ -308,7 +308,7 @@ class TaskMemoryManagerThread extends Th
       if (doCheckPhysicalMemory() &&
           rssMemoryStillInUsage > maxRssMemoryAllowedForAllTasks) {
         LOG.warn("The total physical memory in usage " + rssMemoryStillInUsage
-            + " is still overflowing TTs limits "
+            + " is still overflowing TT's limits "
             + maxRssMemoryAllowedForAllTasks
             + ". Trying to kill a few tasks with the highest memory.");
         killTasksWithMaxRssMemory(rssMemoryStillInUsage);
@@ -440,8 +440,8 @@ class TaskMemoryManagerThread extends Th
         killTask(tid, msg);
       }
     } else {
-      LOG.info("The total memory usage is overflowing TTs limits. "
-          + "But found no alive task to kill for freeing memory.");
+      LOG.info("The total memory usage is overflowing TT's limits, "
+          + "but found no live task to kill to free memory.");
     }
   }
 
@@ -457,8 +457,8 @@ class TaskMemoryManagerThread extends Th
   }
 
   /**
-   * Starting from the tasks use the highest amount of RSS memory,
-   * kill the tasks until the RSS memory meets the requirement
+   * Starting from the tasks that use the highest amount of RSS memory,
+   * kill the tasks until the RSS memory meets the requirement.
    * @param rssMemoryInUsage
    */
   private void killTasksWithMaxRssMemory(long rssMemoryInUsage) {
@@ -500,8 +500,8 @@ class TaskMemoryManagerThread extends Th
         killTask(tid, msg);
       }
     } else {
-      LOG.info("The total physical memory usage is overflowing TTs limits. "
-          + "But found no alive task to kill for freeing memory.");
+      LOG.info("The total physical memory usage is overflowing TT's limits, "
+          + "but found no live task to kill to free memory.");
     }
   }
 

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java?rev=1079250&r1=1079249&r2=1079250&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/server/tasktracker/TTConfig.java
Tue Mar  8 06:00:22 2011
@@ -90,9 +90,10 @@ public interface TTConfig extends MRConf
     "mapreduce.tasktracker.cache.local.numberdirectories";
   public static final String TT_OUTOFBAND_HEARBEAT =
     "mapreduce.tasktracker.outofband.heartbeat";
-  public static final String TT_RESERVED_PHYSCIALMEMORY_MB =
+  public static final String TT_RESERVED_PHYSICALMEMORY_MB =
     "mapreduce.tasktracker.reserved.physicalmemory.mb";
-  public static final String TT_USER_NAME = "mapreduce.tasktracker.kerberos.principal";
+  public static final String TT_USER_NAME =
+    "mapreduce.tasktracker.kerberos.principal";
   public static final String TT_KEYTAB_FILE = 
     "mapreduce.tasktracker.keytab.file";
   public static final String TT_GROUP = 

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJvmReuse.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJvmReuse.java?rev=1079250&r1=1079249&r2=1079250&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJvmReuse.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestJvmReuse.java
Tue Mar  8 06:00:22 2011
@@ -41,6 +41,11 @@ public class TestJvmReuse {
    * A mapper class in which all attempts log taskid. Zeroth attempt of task
    * with id=taskWithCleanup, fails with System.exit to force a cleanup attempt
    * for the task in a new jvm.
+   *
+   * Note that this approach is fundamentally incompatible with uberization
+   * since sub-MapTasks have "fake"/virtual task attempt IDs that are _always_
+   * "zeroth" attempt.  (And sub-MapTasks have no visibility of UberTask's
+   * attempt ID.)
    */
   public static class MapperClass extends MapReduceBase implements
       Mapper<LongWritable, Text, Text, IntWritable> {
@@ -93,6 +98,8 @@ public class TestJvmReuse {
     FileOutputFormat.setOutputPath(conf, outDir);
     // enable jvm reuse
     conf.setNumTasksToExecutePerJvm(-1);
+    // disable uberization (see uber comment in MapperClass above)
+    conf.setBoolean(JobContext.JOB_UBERTASK_ENABLE, false);
     // return the RunningJob handle.
     return new JobClient(conf).submitJob(conf);
   }
@@ -118,7 +125,8 @@ public class TestJvmReuse {
 
   // validates logs of all attempts of the job.
   private void validateJob(RunningJob job, MiniMRCluster mr) throws IOException {
-    assertEquals(JobStatus.SUCCEEDED, job.getJobState());
+    assertEquals("JobStatus not SUCCEEDED", JobStatus.SUCCEEDED,
+        job.getJobState());
     long uses = job.getCounters().findCounter("jvm", "use").getValue();
     assertTrue("maps = " + numMappers + ", jvms = " + uses, numMappers < uses);
 

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java?rev=1079250&r1=1079249&r2=1079250&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java
Tue Mar  8 06:00:22 2011
@@ -117,6 +117,10 @@ public class TestTaskFail extends TestCa
     FileInputFormat.setInputPaths(conf, inDir);
     FileOutputFormat.setOutputPath(conf, outDir);
     conf.setSpeculativeExecution(false);
+    // mapper class explicitly depends on being able to see "real" taskids, but
+    // uberized map (sub-)taskids never change => subtask (and ubertask, and
+    // job) always fails due to being attempt "_0"
+    conf.setBoolean(JobContext.JOB_UBERTASK_ENABLE, false);
     String TEST_ROOT_DIR = new Path(System.getProperty("test.build.data",
                                     "/tmp")).toString().replace(' ', '+');
     conf.set("test.build.data", TEST_ROOT_DIR);
@@ -164,8 +168,8 @@ public class TestTaskFail extends TestCa
 
   private void validateJob(RunningJob job, JobTracker jt) 
   throws IOException {
-    assertEquals(JobStatus.SUCCEEDED, job.getJobState());
-	    
+    assertEquals("job not successful", JobStatus.SUCCEEDED, job.getJobState());
+
     JobID jobId = job.getID();
     // construct the task id of first map task
     // this should not be cleanup attempt since the first attempt 

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskPerformanceSplitTranscription.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskPerformanceSplitTranscription.java?rev=1079250&r1=1079249&r2=1079250&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskPerformanceSplitTranscription.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskPerformanceSplitTranscription.java
Tue Mar  8 06:00:22 2011
@@ -96,6 +96,9 @@ public class TestTaskPerformanceSplitTra
     
     try {
       JobConf jConf = mrCluster.createJobConf();
+      // disable uberization (MR-1220) until JobHistory supports uber-specific
+      // events with complete split info (for Rumen network topology)
+      jConf.setBoolean(JobContext.JOB_UBERTASK_ENABLE, false);
       // construct a job with 1 map and 1 reduce task.
       Job job = MapReduceTestUtil.createJob(jConf, inDir, outDir, 1, 1);
       // disable setup/cleanup

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java?rev=1079250&r1=1079249&r2=1079250&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
Tue Mar  8 06:00:22 2011
@@ -183,7 +183,7 @@ public class TestTaskTrackerMemoryManage
     fConf.setLong(MRConfig.MAPMEMORY_MB, 2 * 1024L);
     fConf.setLong(MRConfig.REDUCEMEMORY_MB, 2 * 1024L);
     // Reserve only 1 mb of the memory on TaskTrackers
-    fConf.setLong(TTConfig.TT_RESERVED_PHYSCIALMEMORY_MB, 1L);
+    fConf.setLong(TTConfig.TT_RESERVED_PHYSICALMEMORY_MB, 1L);
     startCluster(new JobConf());
 
     JobConf conf = new JobConf(miniMRCluster.createJobConf());
@@ -240,7 +240,7 @@ public class TestTaskTrackerMemoryManage
     // very small value, so that no task escapes to successful completion.
     fConf.setInt(TTConfig.TT_MEMORY_MANAGER_MONITORING_INTERVAL, 100);
     // Reserve only 1 mb of the memory on TaskTrackers
-    fConf.setLong(TTConfig.TT_RESERVED_PHYSCIALMEMORY_MB, 1L);
+    fConf.setLong(TTConfig.TT_RESERVED_PHYSICALMEMORY_MB, 1L);
     startCluster(fConf);
     runJobExceedingMemoryLimit(true);
   }
@@ -547,10 +547,11 @@ public class TestTaskTrackerMemoryManage
             new LinuxResourceCalculatorPlugin();
     long totalPhysicalMemory = memoryCalculatorPlugin.getPhysicalMemorySize();
     long reservedPhysicalMemory = totalPhysicalMemory / (1024 * 1024) + 1;
-    fConf.setLong(TTConfig.TT_RESERVED_PHYSCIALMEMORY_MB,
+    fConf.setLong(TTConfig.TT_RESERVED_PHYSICALMEMORY_MB,
                   reservedPhysicalMemory);
     long maxRssMemoryAllowedForAllTasks = totalPhysicalMemory -
                                           reservedPhysicalMemory * 1024 * 1024L;
+    // must match string near line 500 of TaskMemoryManagerThread.java:
     Pattern physicalMemoryOverLimitPattern = Pattern.compile(
         "Killing one of the memory-consuming tasks - .*"
           + ", as the cumulative RSS memory usage of all the tasks on "
@@ -571,13 +572,20 @@ public class TestTaskTrackerMemoryManage
     // Start the job
     Job job = sleepJob.createJob(1, 1, 100000, 1, 100000, 1);
     job.submit();
+    JobInProgress jip = miniMRCluster.getJobTrackerRunner().getJobTracker()
+        .getJob(JobID.downgrade(job.getJobID()));
     boolean TTOverFlowMsgPresent = false;
     while (true) {
       List<TaskReport> allTaskReports = new ArrayList<TaskReport>();
-      allTaskReports.addAll(Arrays.asList(jClient
-          .getSetupTaskReports(JobID.downgrade(job.getJobID()))));
-      allTaskReports.addAll(Arrays.asList(jClient
-          .getMapTaskReports(JobID.downgrade(job.getJobID()))));
+      if (jip.isUber()) {
+        allTaskReports.addAll(Arrays.asList(
+            jClient.getReduceTaskReports(JobID.downgrade(job.getJobID()))));
+      } else {
+        allTaskReports.addAll(Arrays.asList(
+            jClient.getSetupTaskReports(JobID.downgrade(job.getJobID()))));
+        allTaskReports.addAll(Arrays.asList(
+            jClient.getMapTaskReports(JobID.downgrade(job.getJobID()))));
+      }
       for (TaskReport tr : allTaskReports) {
         String[] diag = tr.getDiagnostics();
         for (String str : diag) {

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java?rev=1079250&r1=1079249&r2=1079250&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapred/TestWebUIAuthorization.java
Tue Mar  8 06:00:22 2011
@@ -201,23 +201,27 @@ public class TestWebUIAuthorization exte
     return job;
   }
 
-  // Waits till the map task gets started and gets its tipId from map reports
-  // and returns the tipId
+  // Waits until the map or uber task gets started; gets its tipId from task
+  // reports; and returns the tipId.
   private TaskID getTIPId(MiniMRCluster cluster,
       org.apache.hadoop.mapreduce.JobID jobid) throws Exception {
     JobClient client = new JobClient(cluster.createJobConf());
     JobID jobId = JobID.downgrade(jobid);
-    TaskReport[] mapReports = null;
+    JobInProgress jip =
+        cluster.getJobTrackerRunner().getJobTracker().getJob(jobId);
+    TaskReport[] taskReports = null;
 
     TaskID tipId = null;
     do { // make sure that the map task is running
       Thread.sleep(200);
-      mapReports = client.getMapTaskReports(jobId);
-    } while (mapReports.length == 0);
+      taskReports = jip.isUber()
+          ? client.getReduceTaskReports(jobId)
+          : client.getMapTaskReports(jobId);
+    } while (taskReports.length == 0);
 
-    for (TaskReport r : mapReports) {
+    for (TaskReport r : taskReports) {
       tipId = r.getTaskID();
-      break;// because we have only one map
+      break;// because we have only one map (or one reduce if uberized)
     }
     return tipId;
   }

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/lib/chain/TestChainErrors.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/lib/chain/TestChainErrors.java?rev=1079250&r1=1079249&r2=1079250&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/lib/chain/TestChainErrors.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/lib/chain/TestChainErrors.java
Tue Mar  8 06:00:22 2011
@@ -139,7 +139,7 @@ public class TestChainErrors extends Had
    * 
    * @throws Exception
    */
-  public void testChainMapNoOuptut() throws Exception {
+  public void testChainMapNoOutput() throws Exception {
     Configuration conf = createJobConf();
     String expectedOutput = "";
 
@@ -163,7 +163,7 @@ public class TestChainErrors extends Had
    * 
    * @throws Exception
    */
-  public void testChainReduceNoOuptut() throws Exception {
+  public void testChainReduceNoOutput() throws Exception {
     Configuration conf = createJobConf();
     String expectedOutput = "";
 



Mime
View raw message