hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject svn commit: r808036 [4/4] - in /hadoop/mapreduce/branches/MAPREDUCE-233: ./ .eclipse.templates/ conf/ ivy/ lib/ src/c++/ src/contrib/ src/contrib/capacity-scheduler/ src/contrib/data_join/ src/contrib/dynamic-scheduler/ src/contrib/eclipse-plugin/ src/...
Date Wed, 26 Aug 2009 15:01:34 GMT
Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java?rev=808036&r1=808035&r2=808036&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestNodeRefresh.java
Wed Aug 26 15:01:29 2009
@@ -100,6 +100,7 @@
                    hostsSeen.size());
     } catch (IOException ioe) {
       stopCluster();
+      throw ioe;
     }
   }
 

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java?rev=808036&r1=808035&r2=808036&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestRecoveryManager.java
Wed Aug 26 15:01:29 2009
@@ -79,10 +79,8 @@
     RunningJob rJob1 = (new JobClient(job1)).submitJob(job1);
     LOG.info("Submitted job " + rJob1.getID());
     
-    while (rJob1.mapProgress() < 0.5f) {
-      LOG.info("Waiting for job " + rJob1.getID() + " to be 50% done");
-      UtilsForTests.waitFor(100);
-    }
+    // wait for 50%
+    UtilsForTests.waitForJobHalfDone(rJob1);
     
     JobConf job2 = mr.createJobConf();
     
@@ -94,10 +92,8 @@
     RunningJob rJob2 = (new JobClient(job2)).submitJob(job2);
     LOG.info("Submitted job " + rJob2.getID());
     
-    while (rJob2.mapProgress() < 0.5f) {
-      LOG.info("Waiting for job " + rJob2.getID() + " to be 50% done");
-      UtilsForTests.waitFor(100);
-    }
+    // wait for 50%
+    UtilsForTests.waitForJobHalfDone(rJob2);
     
     // kill the jobtracker
     LOG.info("Stopping jobtracker");

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerBlacklisting.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerBlacklisting.java?rev=808036&r1=808035&r2=808036&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerBlacklisting.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerBlacklisting.java
Wed Aug 26 15:01:29 2009
@@ -26,6 +26,8 @@
 import java.util.Set;
 import java.util.Map.Entry;
 
+import javax.security.auth.login.LoginException;
+
 import junit.extensions.TestSetup;
 import junit.framework.Test;
 import junit.framework.TestCase;
@@ -85,7 +87,7 @@
       org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobTracker {
   
     FakeJobTracker(JobConf conf, Clock clock, String[] tts) throws IOException,
-        InterruptedException {
+        InterruptedException, LoginException {
       super(conf, clock, tts);
     }
 

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java?rev=808036&r1=808035&r2=808036&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java
Wed Aug 26 15:01:29 2009
@@ -196,16 +196,8 @@
       return;
     }
 
-    long PER_TASK_LIMIT = 1L; // Low enough to kill off sleepJob tasks.
-
-    Pattern taskOverLimitPattern =
-        Pattern.compile(String.format(taskOverLimitPatternString, String
-            .valueOf(PER_TASK_LIMIT*1024*1024L)));
-    Matcher mat = null;
-
     // Start cluster with proper configuration.
     JobConf fConf = new JobConf();
-
     // very small value, so that no task escapes to successful completion.
     fConf.set("mapred.tasktracker.taskmemorymanager.monitoring-interval",
         String.valueOf(300));
@@ -215,6 +207,51 @@
         JobTracker.MAPRED_CLUSTER_REDUCE_MEMORY_MB_PROPERTY,
         2 * 1024);
     startCluster(fConf);
+    runJobExceedingMemoryLimit();
+  }
+  
+  /**
+   * Runs tests with tasks beyond limit and using old configuration values for
+   * the TaskTracker.
+   * 
+   * @throws Exception
+   */
+
+  public void testTaskMemoryMonitoringWithDeprecatedConfiguration () 
+    throws Exception {
+    
+    // Run the test only if memory management is enabled
+    if (!isProcfsBasedTreeAvailable()) {
+      return;
+    }
+    // Start cluster with proper configuration.
+    JobConf fConf = new JobConf();
+    // very small value, so that no task escapes to successful completion.
+    fConf.set("mapred.tasktracker.taskmemorymanager.monitoring-interval",
+        String.valueOf(300));
+    //set old values, max vm property per task and upper limit on the tasks
+    //vm
+    //setting the default maximum vmem property to 2 GB
+    fConf.setLong(JobConf.MAPRED_TASK_DEFAULT_MAXVMEM_PROPERTY,
+        (2L * 1024L * 1024L * 1024L));
+    fConf.setLong(JobConf.UPPER_LIMIT_ON_TASK_VMEM_PROPERTY, 
+        (3L * 1024L * 1024L * 1024L));
+    startCluster(fConf);
+    runJobExceedingMemoryLimit();
+  }
+
+  /**
+   * Runs a job which should fail the when run by the memory monitor.
+   * 
+   * @throws IOException
+   */
+  private void runJobExceedingMemoryLimit() throws IOException {
+    long PER_TASK_LIMIT = 1L; // Low enough to kill off sleepJob tasks.
+
+    Pattern taskOverLimitPattern =
+        Pattern.compile(String.format(taskOverLimitPatternString, String
+            .valueOf(PER_TASK_LIMIT*1024*1024L)));
+    Matcher mat = null;
 
     // Set up job.
     JobConf conf = new JobConf(miniMRCluster.createJobConf());

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTrackerReservation.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTrackerReservation.java?rev=808036&r1=808035&r2=808036&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTrackerReservation.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/TestTrackerReservation.java
Wed Aug 26 15:01:29 2009
@@ -20,6 +20,8 @@
 import java.io.IOException;
 import java.util.ArrayList;
 
+import javax.security.auth.login.LoginException;
+
 import org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobInProgress;
 import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
@@ -39,7 +41,7 @@
       org.apache.hadoop.mapred.FakeObjectUtilities.FakeJobTracker {
 
     FakeJobTracker(JobConf conf, Clock clock, String[] tts) throws IOException,
-        InterruptedException {
+        InterruptedException, LoginException {
       super(conf, clock, tts);
     }
 

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java?rev=808036&r1=808035&r2=808036&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java
Wed Aug 26 15:01:29 2009
@@ -242,6 +242,7 @@
       if(System.currentTimeMillis() > timeout) {
         throw new IOException("Timeout waiting for job to get to 50% done");
       }
+      LOG.info("Waiting for job " + job.getID() + " to be 50% done");
       UtilsForTests.waitFor(100);
     }
   }

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/MapReduceTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/MapReduceTestUtil.java?rev=808036&r1=808035&r2=808036&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/MapReduceTestUtil.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/MapReduceTestUtil.java
Wed Aug 26 15:01:29 2009
@@ -18,22 +18,28 @@
 
 package org.apache.hadoop.mapreduce;
 
+import java.io.DataInput;
+import java.io.DataOutput;
 import java.io.BufferedReader;
 import java.io.DataOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.text.NumberFormat;
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Random;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.NullWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableComparable;
@@ -43,6 +49,7 @@
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.util.ReflectionUtils;
 
 /**
  * Utility methods used in various Job Control unit tests.
@@ -237,6 +244,99 @@
     }
   }
 
+  public static class IncomparableKey implements WritableComparable<Object> {
+    public void write(DataOutput out) { }
+    public void readFields(DataInput in) { }
+    public int compareTo(Object o) {
+      throw new RuntimeException("Should never see this.");
+    }
+  }
+
+  public static class FakeSplit extends InputSplit implements Writable {
+    public void write(DataOutput out) throws IOException { }
+    public void readFields(DataInput in) throws IOException { }
+    public long getLength() { return 0L; }
+    public String[] getLocations() { return new String[0]; }
+  }
+
+  public static class Fake_IF<K,V>
+    extends InputFormat<K, V> 
+    implements Configurable {
+
+    public Fake_IF() { }
+
+    public List<InputSplit> getSplits(JobContext context) {
+      List<InputSplit> ret = new ArrayList<InputSplit>(); 
+      ret.add(new FakeSplit());
+      return ret;
+    }
+    public static void setKeyClass(Configuration conf, Class<?> k) {
+      conf.setClass("test.fakeif.keyclass", k, WritableComparable.class);
+    }
+
+    public static void setValClass(Configuration job, Class<?> v) {
+      job.setClass("test.fakeif.valclass", v, Writable.class);
+    }
+
+    protected Class<? extends K> keyclass;
+    protected Class<? extends V> valclass;
+    Configuration conf = null;
+
+    @SuppressWarnings("unchecked")
+    public void setConf(Configuration conf) {
+      this.conf = conf;
+      keyclass = (Class<? extends K>) conf.getClass("test.fakeif.keyclass",
+          NullWritable.class, WritableComparable.class);
+      valclass = (Class<? extends V>) conf.getClass("test.fakeif.valclass",
+          NullWritable.class, WritableComparable.class);
+    }
+
+    public Configuration getConf() {
+      return conf;
+    }
+    
+    public RecordReader<K,V> createRecordReader(
+        InputSplit ignored, TaskAttemptContext context) {
+      return new RecordReader<K,V>() {
+        public boolean nextKeyValue() throws IOException { return false; }
+        public void initialize(InputSplit split, TaskAttemptContext context) 
+            throws IOException, InterruptedException {}
+        public K getCurrentKey() {
+        return null;
+        }
+        public V getCurrentValue() {
+          return null;
+        }
+        public void close() throws IOException { }
+        public float getProgress() throws IOException { return 0.0f; }
+      };
+    }
+  }
+  
+  public static class Fake_RR<K, V> extends RecordReader<K,V> {
+    private Class<? extends K> keyclass;
+    private Class<? extends V> valclass;
+    public boolean nextKeyValue() throws IOException { return false; }
+    @SuppressWarnings("unchecked")
+    public void initialize(InputSplit split, TaskAttemptContext context) 
+        throws IOException, InterruptedException {
+      Configuration conf = context.getConfiguration();
+      keyclass = (Class<? extends K>) conf.getClass("test.fakeif.keyclass",
+        NullWritable.class, WritableComparable.class);
+      valclass = (Class<? extends V>) conf.getClass("test.fakeif.valclass",
+        NullWritable.class, WritableComparable.class);
+      
+    }
+    public K getCurrentKey() {
+      return ReflectionUtils.newInstance(keyclass, null);
+    }
+    public V getCurrentValue() {
+      return ReflectionUtils.newInstance(valclass, null);
+    }
+    public void close() throws IOException { }
+    public float getProgress() throws IOException { return 0.0f; }
+  }
+
   public static Job createJob(Configuration conf, Path inDir, Path outDir, 
       int numInputFiles, int numReds) throws IOException {
     String input = "The quick brown fox\n" + "has many silly\n"

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/TestChild.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/TestChild.java?rev=808036&r1=808035&r2=808036&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/TestChild.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/TestChild.java
Wed Aug 26 15:01:29 2009
@@ -29,6 +29,7 @@
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.HadoopTestCase;
 import org.apache.hadoop.mapred.JobConf;
+import org.apache.log4j.Level;
 
 public class TestChild extends HadoopTestCase {
   private static String TEST_ROOT_DIR =
@@ -68,6 +69,12 @@
                      mapJavaOpts, 
                      mapJavaOpts, MAP_OPTS_VAL);
       }
+      
+      Level logLevel = 
+        Level.toLevel(conf.get(JobConf.MAPRED_MAP_TASK_LOG_LEVEL, 
+                               Level.INFO.toString()));  
+      assertEquals(JobConf.MAPRED_MAP_TASK_LOG_LEVEL + "has value of " + 
+                   logLevel, logLevel, Level.OFF);
     }
   }
   
@@ -94,6 +101,12 @@
                      reduceJavaOpts, 
                      reduceJavaOpts, REDUCE_OPTS_VAL);
       }
+      
+      Level logLevel = 
+        Level.toLevel(conf.get(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL, 
+                               Level.INFO.toString()));  
+      assertEquals(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL + "has value of " + 
+                   logLevel, logLevel, Level.OFF);
     }
   }
   
@@ -108,6 +121,9 @@
       conf.set(JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS, REDUCE_OPTS_VAL);
     }
     
+    conf.set(JobConf.MAPRED_MAP_TASK_LOG_LEVEL, Level.OFF.toString());
+    conf.set(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL, Level.OFF.toString());
+    
     Job job = MapReduceTestUtil.createJob(conf, inDir, outDir, 
                 numMaps, numReds);
     job.setMapperClass(MyMapper.class);

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java?rev=808036&r1=808035&r2=808036&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapreduce/TestMapReduceLocal.java
Wed Aug 26 15:01:29 2009
@@ -161,6 +161,9 @@
     assertEquals("map out = combine in", mapOut, combineIn);
     assertEquals("combine out = reduce in", combineOut, reduceIn);
     assertTrue("combine in > combine out", combineIn > combineOut);
+    String group = "Random Group";
+    CounterGroup ctrGrp = ctrs.getGroup(group);
+    assertEquals(0, ctrGrp.size());
   }
 
   public void runMultiFileWordCount(Configuration  conf) throws Exception  {

Propchange: hadoop/mapreduce/branches/MAPREDUCE-233/src/webapps/job/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Aug 26 15:01:29 2009
@@ -1,3 +1,3 @@
 /hadoop/core/branches/branch-0.19/mapred/src/webapps/job:713112
 /hadoop/core/trunk/src/webapps/job:776175-785643
-/hadoop/mapreduce/trunk/src/webapps/job:804974-805826
+/hadoop/mapreduce/trunk/src/webapps/job:804974-807678



Mime
View raw message