hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r889085 - in /hadoop/mapreduce/trunk: ./ .eclipse.templates/ ivy/ src/test/unit/ src/test/unit/org/ src/test/unit/org/apache/ src/test/unit/org/apache/hadoop/ src/test/unit/org/apache/hadoop/mapred/ src/test/unit/org/apache/hadoop/mapreduce...
Date Thu, 10 Dec 2009 04:39:21 GMT
Author: tomwhite
Date: Thu Dec 10 04:39:20 2009
New Revision: 889085

URL: http://svn.apache.org/viewvc?rev=889085&view=rev
Log:
MAPREDUCE-1050. Introduce a mock object testing framework.

Added:
    hadoop/mapreduce/trunk/src/test/unit/
    hadoop/mapreduce/trunk/src/test/unit/org/
    hadoop/mapreduce/trunk/src/test/unit/org/apache/
    hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/
    hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapred/
    hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapred/TestLostTaskTracker.java
    hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapreduce/
    hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapreduce/lib/
    hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapreduce/lib/jobcontrol/
    hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapreduce/lib/jobcontrol/TestControlledJob.java
    hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControlWithMocks.java
Modified:
    hadoop/mapreduce/trunk/.eclipse.templates/.classpath
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/build.xml
    hadoop/mapreduce/trunk/ivy.xml
    hadoop/mapreduce/trunk/ivy/libraries.properties

Modified: hadoop/mapreduce/trunk/.eclipse.templates/.classpath
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/.eclipse.templates/.classpath?rev=889085&r1=889084&r2=889085&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/.eclipse.templates/.classpath (original)
+++ hadoop/mapreduce/trunk/.eclipse.templates/.classpath Thu Dec 10 04:39:20 2009
@@ -2,6 +2,7 @@
 <classpath>
 	<classpathentry kind="src" path="src/java"/>
 	<classpathentry kind="src" path="src/test/mapred"/>
+	<classpathentry kind="src" path="src/test/unit"/>
 	<classpathentry kind="src" path="src/benchmarks/gridmix2/src/java"/>
 	<classpathentry kind="src" path="src/examples"/>
 	<classpathentry kind="src" path="src/tools"/>

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=889085&r1=889084&r2=889085&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Thu Dec 10 04:39:20 2009
@@ -66,6 +66,8 @@
     MAPREDUCE-1185. Redirect running job url to history url if job is already 
     retired. (Amareshwari Sriramadasu and Sharad Agarwal via sharad)
 
+    MAPREDUCE-1050. Introduce a mock object testing framework. (tomwhite)
+
   OPTIMIZATIONS
 
     MAPREDUCE-270. Fix the tasktracker to optionally send an out-of-band

Modified: hadoop/mapreduce/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/build.xml?rev=889085&r1=889084&r2=889085&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/build.xml (original)
+++ hadoop/mapreduce/trunk/build.xml Thu Dec 10 04:39:20 2009
@@ -476,7 +476,7 @@
 
     <javac 
       encoding="${build.encoding}" 
-      srcdir="${test.src.dir}/mapred"
+      srcdir="${test.src.dir}/mapred;${test.src.dir}/unit"
       includes="org/apache/hadoop/**/*.java"
       destdir="${test.mapred.build.classes}"
       debug="${javac.debug}"
@@ -565,6 +565,7 @@
 
   <macrodef name="macro-test-runner">
     <attribute name="test.file" />
+    <attribute name="suite.type" />
     <sequential>
       <delete dir="${test.build.data}"/>
       <mkdir dir="${test.build.data}"/>
@@ -603,7 +604,7 @@
         <classpath refid="test.classpath"/>
         <formatter type="${test.junit.output.format}" />
         <batchtest todir="${test.build.dir}" unless="testcase">
-          <fileset dir="${test.src.dir}/mapred" excludes="**/${test.exclude}.java">
+          <fileset dir="${test.src.dir}/@{suite.type}" excludes="**/${test.exclude}.java">
             <patternset>
               <includesfile name="@{test.file}"/>
             </patternset>
@@ -611,20 +612,24 @@
         </batchtest>
         <batchtest todir="${test.build.dir}" if="testcase">
           <fileset dir="${test.src.dir}/mapred" includes="**/${testcase}.java"/>
+          <fileset dir="${test.src.dir}/unit" includes="**/${testcase}.java"/>
         </batchtest>
       </junit>
       <antcall target="checkfailure"/>
     </sequential>
   </macrodef>
 
-  <target name="run-test-mapred" depends="compile-mapred-test" description="Run mapred
unit tests">
-    <macro-test-runner test.file="${test.mapred.all.tests.file}" />
+  <target name="run-test-mapred" depends="compile-mapred-test" description="Run mapred
functional and system tests">
+    <macro-test-runner test.file="${test.mapred.all.tests.file}" suite.type="mapred"/>
   </target> 
 
    <target name="run-commit-test" depends="compile-mapred-test" description="Run approximate
10-minute set of unit tests prior to commiting">
-     <macro-test-runner test.file="${test.mapred.commit.tests.file}" />
+     <macro-test-runner test.file="${test.mapred.commit.tests.file}" suite.type="mapred"/>
    </target>
  
+   <target name="run-test-unit" depends="compile-mapred-test" description="Run unit tests">
+     <macro-test-runner test.file="${test.mapred.all.tests.file}" suite.type="unit"/>
+   </target>
 
 
   <target name="checkfailure" if="tests.failed">

Modified: hadoop/mapreduce/trunk/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/ivy.xml?rev=889085&r1=889084&r2=889085&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/ivy.xml (original)
+++ hadoop/mapreduce/trunk/ivy.xml Thu Dec 10 04:39:20 2009
@@ -94,6 +94,9 @@
                rev="${lucene-core.version}" conf="javadoc->default"/>
    <dependency org="org.apache.hadoop" name="avro" rev="${avro.version}" 
                conf="common->default"/>
+   <dependency org="org.mockito" name="mockito-all" rev="${mockito-all.version}" 
+               conf="test->default"/>
+
  </dependencies>
   
 </ivy-module>

Modified: hadoop/mapreduce/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/ivy/libraries.properties?rev=889085&r1=889084&r2=889085&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/ivy/libraries.properties (original)
+++ hadoop/mapreduce/trunk/ivy/libraries.properties Thu Dec 10 04:39:20 2009
@@ -65,6 +65,8 @@
 
 mina-core.version=2.0.0-M5
 
+mockito-all.version=1.8.0
+
 oro.version=2.0.8
 
 paranamer.version=1.5

Added: hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapred/TestLostTaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapred/TestLostTaskTracker.java?rev=889085&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapred/TestLostTaskTracker.java
(added)
+++ hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapred/TestLostTaskTracker.java
Thu Dec 10 04:39:20 2009
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import static org.mockito.Mockito.*;
+
+import java.io.IOException;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.mapred.UtilsForTests.FakeClock;
+import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
+import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker;
+import org.hamcrest.Matcher;
+import org.mockito.ArgumentCaptor;
+import org.mockito.ArgumentMatcher;
+
+/**
+ * Tests that trackers that don't heartbeat within a given time are considered
+ * lost. Note that this test is not a direct replacement for
+ * {@link TestLostTracker} since it doesn't test that a task
+ * running on a lost tracker is retried on another tracker.
+ */
+@SuppressWarnings("deprecation")
+public class TestLostTaskTracker extends TestCase {
+
+  private JobTracker jobTracker;
+ 
+  private FakeClock clock;
+  
+  @Override
+  protected void setUp() throws Exception {
+    JobConf conf = new JobConf();
+    conf.set(JTConfig.JT_IPC_ADDRESS, "localhost:0");
+    conf.set(JTConfig.JT_HTTP_ADDRESS, "0.0.0.0:0");
+    conf.setLong(JTConfig.JT_TRACKER_EXPIRY_INTERVAL, 1000);
+    clock = new FakeClock();
+    // We use a "partial mock" of JobTracker which lets us see when certain
+    // methods are called. If we were writing JobTracker from scratch then
+    // we would make it call another object which we would mock out instead
+    // (and use a real JobTracker) so we could perform assertions on the mock.
+    // See http://mockito.googlecode.com/svn/branches/1.8.0/javadoc/org/mockito/Mockito.html#16
+    jobTracker = spy(new JobTracker(conf, clock));
+  }
+  
+  public void testLostTaskTrackerCalledAfterExpiryTime() throws IOException {
+    
+    String tracker1 = "tracker_tracker1:1000";
+    String tracker2 = "tracker_tracker2:1000";
+    
+    establishFirstContact(tracker1);
+    
+    // Wait long enough for tracker1 to be considered lost
+    // We could have used a Mockito stub here, except we don't know how many 
+    // times JobTracker calls getTime() on the clock, so a static mock
+    // is appropriate.
+    clock.advance(8 * 1000);
+    
+    establishFirstContact(tracker2);
+
+    jobTracker.checkExpiredTrackers();
+    
+    // Now we check that JobTracker's lostTaskTracker() was called for tracker1
+    // but not for tracker2.
+    
+    // We use an ArgumentCaptor to capture the task tracker object
+    // in the lostTaskTracker() call, so we can perform an assertion on its
+    // name. (We could also have used a custom matcher, see below.)
+    // See http://mockito.googlecode.com/svn/branches/1.8.0/javadoc/org/mockito/Mockito.html#15
+    ArgumentCaptor<TaskTracker> argument =
+      ArgumentCaptor.forClass(TaskTracker.class);
+
+    verify(jobTracker).lostTaskTracker(argument.capture());
+    assertEquals(tracker1, argument.getValue().getTrackerName());
+    
+    // Check tracker2 was not lost by using the never() construct
+    // We use a custom Hamcrest matcher to check that it was indeed tracker2
+    // that didn't match (since tracker1 did match).
+    // See http://mockito.googlecode.com/svn/branches/1.8.0/javadoc/org/mockito/Mockito.html#3
+    verify(jobTracker, never()).lostTaskTracker(
+	argThat(taskTrackerWithName(tracker2)));
+  }
+  
+  private Matcher<TaskTracker> taskTrackerWithName(final String name) {
+    return new ArgumentMatcher<TaskTracker>() {
+      public boolean matches(Object taskTracker) {
+          return name.equals(((TaskTracker) taskTracker).getTrackerName());
+      }
+    };
+  }
+
+  private void establishFirstContact(String tracker) throws IOException {
+    TaskTrackerStatus status = new TaskTrackerStatus(tracker, 
+        JobInProgress.convertTrackerNameToHostName(tracker));
+    jobTracker.heartbeat(status, false, true, false, (short) 0);
+  }
+  
+}
\ No newline at end of file

Added: hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapreduce/lib/jobcontrol/TestControlledJob.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapreduce/lib/jobcontrol/TestControlledJob.java?rev=889085&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapreduce/lib/jobcontrol/TestControlledJob.java
(added)
+++ hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapreduce/lib/jobcontrol/TestControlledJob.java
Thu Dec 10 04:39:20 2009
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.jobcontrol;
+
+import static org.junit.Assert.assertFalse;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+/**
+ */
+public class TestControlledJob {
+  
+  @Test
+  public void testAddingDependingJobToRunningJobFails() throws Exception {
+    Configuration conf = new Configuration();
+    ControlledJob job1 = new ControlledJob(conf);
+    job1.setJobState(ControlledJob.State.RUNNING);
+    assertFalse(job1.addDependingJob(new ControlledJob(conf)));
+  }
+
+  @Test
+  public void testAddingDependingJobToCompletedJobFails() throws Exception {
+    Configuration conf = new Configuration();
+    ControlledJob job1 = new ControlledJob(conf);
+    job1.setJobState(ControlledJob.State.SUCCESS);
+    assertFalse(job1.addDependingJob(new ControlledJob(conf)));
+  }
+
+}

Added: hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControlWithMocks.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControlWithMocks.java?rev=889085&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControlWithMocks.java
(added)
+++ hadoop/mapreduce/trunk/src/test/unit/org/apache/hadoop/mapreduce/lib/jobcontrol/TestMapReduceJobControlWithMocks.java
Thu Dec 10 04:39:20 2009
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.jobcontrol;
+
+import static junit.framework.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Job;
+import org.junit.Test;
+
+/**
+ * Tests the JobControl API using mock and stub Job instances.
+ */
+public class TestMapReduceJobControlWithMocks {
+
+  @Test
+  public void testSuccessfulJobs() throws Exception {
+    JobControl jobControl = new JobControl("Test");
+    
+    ControlledJob job1 = createSuccessfulControlledJob(jobControl);
+    ControlledJob job2 = createSuccessfulControlledJob(jobControl);
+    ControlledJob job3 = createSuccessfulControlledJob(jobControl, job1, job2);
+    ControlledJob job4 = createSuccessfulControlledJob(jobControl, job3);
+    
+    runJobControl(jobControl);
+    
+    assertEquals("Success list", 4, jobControl.getSuccessfulJobList().size());
+    assertEquals("Failed list", 0, jobControl.getFailedJobList().size());
+    
+    assertTrue(job1.getJobState() == ControlledJob.State.SUCCESS);
+    assertTrue(job2.getJobState() == ControlledJob.State.SUCCESS);
+    assertTrue(job3.getJobState() == ControlledJob.State.SUCCESS);
+    assertTrue(job4.getJobState() == ControlledJob.State.SUCCESS);
+    
+    jobControl.stop();
+  }
+  
+  @Test
+  public void testFailedJob() throws Exception {
+    JobControl jobControl = new JobControl("Test");
+    
+    ControlledJob job1 = createFailedControlledJob(jobControl);
+    ControlledJob job2 = createSuccessfulControlledJob(jobControl);
+    ControlledJob job3 = createSuccessfulControlledJob(jobControl, job1, job2);
+    ControlledJob job4 = createSuccessfulControlledJob(jobControl, job3);
+    
+    runJobControl(jobControl);
+    
+    assertEquals("Success list", 1, jobControl.getSuccessfulJobList().size());
+    assertEquals("Failed list", 3, jobControl.getFailedJobList().size());
+
+    assertTrue(job1.getJobState() == ControlledJob.State.FAILED);
+    assertTrue(job2.getJobState() == ControlledJob.State.SUCCESS);
+    assertTrue(job3.getJobState() == ControlledJob.State.DEPENDENT_FAILED);
+    assertTrue(job4.getJobState() == ControlledJob.State.DEPENDENT_FAILED);
+    
+    jobControl.stop();
+  }
+  
+  @Test
+  public void testKillJob() throws Exception {
+    JobControl jobControl = new JobControl("Test");
+    
+    ControlledJob job = createFailedControlledJob(jobControl);
+    
+    job.killJob();
+
+    // Verify that killJob() was called on the mock Job
+    verify(job.getJob()).killJob();
+  }
+  
+  private Job createJob(boolean complete, boolean successful)
+  	throws IOException, InterruptedException {
+    // Create a stub Job that responds in a controlled way
+    Job mockJob = mock(Job.class);
+    when(mockJob.getConfiguration()).thenReturn(new Configuration());
+    when(mockJob.isComplete()).thenReturn(complete);
+    when(mockJob.isSuccessful()).thenReturn(successful);
+    return mockJob;
+  }
+  
+  private ControlledJob createControlledJob(JobControl jobControl,
+      	boolean successful, ControlledJob... dependingJobs)
+      	throws IOException, InterruptedException {
+    List<ControlledJob> dependingJobsList = dependingJobs == null ? null :
+      Arrays.asList(dependingJobs);
+    ControlledJob job = new ControlledJob(createJob(true, successful),
+	dependingJobsList);
+    jobControl.addJob(job);
+    return job;
+  }
+  
+  private ControlledJob createSuccessfulControlledJob(JobControl jobControl,
+      ControlledJob... dependingJobs) throws IOException, InterruptedException {
+    return createControlledJob(jobControl, true, dependingJobs);
+  }
+
+  private ControlledJob createFailedControlledJob(JobControl jobControl,
+      ControlledJob... dependingJobs) throws IOException, InterruptedException {
+    return createControlledJob(jobControl, false, dependingJobs);
+  }
+
+  private void runJobControl(JobControl jobControl) {
+    Thread controller = new Thread(jobControl);
+    controller.start();
+    waitTillAllFinished(jobControl);
+  }
+
+  private void waitTillAllFinished(JobControl jobControl) {
+    while (!jobControl.allFinished()) {
+      try {
+        Thread.sleep(100);
+      } catch (InterruptedException e) {
+	// ignore
+      }
+    }
+  }
+}



Mime
View raw message