hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From acmur...@apache.org
Subject svn commit: r987636 - in /hadoop/mapreduce/trunk/src: java/org/apache/hadoop/mapred/ test/mapred/org/apache/hadoop/mapred/
Date Fri, 20 Aug 2010 21:15:00 GMT
Author: acmurthy
Date: Fri Aug 20 21:15:00 2010
New Revision: 987636

URL: http://svn.apache.org/viewvc?rev=987636&view=rev
Log:
MAPREDUCE-1881. Adding the missed files.

Added:
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/CompositeTaskTrackerInstrumentation.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/DummyTaskTrackerInstrumentation.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCompositeTaskTrackerInstrumentation.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerInstrumentation.java

Added: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/CompositeTaskTrackerInstrumentation.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/CompositeTaskTrackerInstrumentation.java?rev=987636&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/CompositeTaskTrackerInstrumentation.java
(added)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/CompositeTaskTrackerInstrumentation.java
Fri Aug 20 21:15:00 2010
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.File;
+import java.util.List;
+
+/**
+ * This TaskTrackerInstrumentation subclass forwards all the events it
+ * receives to a list of instrumentation objects, and can thus be used to
+ * attack multiple instrumentation objects to a TaskTracker.
+ */
+class CompositeTaskTrackerInstrumentation extends TaskTrackerInstrumentation {
+  
+  private List<TaskTrackerInstrumentation> instrumentations;
+
+  public CompositeTaskTrackerInstrumentation(TaskTracker tt,
+      List<TaskTrackerInstrumentation> instrumentations) {
+    super(tt);
+    this.instrumentations = instrumentations;
+  }
+
+  // Package-private getter methods for tests
+  List<TaskTrackerInstrumentation> getInstrumentations() {
+    return instrumentations;
+  }
+  
+  @Override
+  public void completeTask(TaskAttemptID t) {
+    for (TaskTrackerInstrumentation tti: instrumentations) {
+      tti.completeTask(t);
+    }
+  }
+  
+  @Override
+  public void timedoutTask(TaskAttemptID t) {
+    for (TaskTrackerInstrumentation tti: instrumentations) {
+      tti.timedoutTask(t);
+    }
+  }
+  
+  @Override
+  public void taskFailedPing(TaskAttemptID t) {
+    for (TaskTrackerInstrumentation tti: instrumentations) {
+      tti.taskFailedPing(t);
+    }
+  }
+
+  @Override
+  public void reportTaskLaunch(TaskAttemptID t, File stdout, File stderr) {
+    for (TaskTrackerInstrumentation tti: instrumentations) {
+      tti.reportTaskLaunch(t, stdout, stderr);
+    }
+  }
+  
+  @Override
+  public void reportTaskEnd(TaskAttemptID t) {
+    for (TaskTrackerInstrumentation tti: instrumentations) {
+      tti.reportTaskEnd(t);
+    }
+  }
+   
+  @Override
+  public void statusUpdate(Task task, TaskStatus taskStatus) {
+    for (TaskTrackerInstrumentation tti: instrumentations) {
+      tti.statusUpdate(task, taskStatus);
+    }
+  }
+}

Added: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/DummyTaskTrackerInstrumentation.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/DummyTaskTrackerInstrumentation.java?rev=987636&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/DummyTaskTrackerInstrumentation.java
(added)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/DummyTaskTrackerInstrumentation.java
Fri Aug 20 21:15:00 2010
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.File;
+
+/**
+ * Mock instrumentation class used in TaskTrackerInstrumentation tests.
+ * This class just records whether each instrumentation method was called.
+ */
+public class DummyTaskTrackerInstrumentation
+  extends TaskTrackerInstrumentation
+{
+  boolean completeTaskCalled = false;
+  boolean timedoutTaskCalled = false;
+  boolean taskFailedPingCalled = false;
+  boolean reportTaskLaunchCalled = false;
+  boolean reportTaskEndCalled = false;
+  boolean statusUpdateCalled = false;
+
+  public DummyTaskTrackerInstrumentation(TaskTracker tt) {
+    super(tt);
+  }
+
+  @Override
+  public void completeTask(TaskAttemptID t) {
+    completeTaskCalled = true;
+  }
+
+  @Override
+  public void timedoutTask(TaskAttemptID t) {
+    timedoutTaskCalled = true;
+  }
+
+  @Override
+  public void taskFailedPing(TaskAttemptID t) {
+    taskFailedPingCalled = true;
+  }
+
+  @Override
+  public void reportTaskLaunch(TaskAttemptID t, File stdout, File stderr) {
+    reportTaskLaunchCalled = true;
+  }
+
+  @Override
+  public void reportTaskEnd(TaskAttemptID t) {
+    reportTaskEndCalled = true;
+  }
+
+  @Override
+  public void statusUpdate(Task t, TaskStatus s) {
+    statusUpdateCalled = true;
+  }
+}

Added: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCompositeTaskTrackerInstrumentation.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCompositeTaskTrackerInstrumentation.java?rev=987636&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCompositeTaskTrackerInstrumentation.java
(added)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestCompositeTaskTrackerInstrumentation.java
Fri Aug 20 21:15:00 2010
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class TestCompositeTaskTrackerInstrumentation {
+  private static final Log LOG = LogFactory.getLog(
+      TestCompositeTaskTrackerInstrumentation.class);
+
+  @Test
+  public void testCompositeInstrumentation() throws IOException {
+    // Create two instrumentation instances
+    TaskTracker tt = new TaskTracker();
+    DummyTaskTrackerInstrumentation inst1 =
+      new DummyTaskTrackerInstrumentation(tt);
+    DummyTaskTrackerInstrumentation inst2 =
+      new DummyTaskTrackerInstrumentation(tt);
+    
+    // Add them to a composite object
+    ArrayList<TaskTrackerInstrumentation> insts =
+      new ArrayList<TaskTrackerInstrumentation>();
+    insts.add(inst1);
+    insts.add(inst2);
+    CompositeTaskTrackerInstrumentation comp =
+      new CompositeTaskTrackerInstrumentation(tt, insts);
+
+    // Create some dummy objects to pass to instrumentation methods
+    TaskAttemptID tid = new TaskAttemptID();
+    File file = new File("file");
+    Task task = new MapTask();
+    TaskStatus status = new MapTaskStatus();
+
+    // Test that completeTask propagates to listeners
+    assertFalse(inst1.completeTaskCalled);
+    assertFalse(inst2.completeTaskCalled);
+    comp.completeTask(tid);
+    assertTrue(inst1.completeTaskCalled);
+    assertTrue(inst2.completeTaskCalled);
+
+    // Test that timedoutTask propagates to listeners
+    assertFalse(inst1.timedoutTaskCalled);
+    assertFalse(inst2.timedoutTaskCalled);
+    comp.timedoutTask(tid);
+    assertTrue(inst1.timedoutTaskCalled);
+    assertTrue(inst2.timedoutTaskCalled);
+
+    // Test that taskFailedPing propagates to listeners
+    assertFalse(inst1.taskFailedPingCalled);
+    assertFalse(inst2.taskFailedPingCalled);
+    comp.taskFailedPing(tid);
+    assertTrue(inst1.taskFailedPingCalled);
+    assertTrue(inst2.taskFailedPingCalled);
+
+    // Test that reportTaskLaunch propagates to listeners
+    assertFalse(inst1.reportTaskLaunchCalled);
+    assertFalse(inst2.reportTaskLaunchCalled);
+    comp.reportTaskLaunch(tid, file, file);
+    assertTrue(inst1.reportTaskLaunchCalled);
+    assertTrue(inst2.reportTaskLaunchCalled);
+
+    // Test that reportTaskEnd propagates to listeners
+    assertFalse(inst1.reportTaskEndCalled);
+    assertFalse(inst2.reportTaskEndCalled);
+    comp.reportTaskEnd(tid);
+    assertTrue(inst1.reportTaskEndCalled);
+    assertTrue(inst2.reportTaskEndCalled);
+
+    // Test that statusUpdate propagates to listeners
+    assertFalse(inst1.statusUpdateCalled);
+    assertFalse(inst2.statusUpdateCalled);
+    comp.statusUpdate(task, status);
+    assertTrue(inst1.statusUpdateCalled);
+    assertTrue(inst2.statusUpdateCalled);
+  }
+}

Added: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerInstrumentation.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerInstrumentation.java?rev=987636&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerInstrumentation.java
(added)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskTrackerInstrumentation.java
Fri Aug 20 21:15:00 2010
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.server.tasktracker.TTConfig;
+
+public class TestTaskTrackerInstrumentation {
+  private static final Log LOG = LogFactory.getLog(
+      TestTaskTrackerInstrumentation.class);
+
+  @Test
+  public void testCreateInstrumentationWithSingleClass() {
+    // Check that if only a single instrumentation class is given,
+    // that class is used directly
+    JobConf conf = new JobConf();
+    conf.set(TTConfig.TT_INSTRUMENTATION,
+        DummyTaskTrackerInstrumentation.class.getName());
+    TaskTracker tracker = new TaskTracker();
+    TaskTrackerInstrumentation inst =
+      TaskTracker.createInstrumentation(tracker, conf);
+    assertEquals(DummyTaskTrackerInstrumentation.class.getName(),
+        inst.getClass().getName());
+  }
+
+  @Test
+  public void testCreateInstrumentationWithMultipleClasses() {
+    // Set up configuration to create two dummy instrumentation objects
+    JobConf conf = new JobConf();
+    String dummyClass = DummyTaskTrackerInstrumentation.class.getName();
+    String classList = dummyClass + "," + dummyClass;
+    conf.set(TTConfig.TT_INSTRUMENTATION, classList);
+    TaskTracker tracker = new TaskTracker();
+
+    // Check that a composite instrumentation object is created
+    TaskTrackerInstrumentation inst =
+      TaskTracker.createInstrumentation(tracker, conf);
+    assertEquals(CompositeTaskTrackerInstrumentation.class.getName(),
+        inst.getClass().getName());
+    
+    // Check that each member of the composite is a dummy instrumentation
+    CompositeTaskTrackerInstrumentation comp =
+      (CompositeTaskTrackerInstrumentation) inst;
+    List<TaskTrackerInstrumentation> insts = comp.getInstrumentations();
+    assertEquals(2, insts.size());
+    assertEquals(DummyTaskTrackerInstrumentation.class.getName(),
+        insts.get(0).getClass().getName());
+    assertEquals(DummyTaskTrackerInstrumentation.class.getName(),
+        insts.get(1).getClass().getName());
+  }
+
+  @Test
+  public void testCreateInstrumentationWithDefaultClass() {
+    // Check that if no instrumentation class is given, the default
+    // class (TaskTrackerMetricsInst) is used.
+    JobConf conf = new JobConf();
+    TaskTracker tracker = new TaskTracker();
+    tracker.setConf(conf); // Needed to avoid NullPointerExcepton in
+                           // TaskTrackerMetricsInst constructor
+    TaskTrackerInstrumentation inst =
+      TaskTracker.createInstrumentation(tracker, conf);
+    assertEquals(TaskTrackerMetricsInst.class.getName(),
+        inst.getClass().getName());
+  }
+
+  @Test
+  public void testCreateInstrumentationWithEmptyParam() {
+    // Check that if an empty string is given, the default instrumentation
+    // class (TaskTrackerMetricsInst) is used. An error message should also
+    // be written to the log, but we do not capture that.
+    JobConf conf = new JobConf();
+    conf.set(TTConfig.TT_INSTRUMENTATION, "");
+    TaskTracker tracker = new TaskTracker();
+    tracker.setConf(conf); // Needed to avoid NullPointerExcepton in
+                           // TaskTrackerMetricsInst constructor
+    TaskTrackerInstrumentation inst =
+      TaskTracker.createInstrumentation(tracker, conf);
+    assertEquals(TaskTrackerMetricsInst.class.getName(),
+        inst.getClass().getName());
+  }
+
+  @Test
+  public void testCreateInstrumentationWithInvalidParam() {
+    // Check that if an invalid class list is given, the default
+    // instrumentation class (TaskTrackerMetricsInst) is used. An error 
+    // should also be written to the log, but we do not capture that.
+    JobConf conf = new JobConf();
+    conf.set(TTConfig.TT_INSTRUMENTATION, "XYZ,ZZY");
+    TaskTracker tracker = new TaskTracker();
+    tracker.setConf(conf); // Needed to avoid NullPointerExcepton in
+                           // TaskTrackerMetricsInst constructor
+    TaskTrackerInstrumentation inst =
+      TaskTracker.createInstrumentation(tracker, conf);
+    assertEquals(TaskTrackerMetricsInst.class.getName(),
+        inst.getClass().getName());
+  }
+}



Mime
View raw message