hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1309576 - in /hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project: ./ conf/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src...
Date Wed, 04 Apr 2012 20:00:19 GMT
Author: todd
Date: Wed Apr  4 20:00:15 2012
New Revision: 1309576

URL: http://svn.apache.org/viewvc?rev=1309576&view=rev
Log:
Merge trunk into auto-HA branch

Resolved some trivial conflicts in NNHAServiceTarget

Modified:
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/   (props changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/CHANGES.txt   (contents, props
changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/conf/   (props changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
  (props changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/c++/   (props changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/   (props changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/block_forensics/
  (props changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/build-contrib.xml
  (props changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/build.xml   (props
changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/data_join/   (props
changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/eclipse-plugin/
  (props changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/index/   (props
changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/vaidya/   (props
changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/examples/   (props changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/   (props changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/   (props changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
  (props changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
  (props changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
  (props changed)
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java
    hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/webapps/job/   (props changed)

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1309162-1309567

Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/CHANGES.txt?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/CHANGES.txt Wed Apr  4 20:00:15
2012
@@ -139,6 +139,8 @@ Release 2.0.0 - UNRELEASED
     MAPREDUCE-3955. Change MR to use ProtobufRpcEngine from hadoop-common
     instead of ProtoOverHadoopRpcEngine. (Jitendra Nath Pandey via sseth)
 
+    MAPREDUCE-4103. Fix HA docs for changes to shell command fencer args (todd)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -197,7 +199,9 @@ Release 2.0.0 - UNRELEASED
 
     MAPREDUCE-4095. TestJobInProgress#testLocality uses a bogus topology.
     (Colin Patrick McCabe via eli)
-    
+
+    MAPREDUCE-4098. TestMRApps testSetClasspath fails (tucu)
+ 
 Release 0.23.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES
@@ -240,6 +244,16 @@ Release 0.23.3 - UNRELEASED
     MAPREDUCE-4020. Web services returns incorrect JSON for deep queue tree
     (Anupam Seth via tgraves)
 
+    MAPREDUCE-3672. Killed maps shouldn't be counted towards 
+    JobCounter.NUM_FAILED_MAPS. (Anupam Seth via tgraves)
+
+    MAPREDUCE-3682 Tracker URL says AM tasks run on localhost.
+    (Ravi Prakash via tgraves)
+
+    MAPREDUCE-3082. Archive command take wrong path for input file with current
+    directory (John George via bobby)
+
+    MAPREDUCE-3650. testGetTokensForHftpFS() fails (Ravi Prakash via bobby)
 Release 0.23.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1309162-1309567

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1309162-1309567

Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
(original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
Wed Apr  4 20:00:15 2012
@@ -643,7 +643,8 @@ public class MRAppMaster extends Composi
     public synchronized void start() {
       if (job.isUber()) {
         this.containerAllocator = new LocalContainerAllocator(
-            this.clientService, this.context);
+            this.clientService, this.context, nmHost, nmPort, nmHttpPort
+            , containerID);
       } else {
         this.containerAllocator = new RMContainerAllocator(
             this.clientService, this.context);

Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
(original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
Wed Apr  4 20:00:15 2012
@@ -988,6 +988,23 @@ public abstract class TaskAttemptImpl im
     }
     return jce;
   }
+  
+  private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled(
+      TaskAttemptImpl taskAttempt) {
+    TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
+    JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
+    
+    long slotMillisIncrement = computeSlotMillis(taskAttempt);
+    
+    if (taskType == TaskType.MAP) {
+      jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1);
+      jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, slotMillisIncrement);
+    } else {
+      jce.addCounterUpdate(JobCounter.NUM_KILLED_REDUCES, 1);
+      jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, slotMillisIncrement);
+    }
+    return jce;
+  }  
 
   private static
       TaskAttemptUnsuccessfulCompletionEvent
@@ -1214,8 +1231,13 @@ public abstract class TaskAttemptImpl im
         TaskAttemptUnsuccessfulCompletionEvent tauce =
             createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
                 finalState);
-        taskAttempt.eventHandler
+        if(finalState == TaskAttemptState.FAILED) {
+          taskAttempt.eventHandler
             .handle(createJobCounterUpdateEventTAFailed(taskAttempt));
+        } else if(finalState == TaskAttemptState.KILLED) {
+          taskAttempt.eventHandler
+          .handle(createJobCounterUpdateEventTAKilled(taskAttempt));
+        }
         taskAttempt.eventHandler.handle(new JobHistoryEvent(
             taskAttempt.attemptId.getTaskId().getJobId(), tauce));
       } else {
@@ -1441,7 +1463,7 @@ public abstract class TaskAttemptImpl im
       taskAttempt.setFinishTime();
       if (taskAttempt.getLaunchTime() != 0) {
         taskAttempt.eventHandler
-            .handle(createJobCounterUpdateEventTAFailed(taskAttempt));
+            .handle(createJobCounterUpdateEventTAKilled(taskAttempt));
         TaskAttemptUnsuccessfulCompletionEvent tauce =
             createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
                 TaskAttemptState.KILLED);

Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
(original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
Wed Apr  4 20:00:15 2012
@@ -65,14 +65,23 @@ public class LocalContainerAllocator ext
   private AtomicInteger containerCount = new AtomicInteger();
   private long retryInterval;
   private long retrystartTime;
+  private String nmHost;
+  private int nmPort;
+  private int nmHttpPort;
+  private ContainerId containerId;
 
   private final RecordFactory recordFactory =
       RecordFactoryProvider.getRecordFactory(null);
 
   public LocalContainerAllocator(ClientService clientService,
-                                 AppContext context) {
+    AppContext context, String nmHost, int nmPort, int nmHttpPort
+    , ContainerId cId) {
     super(clientService, context);
     this.eventHandler = context.getEventHandler();
+    this.nmHost = nmHost;
+    this.nmPort = nmPort;
+    this.nmHttpPort = nmHttpPort;
+    this.containerId = cId;
   }
 
   @Override
@@ -131,17 +140,17 @@ public class LocalContainerAllocator ext
       LOG.info("Processing the event " + event.toString());
       ContainerId cID = recordFactory.newRecordInstance(ContainerId.class);
       cID.setApplicationAttemptId(applicationAttemptId);
-      // use negative ids to denote that these are local. Need a better way ??
-      cID.setId((-1) * containerCount.getAndIncrement());
+      // Assign the same container ID as the AM
+      cID.setId(this.containerId.getId());
       
       Container container = recordFactory.newRecordInstance(Container.class);
       container.setId(cID);
       NodeId nodeId = Records.newRecord(NodeId.class);
-      nodeId.setHost("localhost");
-      nodeId.setPort(1234);
+      nodeId.setHost(this.nmHost);
+      nodeId.setPort(this.nmPort);
       container.setNodeId(nodeId);
       container.setContainerToken(null);
-      container.setNodeHttpAddress("localhost:8042");
+      container.setNodeHttpAddress(this.nmHost + ":" + this.nmHttpPort);
       // send the container-assigned event to task attempt
 
       if (event.getAttemptID().getTaskId().getTaskType() == TaskType.MAP) {

Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
(original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
Wed Apr  4 20:00:15 2012
@@ -33,6 +33,7 @@ import org.apache.hadoop.mapreduce.v2.ap
 import org.apache.hadoop.mapreduce.v2.util.MRApps;
 import org.apache.hadoop.yarn.YarnException;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 
 import org.junit.Test;
@@ -130,15 +131,12 @@ public class TestMRApps {
     Job job = Job.getInstance();
     Map<String, String> environment = new HashMap<String, String>();
     MRApps.setClasspath(environment, job.getConfiguration());
-    assertEquals("$PWD:$HADOOP_CONF_DIR:" +
-        "$HADOOP_COMMON_HOME/share/hadoop/common/*:" +
-        "$HADOOP_COMMON_HOME/share/hadoop/common/lib/*:" +
-        "$HADOOP_HDFS_HOME/share/hadoop/hdfs/*:" +
-        "$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*:" +
-        "$YARN_HOME/share/hadoop/mapreduce/*:" +
-        "$YARN_HOME/share/hadoop/mapreduce/lib/*:" +
-        "job.jar:$PWD/*",
-        environment.get("CLASSPATH"));
+    assertTrue(environment.get("CLASSPATH").startsWith("$PWD:"));
+    String confClasspath = job.getConfiguration().get(YarnConfiguration.YARN_APPLICATION_CLASSPATH);
+    if (confClasspath != null) {
+      confClasspath = confClasspath.replaceAll(",\\s*", ":").trim();
+    }
+    assertTrue(environment.get("CLASSPATH").contains(confClasspath));
   }
 
  @Test public void testSetClasspathWithUserPrecendence() {

Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java
(original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobCounter.java
Wed Apr  4 20:00:15 2012
@@ -27,6 +27,8 @@ import org.apache.hadoop.classification.
 public enum JobCounter {
   NUM_FAILED_MAPS, 
   NUM_FAILED_REDUCES,
+  NUM_KILLED_MAPS,
+  NUM_KILLED_REDUCES,
   TOTAL_LAUNCHED_MAPS,
   TOTAL_LAUNCHED_REDUCES,
   OTHER_LOCAL_MAPS,

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1309162-1309567

Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties
(original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/org/apache/hadoop/mapreduce/JobCounter.properties
Wed Apr  4 20:00:15 2012
@@ -16,6 +16,8 @@ CounterGroupName=                  Job C
 
 NUM_FAILED_MAPS.name=              Failed map tasks
 NUM_FAILED_REDUCES.name=           Failed reduce tasks
+NUM_KILLED_MAPS.name=              Killed map tasks
+NUM_KILLED_REDUCES.name=           Killed reduce tasks
 TOTAL_LAUNCHED_MAPS.name=          Launched map tasks
 TOTAL_LAUNCHED_REDUCES.name=       Launched reduce tasks
 OTHER_LOCAL_MAPS.name=             Other local map tasks

Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java
(original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestJobCleanup.java
Wed Apr  4 20:00:15 2012
@@ -195,6 +195,8 @@ public class TestJobCleanup {
     RunningJob job = jobClient.submitJob(jc);
     JobID id = job.getID();
     job.waitForCompletion();
+    Counters counters = job.getCounters();
+    assertTrue("No. of failed maps should be 1",counters.getCounter(JobCounter.NUM_FAILED_MAPS)
== 1);
 
     if (fileName != null) {
       Path testFile = new Path(outDir, fileName);
@@ -240,6 +242,9 @@ public class TestJobCleanup {
     job.killJob(); // kill the job
 
     job.waitForCompletion(); // wait for the job to complete
+    
+    counters = job.getCounters();
+    assertTrue("No. of killed maps should be 1", counters.getCounter(JobCounter.NUM_KILLED_MAPS)
== 1);
 
     if (fileName != null) {
       Path testFile = new Path(outDir, fileName);

Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java
(original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java
Wed Apr  4 20:00:15 2012
@@ -234,8 +234,10 @@ public class TestSpeculativeExecution {
             .getValue());
     Assert.assertEquals(2, counters.findCounter(JobCounter.TOTAL_LAUNCHED_REDUCES)
             .getValue());
-    Assert.assertEquals(1, counters.findCounter(JobCounter.NUM_FAILED_MAPS)
+    Assert.assertEquals(0, counters.findCounter(JobCounter.NUM_FAILED_MAPS)
             .getValue());
+    Assert.assertEquals(1, counters.findCounter(JobCounter.NUM_KILLED_MAPS)
+        .getValue());
 
     /*----------------------------------------------------------------------
      * Test that Reducer speculates if REDUCE_SPECULATIVE is true and

Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm
(original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailability.apt.vm
Wed Apr  4 20:00:15 2012
@@ -322,13 +322,40 @@ HDFS High Availability
       The string between '(' and ')' is passed directly to a bash shell and may not
       include any closing parentheses.
 
-      When executed, the first argument to the configured script will be the address
-      of the NameNode to be fenced, followed by all arguments specified in the
-      configuration.
-
       The shell command will be run with an environment set up to contain all of the
       current Hadoop configuration variables, with the '_' character replacing any
-      '.' characters in the configuration keys. If the shell command returns an exit
+      '.' characters in the configuration keys. The configuration used has already had
+      any namenode-specific configurations promoted to their generic forms -- for example
+      <<dfs_namenode_rpc-address>> will contain the RPC address of the target
node, even
+      though the configuration may specify that variable as
+      <<dfs.namenode.rpc-address.ns1.nn1>>.
+      
+      Additionally, the following variables referring to the target node to be fenced
+      are also available:
+
+*-----------------------:-----------------------------------+
+| $target_host          | hostname of the node to be fenced |
+*-----------------------:-----------------------------------+
+| $target_port          | IPC port of the node to be fenced |
+*-----------------------:-----------------------------------+
+| $target_address       | the above two, combined as host:port |
+*-----------------------:-----------------------------------+
+| $target_nameserviceid | the nameservice ID of the NN to be fenced |
+*-----------------------:-----------------------------------+
+| $target_namenodeid    | the namenode ID of the NN to be fenced |
+*-----------------------:-----------------------------------+
+      
+      These environment variables may also be used as substitutions in the shell
+      command itself. For example:
+
+---
+<property>
+  <name>dfs.ha.fencing.methods</name>
+  <value>shell(/path/to/my/script.sh --nameservice=$target_nameserviceid $target_host:$target_port)</value>
+</property>
+---
+      
+      If the shell command returns an exit
       code of 0, the fencing is determined to be successful. If it returns any other
       exit code, the fencing was not successful and the next fencing method in the
       list will be attempted.

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/c++/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/c++:r1309162-1309567

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib:r1309162-1309567

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/block_forensics/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/block_forensics:r1309162-1309567

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build-contrib.xml:r1309162-1309567

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/build.xml
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/build.xml:r1309162-1309567

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/data_join/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/data_join:r1309162-1309567

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/eclipse-plugin:r1309162-1309567

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/index/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:r1309162-1309567

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/vaidya/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:r1309162-1309567

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/examples/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/examples:r1309162-1309567

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/java:r1309162-1309567

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:r1309162-1309567

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:r1309162-1309567

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:r1309162-1309567

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:r1309162-1309567

Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java?rev=1309576&r1=1309575&r2=1309576&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java
(original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapreduce/security/TestTokenCache.java
Wed Apr  4 20:00:15 2012
@@ -321,7 +321,7 @@ public class TestTokenCache {
     final Token<DelegationTokenIdentifier> t = 
       new Token<DelegationTokenIdentifier>(dtId, dtSecretManager);
 
-    final URI uri = new URI("hftp://host:2222/file1");
+    final URI uri = new URI("hftp://127.0.0.1:2222/file1");
     final String fs_addr = 
       SecurityUtil.buildDTServiceName(uri, NameNode.DEFAULT_PORT);
     t.setService(new Text(fs_addr));

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/webapps/job/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:r1309162-1309567



Mime
View raw message