Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java Wed Jan 11 20:53:50 2012
@@ -26,6 +26,7 @@ import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
@@ -44,8 +45,6 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocatorEvent;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -75,16 +74,217 @@ public class ContainerLauncherImpl exten
int nmTimeOut;
+ private ConcurrentHashMap<ContainerId, Container> containers =
+ new ConcurrentHashMap<ContainerId, Container>();
private AppContext context;
- private ThreadPoolExecutor launcherPool;
- private static final int INITIAL_POOL_SIZE = 10;
+ protected ThreadPoolExecutor launcherPool;
+ protected static final int INITIAL_POOL_SIZE = 10;
private int limitOnPoolSize;
private Thread eventHandlingThread;
- private BlockingQueue<ContainerLauncherEvent> eventQueue =
+ protected BlockingQueue<ContainerLauncherEvent> eventQueue =
new LinkedBlockingQueue<ContainerLauncherEvent>();
final Timer commandTimer = new Timer(true);
YarnRPC rpc;
+ private Container getContainer(ContainerId id) {
+ Container c = containers.get(id);
+ if(c == null) {
+ c = new Container();
+ Container old = containers.putIfAbsent(id, c);
+ if(old != null) {
+ c = old;
+ }
+ }
+ return c;
+ }
+
+ private void removeContainerIfDone(ContainerId id) {
+ Container c = containers.get(id);
+ if(c != null && c.isCompletelyDone()) {
+ containers.remove(id);
+ }
+ }
+
+ private static enum ContainerState {
+ PREP, FAILED, RUNNING, DONE, KILLED_BEFORE_LAUNCH
+ }
+
+ private class Container {
+ private ContainerState state;
+
+ public Container() {
+ this.state = ContainerState.PREP;
+ }
+
+ public synchronized boolean isCompletelyDone() {
+ return state == ContainerState.DONE || state == ContainerState.FAILED;
+ }
+
+ @SuppressWarnings("unchecked")
+ public synchronized void launch(ContainerRemoteLaunchEvent event) {
+ TaskAttemptId taskAttemptID = event.getTaskAttemptID();
+ LOG.info("Launching " + taskAttemptID);
+ if(this.state == ContainerState.KILLED_BEFORE_LAUNCH) {
+ state = ContainerState.DONE;
+ sendContainerLaunchFailedMsg(taskAttemptID,
+ "Container was killed before it was launched");
+ return;
+ }
+ CommandTimerTask timerTask = new CommandTimerTask(Thread
+ .currentThread(), event);
+
+ final String containerManagerBindAddr = event.getContainerMgrAddress();
+ ContainerId containerID = event.getContainerID();
+ ContainerToken containerToken = event.getContainerToken();
+
+ ContainerManager proxy = null;
+ try {
+ commandTimer.schedule(timerTask, nmTimeOut);
+
+ proxy = getCMProxy(containerID, containerManagerBindAddr,
+ containerToken);
+
+ // Interrupted during getProxy, but that didn't throw exception
+ if (Thread.interrupted()) {
+ // The timer canceled the command in the mean while.
+ String message = "Container launch failed for " + containerID
+ + " : Start-container for " + event.getContainerID()
+ + " got interrupted. Returning.";
+ this.state = ContainerState.FAILED;
+ sendContainerLaunchFailedMsg(taskAttemptID, message);
+ return;
+ }
+ // Construct the actual Container
+ ContainerLaunchContext containerLaunchContext =
+ event.getContainer();
+
+ // Now launch the actual container
+ StartContainerRequest startRequest = Records
+ .newRecord(StartContainerRequest.class);
+ startRequest.setContainerLaunchContext(containerLaunchContext);
+ StartContainerResponse response = proxy.startContainer(startRequest);
+
+ // container started properly. Stop the timer
+ timerTask.cancel();
+ if (Thread.interrupted()) {
+ // The timer canceled the command in the mean while, but
+ // startContainer didn't throw exception
+ String message = "Container launch failed for " + containerID
+ + " : Start-container for " + event.getContainerID()
+ + " got interrupted. Returning.";
+ this.state = ContainerState.FAILED;
+ sendContainerLaunchFailedMsg(taskAttemptID, message);
+ return;
+ }
+
+ ByteBuffer portInfo = response
+ .getServiceResponse(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID);
+ int port = -1;
+ if(portInfo != null) {
+ port = ShuffleHandler.deserializeMetaData(portInfo);
+ }
+ LOG.info("Shuffle port returned by ContainerManager for "
+ + taskAttemptID + " : " + port);
+
+ if(port < 0) {
+ this.state = ContainerState.FAILED;
+ throw new IllegalStateException("Invalid shuffle port number "
+ + port + " returned for " + taskAttemptID);
+ }
+
+ // after launching, send launched event to task attempt to move
+ // it from ASSIGNED to RUNNING state
+ context.getEventHandler().handle(
+ new TaskAttemptContainerLaunchedEvent(taskAttemptID, port));
+ this.state = ContainerState.RUNNING;
+ } catch (Throwable t) {
+ if (Thread.interrupted()) {
+ // The timer canceled the command in the mean while.
+ LOG.info("Start-container for " + event.getContainerID()
+ + " got interrupted.");
+ }
+ String message = "Container launch failed for " + containerID + " : "
+ + StringUtils.stringifyException(t);
+ this.state = ContainerState.FAILED;
+ sendContainerLaunchFailedMsg(taskAttemptID, message);
+ } finally {
+ timerTask.cancel();
+ if (proxy != null) {
+ ContainerLauncherImpl.this.rpc.stopProxy(proxy, getConfig());
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public synchronized void kill(ContainerLauncherEvent event) {
+ if(this.state == ContainerState.PREP) {
+ this.state = ContainerState.KILLED_BEFORE_LAUNCH;
+ } else {
+ CommandTimerTask timerTask = new CommandTimerTask(Thread
+ .currentThread(), event);
+
+ final String containerManagerBindAddr = event.getContainerMgrAddress();
+ ContainerId containerID = event.getContainerID();
+ ContainerToken containerToken = event.getContainerToken();
+ TaskAttemptId taskAttemptID = event.getTaskAttemptID();
+ LOG.info("KILLING " + taskAttemptID);
+ commandTimer.schedule(timerTask, nmTimeOut);
+
+ ContainerManager proxy = null;
+ try {
+ proxy = getCMProxy(containerID, containerManagerBindAddr,
+ containerToken);
+
+ if (Thread.interrupted()) {
+ // The timer canceled the command in the mean while. No need to
+ // return, send cleaned up event anyways.
+ LOG.info("Stop-container for " + event.getContainerID()
+ + " got interrupted.");
+ } else {
+ // kill the remote container if already launched
+ StopContainerRequest stopRequest = Records
+ .newRecord(StopContainerRequest.class);
+ stopRequest.setContainerId(event.getContainerID());
+ proxy.stopContainer(stopRequest);
+ }
+ } catch (Throwable t) {
+
+ if (Thread.interrupted()) {
+ // The timer canceled the command in the mean while, clear the
+ // interrupt flag
+ LOG.info("Stop-container for " + event.getContainerID()
+ + " got interrupted.");
+ }
+
+ // ignore the cleanup failure
+ String message = "cleanup failed for container "
+ + event.getContainerID() + " : "
+ + StringUtils.stringifyException(t);
+ context.getEventHandler().handle(
+ new TaskAttemptDiagnosticsUpdateEvent(taskAttemptID, message));
+ LOG.warn(message);
+ } finally {
+ timerTask.cancel();
+ if (Thread.interrupted()) {
+ LOG.info("Stop-container for " + event.getContainerID()
+ + " got interrupted.");
+ // ignore the cleanup failure
+ context.getEventHandler().handle(
+ new TaskAttemptDiagnosticsUpdateEvent(taskAttemptID,
+ "cleanup failed for container " + event.getContainerID()));
+ }
+ if (proxy != null) {
+ ContainerLauncherImpl.this.rpc.stopProxy(proxy, getConfig());
+ }
+ }
+ this.state = ContainerState.DONE;
+ }
+ // after killing, send killed event to task attempt
+ context.getEventHandler().handle(
+ new TaskAttemptEvent(event.getTaskAttemptID(),
+ TaskAttemptEventType.TA_CONTAINER_CLEANED));
+ }
+ }
// To track numNodes.
Set<String> allNodes = new HashSet<String>();
@@ -102,11 +302,16 @@ public class ContainerLauncherImpl exten
this.limitOnPoolSize = conf.getInt(
MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT,
MRJobConfig.DEFAULT_MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT);
+ LOG.info("Upper limit on the thread pool size is " + this.limitOnPoolSize);
this.nmTimeOut = conf.getInt(ContainerLauncher.MR_AM_NM_COMMAND_TIMEOUT,
ContainerLauncher.DEFAULT_NM_COMMAND_TIMEOUT);
- this.rpc = YarnRPC.create(conf);
+ this.rpc = createYarnRPC(conf);
super.init(conf);
}
+
+ protected YarnRPC createYarnRPC(Configuration conf) {
+ return YarnRPC.create(conf);
+ }
public void start() {
@@ -118,7 +323,7 @@ public class ContainerLauncherImpl exten
Integer.MAX_VALUE, 1, TimeUnit.HOURS,
new LinkedBlockingQueue<Runnable>(),
tf);
- eventHandlingThread = new Thread(new Runnable() {
+ eventHandlingThread = new Thread() {
@Override
public void run() {
ContainerLauncherEvent event = null;
@@ -141,26 +346,27 @@ public class ContainerLauncherImpl exten
int numNodes = allNodes.size();
int idealPoolSize = Math.min(limitOnPoolSize, numNodes);
- if (poolSize <= idealPoolSize) {
+ if (poolSize < idealPoolSize) {
// Bump up the pool size to idealPoolSize+INITIAL_POOL_SIZE, the
// later is just a buffer so we are not always increasing the
// pool-size
- int newPoolSize = idealPoolSize + INITIAL_POOL_SIZE;
- LOG.info("Setting ContainerLauncher pool size to "
- + newPoolSize);
+ int newPoolSize = Math.min(limitOnPoolSize, idealPoolSize
+ + INITIAL_POOL_SIZE);
+ LOG.info("Setting ContainerLauncher pool size to " + newPoolSize
+ + " as number-of-nodes to talk to is " + numNodes);
launcherPool.setCorePoolSize(newPoolSize);
}
}
// the events from the queue are handled in parallel
// using a thread pool
- launcherPool.execute(new EventProcessor(event));
+ launcherPool.execute(createEventProcessor(event));
// TODO: Group launching of multiple containers to a single
// NodeManager into a single connection
}
}
- });
+ };
eventHandlingThread.setName("ContainerLauncher Event Handler");
eventHandlingThread.start();
super.start();
@@ -172,14 +378,16 @@ public class ContainerLauncherImpl exten
super.stop();
}
+ protected EventProcessor createEventProcessor(ContainerLauncherEvent event) {
+ return new EventProcessor(event);
+ }
+
protected ContainerManager getCMProxy(ContainerId containerID,
final String containerManagerBindAddr, ContainerToken containerToken)
throws IOException {
UserGroupInformation user = UserGroupInformation.getCurrentUser();
- this.allNodes.add(containerManagerBindAddr);
-
if (UserGroupInformation.isSecurityEnabled()) {
Token<ContainerTokenIdentifier> token = new Token<ContainerTokenIdentifier>(
containerToken.getIdentifier().array(), containerToken
@@ -244,182 +452,35 @@ public class ContainerLauncherImpl exten
/**
* Setup and start the container on remote nodemanager.
*/
- private class EventProcessor implements Runnable {
+ class EventProcessor implements Runnable {
private ContainerLauncherEvent event;
EventProcessor(ContainerLauncherEvent event) {
this.event = event;
}
- @SuppressWarnings("unchecked")
@Override
public void run() {
LOG.info("Processing the event " + event.toString());
// Load ContainerManager tokens before creating a connection.
// TODO: Do it only once per NodeManager.
- final String containerManagerBindAddr = event.getContainerMgrAddress();
ContainerId containerID = event.getContainerID();
- ContainerToken containerToken = event.getContainerToken();
- TaskAttemptId taskAttemptID = event.getTaskAttemptID();
-
- ContainerManager proxy = null;
-
- CommandTimerTask timerTask = new CommandTimerTask(Thread
- .currentThread(), event);
+ Container c = getContainer(containerID);
switch(event.getType()) {
case CONTAINER_REMOTE_LAUNCH:
ContainerRemoteLaunchEvent launchEvent
= (ContainerRemoteLaunchEvent) event;
-
- try {
- commandTimer.schedule(timerTask, nmTimeOut);
-
- proxy = getCMProxy(containerID, containerManagerBindAddr,
- containerToken);
-
- // Interruped during getProxy, but that didn't throw exception
- if (Thread.interrupted()) {
- // The timer cancelled the command in the mean while.
- String message = "Container launch failed for " + containerID
- + " : Start-container for " + event.getContainerID()
- + " got interrupted. Returning.";
- sendContainerLaunchFailedMsg(taskAttemptID, message);
- return;
- }
-
- // Construct the actual Container
- ContainerLaunchContext containerLaunchContext =
- launchEvent.getContainer();
-
- // Now launch the actual container
- StartContainerRequest startRequest = Records
- .newRecord(StartContainerRequest.class);
- startRequest.setContainerLaunchContext(containerLaunchContext);
- StartContainerResponse response = proxy.startContainer(startRequest);
-
- // container started properly. Stop the timer
- timerTask.cancel();
- if (Thread.interrupted()) {
- // The timer cancelled the command in the mean while, but
- // startContainer didn't throw exception
- String message = "Container launch failed for " + containerID
- + " : Start-container for " + event.getContainerID()
- + " got interrupted. Returning.";
- sendContainerLaunchFailedMsg(taskAttemptID, message);
- return;
- }
-
- ByteBuffer portInfo = response
- .getServiceResponse(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID);
- int port = -1;
- if(portInfo != null) {
- port = ShuffleHandler.deserializeMetaData(portInfo);
- }
- LOG.info("Shuffle port returned by ContainerManager for "
- + taskAttemptID + " : " + port);
-
- if(port < 0) {
- throw new IllegalStateException("Invalid shuffle port number "
- + port + " returned for " + taskAttemptID);
- }
-
- // after launching, send launched event to task attempt to move
- // it from ASSIGNED to RUNNING state
- context.getEventHandler().handle(
- new TaskAttemptContainerLaunchedEvent(taskAttemptID, port));
- } catch (Throwable t) {
- if (Thread.interrupted()) {
- // The timer cancelled the command in the mean while.
- LOG.info("Start-container for " + event.getContainerID()
- + " got interrupted.");
- }
- String message = "Container launch failed for " + containerID
- + " : " + StringUtils.stringifyException(t);
- sendContainerLaunchFailedMsg(taskAttemptID, message);
- } finally {
- timerTask.cancel();
- if (proxy != null) {
- ContainerLauncherImpl.this.rpc.stopProxy(proxy, getConfig());
- }
- }
-
+ c.launch(launchEvent);
break;
case CONTAINER_REMOTE_CLEANUP:
- // We will have to remove the launch (meant "cleanup"? FIXME) event if it is still in eventQueue
- // and not yet processed
- if (eventQueue.contains(event)) {
- eventQueue.remove(event); // TODO: Any synchro needed?
- //deallocate the container
- context.getEventHandler().handle(
- new ContainerAllocatorEvent(taskAttemptID,
- ContainerAllocator.EventType.CONTAINER_DEALLOCATE));
- } else {
-
- try {
- commandTimer.schedule(timerTask, nmTimeOut);
-
- proxy = getCMProxy(containerID, containerManagerBindAddr,
- containerToken);
-
- if (Thread.interrupted()) {
- // The timer cancelled the command in the mean while. No need to
- // return, send cleanedup event anyways.
- LOG.info("Stop-container for " + event.getContainerID()
- + " got interrupted.");
- } else {
-
- // TODO:check whether container is launched
-
- // kill the remote container if already launched
- StopContainerRequest stopRequest = Records
- .newRecord(StopContainerRequest.class);
- stopRequest.setContainerId(event.getContainerID());
- proxy.stopContainer(stopRequest);
- }
- } catch (Throwable t) {
-
- if (Thread.interrupted()) {
- // The timer cancelled the command in the mean while, clear the
- // interrupt flag
- LOG.info("Stop-container for " + event.getContainerID()
- + " got interrupted.");
- }
-
- // ignore the cleanup failure
- String message = "cleanup failed for container "
- + event.getContainerID() + " : "
- + StringUtils.stringifyException(t);
- context.getEventHandler()
- .handle(
- new TaskAttemptDiagnosticsUpdateEvent(taskAttemptID,
- message));
- LOG.warn(message);
- } finally {
- timerTask.cancel();
- if (Thread.interrupted()) {
- LOG.info("Stop-container for " + event.getContainerID()
- + " got interrupted.");
- // ignore the cleanup failure
- context.getEventHandler()
- .handle(new TaskAttemptDiagnosticsUpdateEvent(taskAttemptID,
- "cleanup failed for container " + event.getContainerID()));
- }
- if (proxy != null) {
- ContainerLauncherImpl.this.rpc.stopProxy(proxy, getConfig());
- }
- }
-
- // after killing, send killed event to taskattempt
- context.getEventHandler().handle(
- new TaskAttemptEvent(event.getTaskAttemptID(),
- TaskAttemptEventType.TA_CONTAINER_CLEANED));
- }
+ c.kill(event);
break;
}
+ removeContainerIfDone(containerID);
}
}
@@ -438,6 +499,7 @@ public class ContainerLauncherImpl exten
public void handle(ContainerLauncherEvent event) {
try {
eventQueue.put(event);
+ this.allNodes.add(event.getContainerMgrAddress());
} catch (InterruptedException e) {
throw new YarnException(e);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerRemoteLaunchEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerRemoteLaunchEvent.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerRemoteLaunchEvent.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerRemoteLaunchEvent.java Wed Jan 11 20:53:50 2012
@@ -46,6 +46,11 @@ public class ContainerRemoteLaunchEvent
public Task getRemoteTask() {
return this.task;
}
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
@Override
public boolean equals(Object obj) {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/recover/RecoveryService.java Wed Jan 11 20:53:50 2012
@@ -92,7 +92,6 @@ import org.apache.hadoop.yarn.util.Conve
//TODO:
//task cleanup for all non completed tasks
-
public class RecoveryService extends CompositeService implements Recovery {
private static final Log LOG = LogFactory.getLog(RecoveryService.class);
@@ -411,8 +410,7 @@ public class RecoveryService extends Com
if (cntrs == null) {
taskAttemptStatus.counters = null;
} else {
- taskAttemptStatus.counters = TypeConverter.toYarn(attemptInfo
- .getCounters());
+ taskAttemptStatus.counters = cntrs;
}
actualHandler.handle(new TaskAttemptStatusUpdateEvent(
taskAttemptStatus.id, taskAttemptStatus));
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AMWebServices.java Wed Jan 11 20:53:50 2012
@@ -33,6 +33,7 @@ import javax.ws.rs.core.Response.Status;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobACL;
+import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
@@ -42,6 +43,8 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.AppInfo;
+import org.apache.hadoop.mapreduce.v2.app.webapp.dao.AMAttemptInfo;
+import org.apache.hadoop.mapreduce.v2.app.webapp.dao.AMAttemptsInfo;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.ConfInfo;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.JobCounterInfo;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.JobInfo;
@@ -211,6 +214,21 @@ public class AMWebServices {
}
@GET
+ @Path("/jobs/{jobid}/jobattempts")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public AMAttemptsInfo getJobAttempts(@PathParam("jobid") String jid) {
+
+ Job job = getJobFromJobIdString(jid, appCtx);
+ AMAttemptsInfo amAttempts = new AMAttemptsInfo();
+ for (AMInfo amInfo : job.getAMInfos()) {
+ AMAttemptInfo attempt = new AMAttemptInfo(amInfo, MRApps.toString(
+ job.getID()), job.getUserName());
+ amAttempts.add(attempt);
+ }
+ return amAttempts;
+ }
+
+ @GET
@Path("/jobs/{jobid}/counters")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public JobCounterInfo getJobCounters(@Context HttpServletRequest hsr,
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/CountersBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/CountersBlock.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/CountersBlock.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/CountersBlock.java Wed Jan 11 20:53:50 2012
@@ -18,25 +18,32 @@
package org.apache.hadoop.mapreduce.v2.app.webapp;
-import com.google.inject.Inject;
+import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.JOB_ID;
+import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.TASK_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_TABLE;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
+
import java.util.Map;
-import org.apache.hadoop.mapreduce.v2.api.records.Counter;
-import org.apache.hadoop.mapreduce.v2.api.records.CounterGroup;
-import org.apache.hadoop.mapreduce.v2.api.records.Counters;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
-import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.*;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TD;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.THEAD;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
-import static org.apache.hadoop.mapreduce.v2.app.webapp.AMWebApp.*;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
+import com.google.inject.Inject;
public class CountersBlock extends HtmlBlock {
Job job;
@@ -62,8 +69,7 @@ public class CountersBlock extends HtmlB
return;
}
- if(total == null || total.getAllCounterGroups() == null ||
- total.getAllCounterGroups().size() <= 0) {
+ if(total == null || total.getGroupNames() == null) {
String type = $(TASK_ID);
if(type == null || type.isEmpty()) {
type = $(JOB_ID, "the job");
@@ -93,9 +99,9 @@ public class CountersBlock extends HtmlB
th(".group.ui-state-default", "Counter Group").
th(".ui-state-default", "Counters")._()._().
tbody();
- for (CounterGroup g : total.getAllCounterGroups().values()) {
- CounterGroup mg = map == null ? null : map.getCounterGroup(g.getName());
- CounterGroup rg = reduce == null ? null : reduce.getCounterGroup(g.getName());
+ for (CounterGroup g : total) {
+ CounterGroup mg = map == null ? null : map.getGroup(g.getName());
+ CounterGroup rg = reduce == null ? null : reduce.getGroup(g.getName());
++numGroups;
// This is mostly for demonstration :) Typically we'd introduced
// a CounterGroup block to reduce the verbosity. OTOH, this
@@ -116,7 +122,7 @@ public class CountersBlock extends HtmlB
TBODY<TABLE<TD<TR<TBODY<TABLE<DIV<Hamlet>>>>>>> group = groupHeadRow.
th(map == null ? "Value" : "Total")._()._().
tbody();
- for (Counter counter : g.getAllCounters().values()) {
+ for (Counter counter : g) {
// Ditto
TR<TBODY<TABLE<TD<TR<TBODY<TABLE<DIV<Hamlet>>>>>>>> groupRow = group.
tr();
@@ -130,8 +136,8 @@ public class CountersBlock extends HtmlB
_();
}
if (map != null) {
- Counter mc = mg == null ? null : mg.getCounter(counter.getName());
- Counter rc = rg == null ? null : rg.getCounter(counter.getName());
+ Counter mc = mg == null ? null : mg.findCounter(counter.getName());
+ Counter rc = rg == null ? null : rg.findCounter(counter.getName());
groupRow.
td(mc == null ? "0" : String.valueOf(mc.getValue())).
td(rc == null ? "0" : String.valueOf(rc.getValue()));
@@ -173,14 +179,14 @@ public class CountersBlock extends HtmlB
}
// Get all types of counters
Map<TaskId, Task> tasks = job.getTasks();
- total = job.getCounters();
- map = JobImpl.newCounters();
- reduce = JobImpl.newCounters();
+ total = job.getAllCounters();
+ map = new Counters();
+ reduce = new Counters();
for (Task t : tasks.values()) {
Counters counters = t.getCounters();
switch (t.getType()) {
- case MAP: JobImpl.incrAllCounters(map, counters); break;
- case REDUCE: JobImpl.incrAllCounters(reduce, counters); break;
+ case MAP: map.incrAllCounters(counters); break;
+ case REDUCE: reduce.incrAllCounters(counters); break;
}
}
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JAXBContextResolver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JAXBContextResolver.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JAXBContextResolver.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JAXBContextResolver.java Wed Jan 11 20:53:50 2012
@@ -30,6 +30,8 @@ import javax.ws.rs.ext.ContextResolver;
import javax.ws.rs.ext.Provider;
import javax.xml.bind.JAXBContext;
+import org.apache.hadoop.mapreduce.v2.app.webapp.dao.AMAttemptInfo;
+import org.apache.hadoop.mapreduce.v2.app.webapp.dao.AMAttemptsInfo;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.AppInfo;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.ConfInfo;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.ConfEntryInfo;
@@ -54,22 +56,22 @@ public class JAXBContextResolver impleme
private JAXBContext context;
private final Set<Class> types;
-
+
// you have to specify all the dao classes here
- private final Class[] cTypes = {AppInfo.class, CounterInfo.class,
- JobTaskAttemptCounterInfo.class, JobTaskCounterInfo.class,
- TaskCounterGroupInfo.class, ConfInfo.class, JobCounterInfo.class,
- TaskCounterInfo.class, CounterGroupInfo.class, JobInfo.class,
- JobsInfo.class, ReduceTaskAttemptInfo.class, TaskAttemptInfo.class,
- TaskInfo.class, TasksInfo.class, TaskAttemptsInfo.class,
- ConfEntryInfo.class};
-
+ private final Class[] cTypes = {AMAttemptInfo.class, AMAttemptsInfo.class,
+ AppInfo.class, CounterInfo.class, JobTaskAttemptCounterInfo.class,
+ JobTaskCounterInfo.class, TaskCounterGroupInfo.class, ConfInfo.class,
+ JobCounterInfo.class, TaskCounterInfo.class, CounterGroupInfo.class,
+ JobInfo.class, JobsInfo.class, ReduceTaskAttemptInfo.class,
+ TaskAttemptInfo.class, TaskInfo.class, TasksInfo.class,
+ TaskAttemptsInfo.class, ConfEntryInfo.class};
+
public JAXBContextResolver() throws Exception {
this.types = new HashSet<Class>(Arrays.asList(cTypes));
this.context = new JSONJAXBContext(JSONConfiguration.natural().
rootUnwrapping(false).build(), cTypes);
}
-
+
@Override
public JAXBContext getContext(Class<?> objectType) {
return (types.contains(objectType)) ? context : null;
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/JobBlock.java Wed Jan 11 20:53:50 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.v2.a
import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.JOB_ID;
import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._EVEN;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._ODD;
@@ -28,14 +29,22 @@ import static org.apache.hadoop.yarn.web
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
import java.util.Date;
+import java.util.List;
+import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.app.webapp.dao.AMAttemptInfo;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.JobInfo;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.mapreduce.v2.util.MRApps.TaskAttemptStateUI;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
@@ -62,6 +71,11 @@ public class JobBlock extends HtmlBlock
p()._("Sorry, ", jid, " not found.")._();
return;
}
+
+ List<AMInfo> amInfos = job.getAMInfos();
+ String amString =
+ amInfos.size() == 1 ? "ApplicationMaster" : "ApplicationMasters";
+
JobInfo jinfo = new JobInfo(job, true);
info("Job Overview").
_("Job Name:", jinfo.getName()).
@@ -69,10 +83,40 @@ public class JobBlock extends HtmlBlock
_("Uberized:", jinfo.isUberized()).
_("Started:", new Date(jinfo.getStartTime())).
_("Elapsed:", StringUtils.formatTime(jinfo.getElapsedTime()));
- html.
+ DIV<Hamlet> div = html.
_(InfoBlock.class).
- div(_INFO_WRAP).
+ div(_INFO_WRAP);
+
+ // MRAppMasters Table
+ TABLE<DIV<Hamlet>> table = div.table("#job");
+ table.
+ tr().
+ th(amString).
+ _().
+ tr().
+ th(_TH, "Attempt Number").
+ th(_TH, "Start Time").
+ th(_TH, "Node").
+ th(_TH, "Logs").
+ _();
+ for (AMInfo amInfo : amInfos) {
+ AMAttemptInfo attempt = new AMAttemptInfo(amInfo,
+ jinfo.getId(), jinfo.getUserName());
+
+ table.tr().
+ td(String.valueOf(attempt.getAttemptId())).
+ td(new Date(attempt.getStartTime()).toString()).
+ td().a(".nodelink", url("http://", attempt.getNodeHttpAddress()),
+ attempt.getNodeHttpAddress())._().
+ td().a(".logslink", url(attempt.getLogsLink()),
+ "logs")._().
+ _();
+ }
+
+ table._();
+ div._();
+ html.div(_INFO_WRAP).
// Tasks table
table("#job").
tr().
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/SingleCounterBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/SingleCounterBlock.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/SingleCounterBlock.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/SingleCounterBlock.java Wed Jan 11 20:53:50 2012
@@ -18,13 +18,18 @@
package org.apache.hadoop.mapreduce.v2.app.webapp;
-import com.google.inject.Inject;
+import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.COUNTER_GROUP;
+import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.COUNTER_NAME;
+import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.JOB_ID;
+import static org.apache.hadoop.mapreduce.v2.app.webapp.AMParams.TASK_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
+
import java.util.Map;
import java.util.TreeMap;
-import org.apache.hadoop.mapreduce.v2.api.records.Counter;
-import org.apache.hadoop.mapreduce.v2.api.records.CounterGroup;
-import org.apache.hadoop.mapreduce.v2.api.records.Counters;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
@@ -40,8 +45,7 @@ import org.apache.hadoop.yarn.webapp.ham
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
-import static org.apache.hadoop.mapreduce.v2.app.webapp.AMWebApp.*;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
+import com.google.inject.Inject;
public class SingleCounterBlock extends HtmlBlock {
protected TreeMap<String, Long> values = new TreeMap<String, Long>();
@@ -122,10 +126,10 @@ public class SingleCounterBlock extends
task.getAttempts().entrySet()) {
long value = 0;
Counters counters = entry.getValue().getCounters();
- CounterGroup group = (counters != null)
- ? counters.getCounterGroup($(COUNTER_GROUP)) : null;
+ CounterGroup group = (counters != null) ? counters
+ .getGroup($(COUNTER_GROUP)) : null;
if(group != null) {
- Counter c = group.getCounter($(COUNTER_NAME));
+ Counter c = group.findCounter($(COUNTER_NAME));
if(c != null) {
value = c.getValue();
}
@@ -140,9 +144,9 @@ public class SingleCounterBlock extends
for(Map.Entry<TaskId, Task> entry : tasks.entrySet()) {
long value = 0;
CounterGroup group = entry.getValue().getCounters()
- .getCounterGroup($(COUNTER_GROUP));
+ .getGroup($(COUNTER_GROUP));
if(group != null) {
- Counter c = group.getCounter($(COUNTER_NAME));
+ Counter c = group.findCounter($(COUNTER_NAME));
if(c != null) {
value = c.getValue();
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/CounterGroupInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/CounterGroupInfo.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/CounterGroupInfo.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/CounterGroupInfo.java Wed Jan 11 20:53:50 2012
@@ -24,8 +24,8 @@ import javax.xml.bind.annotation.XmlAcce
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
-import org.apache.hadoop.mapreduce.v2.api.records.Counter;
-import org.apache.hadoop.mapreduce.v2.api.records.CounterGroup;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
@XmlRootElement(name = "counterGroup")
@XmlAccessorType(XmlAccessType.FIELD)
@@ -38,14 +38,14 @@ public class CounterGroupInfo {
public CounterGroupInfo() {
}
- public CounterGroupInfo(String name, CounterGroup g, CounterGroup mg,
+ public CounterGroupInfo(String name, CounterGroup group, CounterGroup mg,
CounterGroup rg) {
this.counterGroupName = name;
this.counter = new ArrayList<CounterInfo>();
- for (Counter c : g.getAllCounters().values()) {
- Counter mc = mg == null ? null : mg.getCounter(c.getName());
- Counter rc = rg == null ? null : rg.getCounter(c.getName());
+ for (Counter c : group) {
+ Counter mc = mg == null ? null : mg.findCounter(c.getName());
+ Counter rc = rg == null ? null : rg.findCounter(c.getName());
CounterInfo cinfo = new CounterInfo(c, mc, rc);
this.counter.add(cinfo);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/CounterInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/CounterInfo.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/CounterInfo.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/CounterInfo.java Wed Jan 11 20:53:50 2012
@@ -21,7 +21,7 @@ import javax.xml.bind.annotation.XmlAcce
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
-import org.apache.hadoop.mapreduce.v2.api.records.Counter;
+import org.apache.hadoop.mapreduce.Counter;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@@ -35,9 +35,9 @@ public class CounterInfo {
public CounterInfo() {
}
- public CounterInfo(Counter counter, Counter mc, Counter rc) {
- this.name = counter.getName();
- this.totalCounterValue = counter.getValue();
+ public CounterInfo(Counter c, Counter mc, Counter rc) {
+ this.name = c.getName();
+ this.totalCounterValue = c.getValue();
this.mapCounterValue = mc == null ? 0 : mc.getValue();
this.reduceCounterValue = rc == null ? 0 : rc.getValue();
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobCounterInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobCounterInfo.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobCounterInfo.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobCounterInfo.java Wed Jan 11 20:53:50 2012
@@ -25,13 +25,12 @@ import javax.xml.bind.annotation.XmlAcce
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
-import org.apache.hadoop.mapreduce.v2.api.records.CounterGroup;
-import org.apache.hadoop.mapreduce.v2.api.records.Counters;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
-import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
@XmlRootElement(name = "jobCounters")
@@ -56,18 +55,15 @@ public class JobCounterInfo {
counterGroup = new ArrayList<CounterGroupInfo>();
this.id = MRApps.toString(job.getID());
- int numGroups = 0;
-
if (total != null) {
- for (CounterGroup g : total.getAllCounterGroups().values()) {
+ for (CounterGroup g : total) {
if (g != null) {
- CounterGroup mg = map == null ? null : map.getCounterGroup(g
- .getName());
- CounterGroup rg = reduce == null ? null : reduce.getCounterGroup(g
- .getName());
- ++numGroups;
+ CounterGroup mg = map == null ? null : map.getGroup(g.getName());
+ CounterGroup rg = reduce == null ? null : reduce
+ .getGroup(g.getName());
- CounterGroupInfo cginfo = new CounterGroupInfo(g.getName(), g, mg, rg);
+ CounterGroupInfo cginfo = new CounterGroupInfo(g.getName(), g,
+ mg, rg);
counterGroup.add(cginfo);
}
}
@@ -75,23 +71,23 @@ public class JobCounterInfo {
}
private void getCounters(AppContext ctx, Job job) {
- total = JobImpl.newCounters();
+ total = new Counters();
if (job == null) {
return;
}
- map = JobImpl.newCounters();
- reduce = JobImpl.newCounters();
+ map = new Counters();
+ reduce = new Counters();
// Get all types of counters
Map<TaskId, Task> tasks = job.getTasks();
for (Task t : tasks.values()) {
Counters counters = t.getCounters();
- JobImpl.incrAllCounters(total, counters);
+ total.incrAllCounters(counters);
switch (t.getType()) {
case MAP:
- JobImpl.incrAllCounters(map, counters);
+ map.incrAllCounters(counters);
break;
case REDUCE:
- JobImpl.incrAllCounters(reduce, counters);
+ reduce.incrAllCounters(counters);
break;
}
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobInfo.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobInfo.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobInfo.java Wed Jan 11 20:53:50 2012
@@ -214,7 +214,7 @@ public class JobInfo {
return this.state.toString();
}
- public String getUser() {
+ public String getUserName() {
return this.user;
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobTaskAttemptCounterInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobTaskAttemptCounterInfo.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobTaskAttemptCounterInfo.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobTaskAttemptCounterInfo.java Wed Jan 11 20:53:50 2012
@@ -25,8 +25,8 @@ import javax.xml.bind.annotation.XmlAcce
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
-import org.apache.hadoop.mapreduce.v2.api.records.CounterGroup;
-import org.apache.hadoop.mapreduce.v2.api.records.Counters;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
@@ -49,7 +49,7 @@ public class JobTaskAttemptCounterInfo {
total = taskattempt.getCounters();
taskAttemptCounterGroup = new ArrayList<TaskCounterGroupInfo>();
if (total != null) {
- for (CounterGroup g : total.getAllCounterGroups().values()) {
+ for (CounterGroup g : total) {
if (g != null) {
TaskCounterGroupInfo cginfo = new TaskCounterGroupInfo(g.getName(), g);
if (cginfo != null) {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobTaskCounterInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobTaskCounterInfo.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobTaskCounterInfo.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/JobTaskCounterInfo.java Wed Jan 11 20:53:50 2012
@@ -25,8 +25,8 @@ import javax.xml.bind.annotation.XmlAcce
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
-import org.apache.hadoop.mapreduce.v2.api.records.CounterGroup;
-import org.apache.hadoop.mapreduce.v2.api.records.Counters;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
@@ -48,7 +48,7 @@ public class JobTaskCounterInfo {
this.id = MRApps.toString(task.getID());
taskCounterGroup = new ArrayList<TaskCounterGroupInfo>();
if (total != null) {
- for (CounterGroup g : total.getAllCounterGroups().values()) {
+ for (CounterGroup g : total) {
if (g != null) {
TaskCounterGroupInfo cginfo = new TaskCounterGroupInfo(g.getName(), g);
taskCounterGroup.add(cginfo);
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskCounterGroupInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskCounterGroupInfo.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskCounterGroupInfo.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskCounterGroupInfo.java Wed Jan 11 20:53:50 2012
@@ -24,8 +24,8 @@ import javax.xml.bind.annotation.XmlAcce
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
-import org.apache.hadoop.mapreduce.v2.api.records.Counter;
-import org.apache.hadoop.mapreduce.v2.api.records.CounterGroup;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
@@ -37,11 +37,11 @@ public class TaskCounterGroupInfo {
public TaskCounterGroupInfo() {
}
- public TaskCounterGroupInfo(String name, CounterGroup g) {
+ public TaskCounterGroupInfo(String name, CounterGroup group) {
this.counterGroupName = name;
this.counter = new ArrayList<TaskCounterInfo>();
- for (Counter c : g.getAllCounters().values()) {
+ for (Counter c : group) {
TaskCounterInfo cinfo = new TaskCounterInfo(c.getName(), c.getValue());
this.counter.add(cinfo);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapred/TestTaskAttemptListenerImpl.java Wed Jan 11 20:53:50 2012
@@ -43,7 +43,7 @@ public class TestTaskAttemptListenerImpl
}
@Override
- protected void registerHeartbeatHandler() {
+ protected void registerHeartbeatHandler(Configuration conf) {
//Empty
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java Wed Jan 11 20:53:50 2012
@@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobACLsManager;
import org.apache.hadoop.mapred.ShuffleHandler;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.FileSystemCounter;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.JobCounter;
@@ -37,7 +38,6 @@ import org.apache.hadoop.mapreduce.MRCon
import org.apache.hadoop.mapreduce.TaskCounter;
import org.apache.hadoop.mapreduce.TypeConverter;
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
-import org.apache.hadoop.mapreduce.v2.api.records.Counters;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
@@ -144,7 +144,7 @@ public class MockJobs extends MockApps {
report.setFinishTime(System.currentTimeMillis()
+ (int) (Math.random() * DT) + 1);
report.setProgress((float) Math.random());
- report.setCounters(newCounters());
+ report.setCounters(TypeConverter.toYarn(newCounters()));
report.setTaskState(TASK_STATES.next());
return report;
}
@@ -159,13 +159,12 @@ public class MockJobs extends MockApps {
report.setPhase(PHASES.next());
report.setTaskAttemptState(TASK_ATTEMPT_STATES.next());
report.setProgress((float) Math.random());
- report.setCounters(newCounters());
+ report.setCounters(TypeConverter.toYarn(newCounters()));
return report;
}
- @SuppressWarnings("deprecation")
public static Counters newCounters() {
- org.apache.hadoop.mapred.Counters hc = new org.apache.hadoop.mapred.Counters();
+ Counters hc = new Counters();
for (JobCounter c : JobCounter.values()) {
hc.findCounter(c).setValue((long) (Math.random() * 1000));
}
@@ -183,7 +182,7 @@ public class MockJobs extends MockApps {
hc.findCounter(USER_COUNTER_GROUPS.next(), USER_COUNTERS.next())
.setValue((long) (Math.random() * 100000));
}
- return TypeConverter.toYarn(hc);
+ return hc;
}
public static Map<TaskAttemptId, TaskAttempt> newTaskAttempts(TaskId tid,
@@ -231,7 +230,10 @@ public class MockJobs extends MockApps {
@Override
public Counters getCounters() {
- return report.getCounters();
+ if (report != null && report.getCounters() != null) {
+ return new Counters(TypeConverter.fromYarn(report.getCounters()));
+ }
+ return null;
}
@Override
@@ -327,7 +329,8 @@ public class MockJobs extends MockApps {
@Override
public Counters getCounters() {
- return report.getCounters();
+ return new Counters(
+ TypeConverter.fromYarn(report.getCounters()));
}
@Override
@@ -373,8 +376,9 @@ public class MockJobs extends MockApps {
};
}
- public static Counters getCounters(Collection<Task> tasks) {
- Counters counters = JobImpl.newCounters();
+ public static Counters getCounters(
+ Collection<Task> tasks) {
+ Counters counters = new Counters();
return JobImpl.incrTaskCounters(counters, tasks);
}
@@ -419,7 +423,8 @@ public class MockJobs extends MockApps {
final JobReport report = newJobReport(id);
final Map<TaskId, Task> tasks = newTasks(id, n, m);
final TaskCount taskCount = getTaskCount(tasks.values());
- final Counters counters = getCounters(tasks.values());
+ final Counters counters = getCounters(tasks
+ .values());
final Path configFile = confFile;
Map<JobACL, AccessControlList> tmpJobACLs = new HashMap<JobACL, AccessControlList>();
@@ -457,7 +462,7 @@ public class MockJobs extends MockApps {
}
@Override
- public Counters getCounters() {
+ public Counters getAllCounters() {
return counters;
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java Wed Jan 11 20:53:50 2012
@@ -18,6 +18,8 @@
package org.apache.hadoop.mapreduce.v2.app;
+import java.net.Proxy;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
@@ -71,6 +73,34 @@ public class TestJobEndNotifier extends
waitInterval == 5);
}
+ private void testProxyConfiguration(Configuration conf) {
+ conf.set(MRJobConfig.MR_JOB_END_NOTIFICATION_PROXY, "somehost");
+ setConf(conf);
+ Assert.assertTrue("Proxy shouldn't be set because port wasn't specified",
+ proxyToUse.type() == Proxy.Type.DIRECT);
+ conf.set(MRJobConfig.MR_JOB_END_NOTIFICATION_PROXY, "somehost:someport");
+ setConf(conf);
+ Assert.assertTrue("Proxy shouldn't be set because port wasn't numeric",
+ proxyToUse.type() == Proxy.Type.DIRECT);
+ conf.set(MRJobConfig.MR_JOB_END_NOTIFICATION_PROXY, "somehost:1000");
+ setConf(conf);
+ Assert.assertTrue("Proxy should have been set but wasn't ",
+ proxyToUse.toString().equals("HTTP @ somehost:1000"));
+ conf.set(MRJobConfig.MR_JOB_END_NOTIFICATION_PROXY, "socks@somehost:1000");
+ setConf(conf);
+ Assert.assertTrue("Proxy should have been socks but wasn't ",
+ proxyToUse.toString().equals("SOCKS @ somehost:1000"));
+ conf.set(MRJobConfig.MR_JOB_END_NOTIFICATION_PROXY, "SOCKS@somehost:1000");
+ setConf(conf);
+ Assert.assertTrue("Proxy should have been socks but wasn't ",
+ proxyToUse.toString().equals("SOCKS @ somehost:1000"));
+ conf.set(MRJobConfig.MR_JOB_END_NOTIFICATION_PROXY, "sfafn@somehost:1000");
+ setConf(conf);
+ Assert.assertTrue("Proxy should have been http but wasn't ",
+ proxyToUse.toString().equals("HTTP @ somehost:1000"));
+
+ }
+
/**
* Test that setting parameters has the desired effect
*/
@@ -79,6 +109,7 @@ public class TestJobEndNotifier extends
Configuration conf = new Configuration();
testNumRetries(conf);
testWaitInterval(conf);
+ testProxyConfiguration(conf);
}
protected int notificationCount = 0;
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java Wed Jan 11 20:53:50 2012
@@ -18,9 +18,6 @@
package org.apache.hadoop.mapreduce.v2.app;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -30,11 +27,14 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
-import org.apache.hadoop.mapreduce.v2.api.records.Counters;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
@@ -46,13 +46,12 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.TaskReport;
import org.apache.hadoop.mapreduce.v2.api.records.TaskState;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
-import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
import org.apache.hadoop.mapreduce.v2.app.speculate.DefaultSpeculator;
import org.apache.hadoop.mapreduce.v2.app.speculate.ExponentiallySmoothedTaskRuntimeEstimator;
import org.apache.hadoop.mapreduce.v2.app.speculate.LegacyTaskRuntimeEstimator;
@@ -74,7 +73,7 @@ import org.apache.hadoop.yarn.service.Co
import org.junit.Assert;
import org.junit.Test;
-
+@SuppressWarnings({"unchecked", "rawtypes"})
public class TestRuntimeEstimators {
private static int INITIAL_NUMBER_FREE_SLOTS = 600;
@@ -399,7 +398,7 @@ public class TestRuntimeEstimators {
}
@Override
- public Counters getCounters() {
+ public Counters getAllCounters() {
throw new UnsupportedOperationException("Not supported yet.");
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesAttempts.java Wed Jan 11 20:53:50 2012
@@ -686,7 +686,7 @@ public class TestAMWebServicesAttempts e
assertTrue("name not set", (name != null && !name.isEmpty()));
JSONArray counters = counterGroup.getJSONArray("counter");
for (int j = 0; j < counters.length(); j++) {
- JSONObject counter = counters.getJSONObject(i);
+ JSONObject counter = counters.getJSONObject(j);
String counterName = counter.getString("name");
assertTrue("name not set",
(counterName != null && !counterName.isEmpty()));
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesJobs.java Wed Jan 11 20:53:50 2012
@@ -18,6 +18,7 @@
package org.apache.hadoop.mapreduce.v2.app.webapp;
+import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -33,6 +34,7 @@ import javax.xml.parsers.DocumentBuilder
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobACL;
+import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
@@ -44,6 +46,7 @@ import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
@@ -76,6 +79,7 @@ import com.sun.jersey.test.framework.Web
* /ws/v1/mapreduce/jobs
* /ws/v1/mapreduce/jobs/{jobid}
* /ws/v1/mapreduce/jobs/{jobid}/counters
+ * /ws/v1/mapreduce/jobs/{jobid}/jobattempts
*/
public class TestAMWebServicesJobs extends JerseyTest {
@@ -777,4 +781,136 @@ public class TestAMWebServicesJobs exten
}
}
+ @Test
+ public void testJobAttempts() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1")
+ .path("mapreduce").path("jobs").path(jobId).path("jobattempts")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("jobAttempts");
+ verifyJobAttempts(info, jobsMap.get(id));
+ }
+ }
+
+ @Test
+ public void testJobAttemptsSlash() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1")
+ .path("mapreduce").path("jobs").path(jobId).path("jobattempts/")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("jobAttempts");
+ verifyJobAttempts(info, jobsMap.get(id));
+ }
+ }
+
+ @Test
+ public void testJobAttemptsDefault() throws JSONException, Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1")
+ .path("mapreduce").path("jobs").path(jobId).path("jobattempts")
+ .get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject info = json.getJSONObject("jobAttempts");
+ verifyJobAttempts(info, jobsMap.get(id));
+ }
+ }
+
+ @Test
+ public void testJobAttemptsXML() throws Exception {
+ WebResource r = resource();
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ for (JobId id : jobsMap.keySet()) {
+ String jobId = MRApps.toString(id);
+
+ ClientResponse response = r.path("ws").path("v1")
+ .path("mapreduce").path("jobs").path(jobId).path("jobattempts")
+ .accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+ String xml = response.getEntity(String.class);
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+ Document dom = db.parse(is);
+ NodeList attempts = dom.getElementsByTagName("jobAttempts");
+ assertEquals("incorrect number of elements", 1, attempts.getLength());
+ NodeList info = dom.getElementsByTagName("jobAttempt");
+ verifyJobAttemptsXML(info, jobsMap.get(id));
+ }
+ }
+
+ public void verifyJobAttempts(JSONObject info, Job job)
+ throws JSONException {
+
+ JSONArray attempts = info.getJSONArray("jobAttempt");
+ assertEquals("incorrect number of elements", 2, attempts.length());
+ for (int i = 0; i < attempts.length(); i++) {
+ JSONObject attempt = attempts.getJSONObject(i);
+ verifyJobAttemptsGeneric(job, attempt.getString("nodeHttpAddress"),
+ attempt.getString("nodeId"), attempt.getInt("id"),
+ attempt.getLong("startTime"), attempt.getString("containerId"),
+ attempt.getString("logsLink"));
+ }
+ }
+
+ public void verifyJobAttemptsXML(NodeList nodes, Job job) {
+
+ assertEquals("incorrect number of elements", 2, nodes.getLength());
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Element element = (Element) nodes.item(i);
+ verifyJobAttemptsGeneric(job,
+ WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
+ WebServicesTestUtils.getXmlString(element, "nodeId"),
+ WebServicesTestUtils.getXmlInt(element, "id"),
+ WebServicesTestUtils.getXmlLong(element, "startTime"),
+ WebServicesTestUtils.getXmlString(element, "containerId"),
+ WebServicesTestUtils.getXmlString(element, "logsLink"));
+ }
+ }
+
+ public void verifyJobAttemptsGeneric(Job job, String nodeHttpAddress,
+ String nodeId, int id, long startTime, String containerId, String logsLink) {
+ boolean attemptFound = false;
+ for (AMInfo amInfo : job.getAMInfos()) {
+ if (amInfo.getAppAttemptId().getAttemptId() == id) {
+ attemptFound = true;
+ String nmHost = amInfo.getNodeManagerHost();
+ int nmHttpPort = amInfo.getNodeManagerHttpPort();
+ int nmPort = amInfo.getNodeManagerPort();
+ WebServicesTestUtils.checkStringMatch("nodeHttpAddress", nmHost + ":"
+ + nmHttpPort, nodeHttpAddress);
+ WebServicesTestUtils.checkStringMatch("nodeId",
+ BuilderUtils.newNodeId(nmHost, nmPort).toString(), nodeId);
+ assertTrue("startime not greater than 0", startTime > 0);
+ WebServicesTestUtils.checkStringMatch("containerId", amInfo
+ .getContainerId().toString(), containerId);
+
+ String localLogsLink = ujoin("node", "containerlogs", containerId);
+
+ assertTrue("logsLink", logsLink.contains(localLogsLink));
+ }
+ }
+ assertTrue("attempt: " + id + " was not found", attemptFound);
+ }
+
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/webapp/TestAMWebServicesTasks.java Wed Jan 11 20:53:50 2012
@@ -774,7 +774,7 @@ public class TestAMWebServicesTasks exte
assertTrue("name not set", (name != null && !name.isEmpty()));
JSONArray counters = counterGroup.getJSONArray("counter");
for (int j = 0; j < counters.length(); j++) {
- JSONObject counter = counters.getJSONObject(i);
+ JSONObject counter = counters.getJSONObject(j);
String counterName = counter.getString("name");
assertTrue("name not set",
(counterName != null && !counterName.isEmpty()));
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java Wed Jan 11 20:53:50 2012
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
-import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.JobPriority;
import org.apache.hadoop.mapred.TaskCompletionEvent;
@@ -45,14 +44,15 @@ import org.apache.hadoop.mapreduce.v2.ut
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+@SuppressWarnings("deprecation")
public class TypeConverter {
private static RecordFactory recordFactory;
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/MRClientProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/MRClientProtocol.java?rev=1230248&r1=1230247&r2=1230248&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/MRClientProtocol.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/api/MRClientProtocol.java Wed Jan 11 20:53:50 2012
@@ -22,6 +22,8 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.FailTaskAttemptResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetCountersResponse;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDelegationTokenResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetDiagnosticsResponse;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest;
@@ -54,4 +56,5 @@ public interface MRClientProtocol {
public KillTaskResponse killTask(KillTaskRequest request) throws YarnRemoteException;
public KillTaskAttemptResponse killTaskAttempt(KillTaskAttemptRequest request) throws YarnRemoteException;
public FailTaskAttemptResponse failTaskAttempt(FailTaskAttemptRequest request) throws YarnRemoteException;
+ public GetDelegationTokenResponse getDelegationToken(GetDelegationTokenRequest request) throws YarnRemoteException;
}
|