tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bi...@apache.org
Subject git commit: TEZ-245. YARN-787 build break fix (bikas)
Date Sun, 16 Jun 2013 23:03:51 GMT
Updated Branches:
  refs/heads/master f8b47f19a -> e4fd8c312


TEZ-245. YARN-787 build break fix (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/e4fd8c31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/e4fd8c31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/e4fd8c31

Branch: refs/heads/master
Commit: e4fd8c3124806d8cc60dd35926bc8e5cd905a2f9
Parents: f8b47f1
Author: Bikas Saha <bikas@apache.org>
Authored: Sun Jun 16 16:00:07 2013 -0700
Committer: Bikas Saha <bikas@apache.org>
Committed: Sun Jun 16 16:00:07 2013 -0700

----------------------------------------------------------------------
 .../tez/dag/app/dag/impl/TaskAttemptImpl.java   | 50 ++++++++++----------
 .../apache/tez/dag/app/rm/TaskScheduler.java    |  3 --
 .../dag/app/rm/TaskSchedulerEventHandler.java   | 31 +-----------
 .../tez/dag/app/rm/TestTaskScheduler.java       |  6 +--
 4 files changed, 27 insertions(+), 63 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e4fd8c31/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
index 0eddbfb..b7ba7b8 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/TaskAttemptImpl.java
@@ -625,8 +625,8 @@ public class TaskAttemptImpl implements TaskAttempt,
             ta.getDAGID()
             );
 
-    long slotMillis = computeSlotMillis(ta);
-    jce.addCounterUpdate(DAGCounter.SLOTS_MILLIS_TASKS, slotMillis);
+//    long slotMillis = computeSlotMillis(ta);
+//    jce.addCounterUpdate(DAGCounter.SLOTS_MILLIS_TASKS, slotMillis);
     return jce;
   }
 
@@ -637,38 +637,38 @@ public class TaskAttemptImpl implements TaskAttempt,
         new DAGEventCounterUpdate(
             taskAttempt.getDAGID());
 
-    long slotMillisIncrement = computeSlotMillis(taskAttempt);
-
     if (taState == TaskAttemptStateInternal.FAILED) {
       jce.addCounterUpdate(DAGCounter.NUM_FAILED_TASKS, 1);
     } else if (taState == TaskAttemptStateInternal.KILLED) {
       jce.addCounterUpdate(DAGCounter.NUM_KILLED_TASKS, 1);
     }
-    if (!taskAlreadyCompleted) {
-      // dont double count the elapsed time
-      jce.addCounterUpdate(DAGCounter.SLOTS_MILLIS_TASKS, slotMillisIncrement);
-    }
+    
+//    long slotMillisIncrement = computeSlotMillis(taskAttempt);
+//    if (!taskAlreadyCompleted) {
+//      // dont double count the elapsed time
+//      jce.addCounterUpdate(DAGCounter.SLOTS_MILLIS_TASKS, slotMillisIncrement);
+//    }
 
     return jce;
   }
 
-  private static long computeSlotMillis(TaskAttemptImpl taskAttempt) {
-    int slotMemoryReq =
-        taskAttempt.taskResource.getMemory();
-
-    int minSlotMemSize =
-        taskAttempt.appContext.getClusterInfo().getMinContainerCapability()
-            .getMemory();
-
-    int simSlotsRequired =
-        minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) slotMemoryReq
-            / minSlotMemSize);
-
-    long slotMillisIncrement =
-        simSlotsRequired
-            * (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
-    return slotMillisIncrement;
-  }
+//  private static long computeSlotMillis(TaskAttemptImpl taskAttempt) {
+//    int slotMemoryReq =
+//        taskAttempt.taskResource.getMemory();
+//
+//    int minSlotMemSize =
+//        taskAttempt.appContext.getClusterInfo().getMinContainerCapability()
+//            .getMemory();
+//
+//    int simSlotsRequired =
+//        minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) slotMemoryReq
+//            / minSlotMemSize);
+//
+//    long slotMillisIncrement =
+//        simSlotsRequired
+//            * (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
+//    return slotMillisIncrement;
+//  }
 
   // TODO: JobHistory
   // TODO Change to return a JobHistoryEvent.

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e4fd8c31/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
index 95120c4..6068919 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskScheduler.java
@@ -47,7 +47,6 @@ import org.apache.hadoop.yarn.client.AMRMClientAsync;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.service.AbstractService;
 import org.apache.hadoop.yarn.util.RackResolver;
-import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.app.rm.TaskScheduler.TaskSchedulerAppCallback.AppFinalStatus;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -90,7 +89,6 @@ public class TaskScheduler extends AbstractService
     public void nodesUpdated(List<NodeReport> updatedNodes);
     public void appShutdownRequested();
     public void setApplicationRegistrationData(
-                                Resource minContainerCapability,
                                 Resource maxContainerCapability,
                                 Map<ApplicationAccessType, String> appAcls
                                 );
@@ -191,7 +189,6 @@ public class TaskScheduler extends AbstractService
       }
       // upcall to app outside locks
       appClient.setApplicationRegistrationData(
-                                      response.getMinimumResourceCapability(),
                                       response.getMaximumResourceCapability(),
                                       response.getApplicationACLs());
     } catch (YarnException e) {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e4fd8c31/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
index fe128b6..e4553dd 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/TaskSchedulerEventHandler.java
@@ -269,33 +269,6 @@ public class TaskSchedulerEventHandler extends AbstractService
     }
   }
 
-  /*
-  @SuppressWarnings("unchecked")
-  private int maybeComputeNormalizedRequestForType(
-      AMSchedulerTALaunchRequestEvent event, int prevComputedSize) {
-    if (prevComputedSize == 0) {
-      int supportedMaxContainerCapability = appContext.getClusterInfo()
-          .getMaxContainerCapability().getMemory();
-      prevComputedSize = event.getCapability().getMemory();
-      int minSlotMemSize = appContext.getClusterInfo()
-          .getMinContainerCapability().getMemory();
-      prevComputedSize = (int) Math.ceil((float) prevComputedSize
-          / minSlotMemSize)
-          * minSlotMemSize;
-      if (prevComputedSize > supportedMaxContainerCapability) {
-        String diagMsg = " capability required is more than the supported "
-            + "max container capability in the cluster. Killing the Job. "
-            + "ResourceReqt: " + prevComputedSize
-            + " maxContainerCapability:" + supportedMaxContainerCapability;
-        LOG.info(diagMsg);
-        eventHandler.handle(new DAGDiagnosticsUpdateEvent(job.getID(), diagMsg));
-        eventHandler.handle(new DAGEvent(job.getID(), DAGEventType.JOB_KILL));
-      }
-    }
-    return prevComputedSize;
-  }
-  */
-
   private void handleTaLaunchRequest(AMSchedulerEventTALaunchRequest event) {
     /**
          // Add to queue of pending tasks.
@@ -490,11 +463,9 @@ public class TaskSchedulerEventHandler extends AbstractService
   }
 
   @Override
-  public synchronized void setApplicationRegistrationData(Resource minContainerCapability,
+  public synchronized void setApplicationRegistrationData(
       Resource maxContainerCapability,
       Map<ApplicationAccessType, String> appAcls) {
-    this.appContext.getClusterInfo().setMinContainerCapability(
-        minContainerCapability);
     this.appContext.getClusterInfo().setMaxContainerCapability(
         maxContainerCapability);
     this.appAcls = appAcls;

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/e4fd8c31/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
index 5f66cf2..ebc0916 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestTaskScheduler.java
@@ -82,11 +82,8 @@ public class TestTaskScheduler {
     
     RegisterApplicationMasterResponse mockRegResponse = 
                                 mock(RegisterApplicationMasterResponse.class);
-    Resource mockMinResource = mock(Resource.class);
     Resource mockMaxResource = mock(Resource.class);
     Map<ApplicationAccessType, String> mockAcls = mock(Map.class);
-    when(mockRegResponse.getMinimumResourceCapability()).
-                                                   thenReturn(mockMinResource);
     when(mockRegResponse.getMaximumResourceCapability()).
                                                    thenReturn(mockMaxResource);
     when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls);    
@@ -96,8 +93,7 @@ public class TestTaskScheduler {
     scheduler.start();
     verify(mockRMClient).start();
     verify(mockRMClient).registerApplicationMaster(appHost, appPort, appUrl);
-    verify(mockApp).setApplicationRegistrationData(mockMinResource, 
-                                                   mockMaxResource, 
+    verify(mockApp).setApplicationRegistrationData(mockMaxResource, 
                                                    mockAcls);
     
     when(mockRMClient.getClusterNodeCount()).thenReturn(5);


Mime
View raw message