tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zjf...@apache.org
Subject [1/2] git commit: TEZ-1584. Restore counters from DAGFinishedEvent when DAG is completed. (zjffdu)
Date Mon, 20 Oct 2014 02:11:25 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.5 eb054c8cd -> 9b6fc068c


TEZ-1584. Restore counters from DAGFinishedEvent when DAG is completed. (zjffdu)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/75a22293
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/75a22293
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/75a22293

Branch: refs/heads/branch-0.5
Commit: 75a22293d64317b7aa6fa0df4c8bd5a4609e419d
Parents: eb054c8
Author: Jeff Zhang <zjffdu@apache.org>
Authored: Fri Oct 17 10:18:59 2014 +0800
Committer: Jeff Zhang <zjffdu@apache.org>
Committed: Mon Oct 20 10:10:10 2014 +0800

----------------------------------------------------------------------
 .../org/apache/tez/dag/app/dag/impl/DAGImpl.java   |  4 +++-
 .../tez/dag/app/dag/impl/TestDAGRecovery.java      | 17 ++++++++++-------
 2 files changed, 13 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/75a22293/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 823626d..ca83420 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -163,7 +163,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
   Map<String, Edge> edges = new HashMap<String, Edge>();
   private TezCounters dagCounters = new TezCounters();
   private Object fullCountersLock = new Object();
-  private TezCounters fullCounters = null;
+  @VisibleForTesting
+  TezCounters fullCounters = null;
   private Set<TezVertexID> reRunningVertices = new HashSet<TezVertexID>();
 
   public final Configuration conf;
@@ -540,6 +541,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
         DAGFinishedEvent finishedEvent = (DAGFinishedEvent) historyEvent;
         this.finishTime = finishedEvent.getFinishTime();
         recoveredState = finishedEvent.getState();
+        this.fullCounters = finishedEvent.getTezCounters();
         return recoveredState;
       default:
         throw new RuntimeException("Unexpected event received for restoring"

http://git-wip-us.apache.org/repos/asf/tez/blob/75a22293/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java
index da0186e..f05a330 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGRecovery.java
@@ -72,13 +72,14 @@ public class TestDAGRecovery {
   private long startTime = initTime + 200L;
   private long commitStartTime = startTime + 200L;
   private long finishTime = commitStartTime + 200L;
+  private TezCounters tezCounters = new TezCounters();
 
   @Before
   public void setUp() {
-
     mockAppContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
     when(mockAppContext.getCurrentDAG().getDagUGI()).thenReturn(null);
     mockEventHandler = mock(EventHandler.class);
+    tezCounters.findCounter("grp_1", "counter_1").increment(1);
 
     DAGPlan dagPlan = TestDAGImpl.createTestDAGPlan();
     dag =
@@ -143,10 +144,11 @@ public class TestDAGRecovery {
   private void restoreFromDAGFinishedEvent(DAGState finalState) {
     DAGState recoveredState =
         dag.restoreFromEvent(new DAGFinishedEvent(dagId, startTime, finishTime,
-            finalState, "", new TezCounters(), user, dagName));
+            finalState, "", tezCounters, user, dagName));
     assertEquals(finishTime, dag.finishTime);
     assertFalse(dag.recoveryCommitInProgress);
     assertEquals(finalState, recoveredState);
+    assertEquals(tezCounters, dag.fullCounters);
   }
 
   /**
@@ -230,7 +232,7 @@ public class TestDAGRecovery {
 
     dag.handle(new DAGEventRecoverEvent(dagId, new ArrayList<URL>()));
     assertEquals(DAGState.SUCCEEDED, dag.getState());
-
+    assertEquals(tezCounters, dag.getAllCounters());
     // recover all the vertices to SUCCEED
     ArgumentCaptor<Event> eventCaptor = ArgumentCaptor.forClass(Event.class);
     verify(mockEventHandler, times(7)).handle(eventCaptor.capture());
@@ -263,7 +265,7 @@ public class TestDAGRecovery {
 
     dag.handle(new DAGEventRecoverEvent(dagId, new ArrayList<URL>()));
     assertEquals(DAGState.FAILED, dag.getState());
-
+    assertEquals(tezCounters, dag.getAllCounters());
     // recover all the vertices to FAILED
     ArgumentCaptor<Event> eventCaptor = ArgumentCaptor.forClass(Event.class);
     verify(mockEventHandler, times(7)).handle(eventCaptor.capture());
@@ -296,7 +298,7 @@ public class TestDAGRecovery {
 
     dag.handle(new DAGEventRecoverEvent(dagId, new ArrayList<URL>()));
     assertEquals(DAGState.KILLED, dag.getState());
-
+    assertEquals(tezCounters, dag.getAllCounters());
     // recover all the vertices to KILLED
     ArgumentCaptor<Event> eventCaptor = ArgumentCaptor.forClass(Event.class);
     verify(mockEventHandler, times(7)).handle(eventCaptor.capture());
@@ -329,7 +331,7 @@ public class TestDAGRecovery {
 
     dag.handle(new DAGEventRecoverEvent(dagId, new ArrayList<URL>()));
     assertEquals(DAGState.ERROR, dag.getState());
-
+    assertEquals(tezCounters, dag.getAllCounters());
     // recover all the vertices to KILLED
     ArgumentCaptor<Event> eventCaptor = ArgumentCaptor.forClass(Event.class);
     verify(mockEventHandler, times(7)).handle(eventCaptor.capture());
@@ -491,7 +493,8 @@ public class TestDAGRecovery {
     restoreFromDAGFinishedEvent(DAGState.SUCCEEDED);
 
     dag.handle(new DAGEventRecoverEvent(dagId, new ArrayList<URL>()));
-
+    assertEquals(DAGState.SUCCEEDED, dag.getState());
+    assertEquals(tezCounters, dag.getAllCounters());
     // recover all the vertices to SUCCEEDED
     ArgumentCaptor<Event> eventCaptor = ArgumentCaptor.forClass(Event.class);
     verify(mockEventHandler, times(7)).handle(eventCaptor.capture());


Mime
View raw message