tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bi...@apache.org
Subject git commit: TEZ-745. Rename TEZ_AM_ABORT_ALL_OUTPUTS_ON_DAG_FAILURE in TezConfiguration to TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS (Jonathan Eagles via bikas)
Date Wed, 05 Feb 2014 21:31:50 GMT
Updated Branches:
  refs/heads/master bd5624179 -> 2c5343998


TEZ-745. Rename TEZ_AM_ABORT_ALL_OUTPUTS_ON_DAG_FAILURE in TezConfiguration to TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS
(Jonathan Eagles via bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/2c534399
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/2c534399
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/2c534399

Branch: refs/heads/master
Commit: 2c53439981f5d2d2b8ce8543a6f219fae987e00a
Parents: bd56241
Author: Bikas Saha <bikas@apache.org>
Authored: Wed Feb 5 13:31:42 2014 -0800
Committer: Bikas Saha <bikas@apache.org>
Committed: Wed Feb 5 13:31:42 2014 -0800

----------------------------------------------------------------------
 .../apache/tez/dag/api/TezConfiguration.java    |  6 +++---
 .../apache/tez/dag/app/dag/impl/DAGImpl.java    | 20 ++++++++++----------
 .../tez/dag/app/dag/impl/TestDAGImpl.java       |  2 +-
 3 files changed, 14 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2c534399/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index ba80854..1616adc 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -52,9 +52,9 @@ public class TezConfiguration extends Configuration {
   public static final String TEZ_AM_LOG_LEVEL = TEZ_AM_PREFIX+"log.level";
   public static final String TEZ_AM_LOG_LEVEL_DEFAULT = "INFO";
 
-  public static final String TEZ_AM_ABORT_ALL_OUTPUTS_ON_DAG_FAILURE =
-      TEZ_AM_PREFIX + "abort-all-outputs-on-dag-failure";
-  public static final boolean TEZ_AM_ABORT_ALL_OUTPUTS_ON_DAG_FAILURE_DEFAULT = true;
+  public static final String TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS =
+      TEZ_AM_PREFIX + "commit-all-outputs-on-dag-success";
+  public static final boolean TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS_DEFAULT = true;
 
   /** Java options for the Tez AppMaster process. */
   public static final String TEZ_AM_JAVA_OPTS = TEZ_AM_PREFIX

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2c534399/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 1c558d1..85f102c 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -133,7 +133,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
   
   private volatile boolean committedOrAborted = false;
   private volatile boolean allOutputsCommitted = false;
-  boolean abortAllOutputsOnFailure = true;
+  boolean commitAllOutputsOnSuccess = true;
 
   @VisibleForTesting
   DAGScheduler dagScheduler;
@@ -662,7 +662,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
     LOG.info("Calling DAG commit/abort for dag: " + getID());
     this.committedOrAborted = true;
     
-    boolean successfulOutputsAlreadyCommitted = !abortAllOutputsOnFailure;
+    boolean successfulOutputsAlreadyCommitted = !commitAllOutputsOnSuccess;
     boolean failedWhileCommitting = false;
     if (dagSucceeded && !successfulOutputsAlreadyCommitted) {
       // commit all shared outputs
@@ -743,8 +743,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
         for (Map.Entry<String, OutputCommitter> entry : outputCommitters
             .entrySet()) {
           final OutputCommitter committer = entry.getValue();
-          if (abortAllOutputsOnFailure // abort all outputs on failure
-              || vertex.getState() != VertexState.SUCCEEDED // always abort non-successful
outputs
+          if (commitAllOutputsOnSuccess // commit all outputs on success
+              || vertex.getState() != VertexState.SUCCEEDED // never commit unsuccessful
outputs
               ) {
             LOG.info("Aborting output: " + entry.getKey() + " for vertex: "
                 + vertex.getVertexId());
@@ -1067,9 +1067,9 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
       //dag.metrics.preparingJob(dag);
 
       dag.initTime = dag.clock.getTime();
-      dag.abortAllOutputsOnFailure = dag.conf.getBoolean(
-          TezConfiguration.TEZ_AM_ABORT_ALL_OUTPUTS_ON_DAG_FAILURE,
-          TezConfiguration.TEZ_AM_ABORT_ALL_OUTPUTS_ON_DAG_FAILURE_DEFAULT);
+      dag.commitAllOutputsOnSuccess = dag.conf.getBoolean(
+          TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS,
+          TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS_DEFAULT);
 
       // If we have no vertices, fail the dag
       dag.numVertices = dag.getJobPlan().getVertexCount();
@@ -1179,7 +1179,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
           vertexId, vertexPlan, vertexName, dag.conf,
           dag.eventHandler, dag.taskAttemptListener, 
           dag.clock, dag.taskHeartbeatHandler,
-          !dag.abortAllOutputsOnFailure, dag.appContext, vertexLocationHint,
+          !dag.commitAllOutputsOnSuccess, dag.appContext, vertexLocationHint,
           dag.vertexGroups);
       return v;
     }
@@ -1402,7 +1402,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
   private boolean vertexSucceeded(Vertex vertex) {
     numSuccessfulVertices++;
     boolean failedCommit = false;
-    if (!abortAllOutputsOnFailure) {
+    if (!commitAllOutputsOnSuccess) {
       // committing successful outputs immediately. check for shared outputs
       List<VertexGroupInfo> groupsList = vertexGroupInfo.get(vertex.getName());
       if (groupsList != null) {
@@ -1453,7 +1453,7 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
       + ", vertexName=" + vertex.getName()
       + ", vertexId=" + vertex.getVertexId());
     
-    if (!abortAllOutputsOnFailure) {
+    if (!commitAllOutputsOnSuccess) {
       // partial output may already have been committed. fail if so
       List<VertexGroupInfo> groupList = vertexGroupInfo.get(vertex.getName());
       if (groupList != null) {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/2c534399/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index 1d8274b..4e02f59 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -896,7 +896,7 @@ public class TestDAGImpl {
   @Test(timeout=5000)
   public void testDAGErrorAbortNonSuccessfulOutputs() {
     // vertex success -> vertex output commit. failed dag aborts only non-successful vertices
-    conf.setBoolean(TezConfiguration.TEZ_AM_ABORT_ALL_OUTPUTS_ON_DAG_FAILURE, false);
+    conf.setBoolean(TezConfiguration.TEZ_AM_COMMIT_ALL_OUTPUTS_ON_DAG_SUCCESS, false);
     initDAG(mrrDag);
     dispatcher.await();
     startDAG(mrrDag);


Mime
View raw message