tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bi...@apache.org
Subject git commit: TEZ-221. Deadlock in DAGSchedulerMRR when setting slow start to 0.5f on a small cluster (bikas)
Date Wed, 20 Nov 2013 01:58:55 GMT
Updated Branches:
  refs/heads/master 207264823 -> 808be4d04


TEZ-221. Deadlock in DAGSchedulerMRR when setting slow start to 0.5f on a small cluster (bikas)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/808be4d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/808be4d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/808be4d0

Branch: refs/heads/master
Commit: 808be4d0476eea14114096953dd46b06a8d74233
Parents: 2072648
Author: Bikas Saha <bikas@apache.org>
Authored: Tue Nov 19 17:55:25 2013 -0800
Committer: Bikas Saha <bikas@apache.org>
Committed: Tue Nov 19 17:55:25 2013 -0800

----------------------------------------------------------------------
 .../apache/tez/dag/api/TezConfiguration.java    |  4 --
 .../apache/tez/dag/app/dag/impl/DAGImpl.java    | 43 +-------------------
 .../dag/app/dag/impl/ShuffleVertexManager.java  |  7 ----
 .../tez/dag/app/dag/impl/TestDAGImpl.java       |  1 +
 4 files changed, 3 insertions(+), 52 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/808be4d0/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index 6392ded..c41a9ea 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -136,10 +136,6 @@ public class TezConfiguration extends Configuration {
   public static final float
           TEZ_AM_SLOWSTART_DAG_SCHEDULER_MIN_SHUFFLE_RESOURCE_FRACTION_DEFAULT = 0.5f;
 
-  public static final String TEZ_AM_AGGRESSIVE_SCHEDULING = TEZ_AM_PREFIX +
-      "aggressive.scheduling";
-  public static boolean TEZ_AM_AGGRESSIVE_SCHEDULING_DEFAULT = false;
-
   /**
    * The complete path to the serialized dag plan file
    * <code>TEZ_AM_PLAN_PB_BINARY</code>. Used to make the plan available to

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/808be4d0/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index ee6b832..9fb81e0 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -50,7 +50,6 @@ import org.apache.hadoop.yarn.util.Clock;
 import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.DagTypeConverters;
 import org.apache.tez.dag.api.EdgeProperty;
-import org.apache.tez.dag.api.TezConfiguration;
 import org.apache.tez.dag.api.TezUncheckedException;
 import org.apache.tez.dag.api.VertexLocationHint;
 import org.apache.tez.dag.api.client.DAGStatus;
@@ -881,46 +880,8 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
     }
 
     private void assignDAGScheduler(DAGImpl dag) {
-      if (dag.conf.getBoolean(TezConfiguration.TEZ_AM_AGGRESSIVE_SCHEDULING,
-          TezConfiguration.TEZ_AM_AGGRESSIVE_SCHEDULING_DEFAULT)) {
-        LOG.info("Using Natural order dag scheduler due to aggressive scheduling");
-        dag.dagScheduler = new DAGSchedulerNaturalOrder(dag, dag.eventHandler);
-      } else {
-        boolean isMRR = true;
-        for (Vertex vertex : dag.vertices.values()) {
-          Map<Vertex, Edge> outVertices = vertex.getOutputVertices();
-          Map<Vertex, Edge> inVertices = vertex.getInputVertices();
-          if (!(outVertices == null || outVertices.isEmpty() || (outVertices
-              .size() == 1 && outVertices.values().iterator().next().getEdgeProperty()
-              .getDataMovementType() == EdgeProperty.DataMovementType.SCATTER_GATHER))) {
-            // more than 1 output OR single output is not bipartite
-            isMRR = false;
-            break;
-          }
-          if (!(inVertices == null || inVertices.isEmpty() || (inVertices
-              .size() == 1 && inVertices.values().iterator().next().getEdgeProperty()
-              .getDataMovementType() == EdgeProperty.DataMovementType.SCATTER_GATHER))) {
-            // more than 1 output OR single output is not bipartite
-            isMRR = false;
-            break;
-          }
-        }
-
-        if (isMRR) {
-          LOG.info("Using MRR dag scheduler");
-          dag.dagScheduler = new DAGSchedulerMRR(
-              dag,
-              dag.eventHandler,
-              dag.appContext.getTaskScheduler(),
-              dag.conf
-                  .getFloat(
-                      TezConfiguration.TEZ_AM_SLOWSTART_DAG_SCHEDULER_MIN_SHUFFLE_RESOURCE_FRACTION,
-                      TezConfiguration.TEZ_AM_SLOWSTART_DAG_SCHEDULER_MIN_SHUFFLE_RESOURCE_FRACTION_DEFAULT));
-        } else {
-          LOG.info("Using Natural order dag scheduler");
-          dag.dagScheduler = new DAGSchedulerNaturalOrder(dag, dag.eventHandler);
-        }
-      }
+      LOG.info("Using Natural order dag scheduler");
+      dag.dagScheduler = new DAGSchedulerNaturalOrder(dag, dag.eventHandler);
     }
 
     private VertexImpl createVertex(DAGImpl dag, String vertexName, int vId) {

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/808be4d0/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ShuffleVertexManager.java
b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ShuffleVertexManager.java
index 5143868..eda2dd5 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ShuffleVertexManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/ShuffleVertexManager.java
@@ -414,13 +414,6 @@ public class ShuffleVertexManager implements VertexScheduler {
           "max cannot be < min.");
     }
     
-    if (conf.getBoolean(TezConfiguration.TEZ_AM_AGGRESSIVE_SCHEDULING,
-        TezConfiguration.TEZ_AM_AGGRESSIVE_SCHEDULING_DEFAULT)) {
-      LOG.info("Setting min/max threshold to 0 due to aggressive scheduling");
-      this.slowStartMinSrcCompletionFraction = 0;
-      this.slowStartMaxSrcCompletionFraction = 0;
-    }
-    
     enableAutoParallelism = conf
         .getBoolean(
             TezConfiguration.TEZ_AM_SHUFFLE_VERTEX_MANAGER_ENABLE_AUTO_PARALLEL,

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/808be4d0/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index 55b846d..b0ad929 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -844,6 +844,7 @@ public class TestDAGImpl {
     // FIXME need to implement
   }
 
+  @Ignore
   @SuppressWarnings("unchecked")
   @Test
   public void testOutofBandFailureForMRRScheduler() {


Mime
View raw message