crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject crunch git commit: CRUNCH-638: Improve dot file generation for better supportability. Contributed by Gergő Pásztor.
Date Tue, 07 Mar 2017 14:41:25 GMT
Repository: crunch
Updated Branches:
  refs/heads/master e3ba6fcd0 -> e520d9f6e


CRUNCH-638: Improve dot file generation for better supportability. Contributed by Gergő Pásztor.


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/e520d9f6
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/e520d9f6
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/e520d9f6

Branch: refs/heads/master
Commit: e520d9f6e55bdcd16e5ad9715f46ddce4ce96f76
Parents: e3ba6fc
Author: Tom White <tomwhite@apache.org>
Authored: Tue Mar 7 14:38:52 2017 +0000
Committer: Tom White <tomwhite@apache.org>
Committed: Tue Mar 7 14:38:52 2017 +0000

----------------------------------------------------------------------
 .../apache/crunch/impl/mr/plan/DotfilesIT.java  |  3 +-
 .../apache/crunch/impl/mr/plan/DotfileUtil.java | 30 ++++++++++++++++----
 .../apache/crunch/impl/mr/plan/MSCRPlanner.java |  5 ++--
 3 files changed, 29 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/e520d9f6/crunch-core/src/it/java/org/apache/crunch/impl/mr/plan/DotfilesIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/impl/mr/plan/DotfilesIT.java b/crunch-core/src/it/java/org/apache/crunch/impl/mr/plan/DotfilesIT.java
index c33348a..99e25b7 100644
--- a/crunch-core/src/it/java/org/apache/crunch/impl/mr/plan/DotfilesIT.java
+++ b/crunch-core/src/it/java/org/apache/crunch/impl/mr/plan/DotfilesIT.java
@@ -116,10 +116,11 @@ public class DotfilesIT {
 
     String[] dotfileNames = dotfileNames(dotfileDir.getRootFile());
 
-    assertEquals(5, dotfileNames.length);
+    assertEquals(6, dotfileNames.length);
 
     assertTrue(containsFileEndingWith(dotfileNames, "jobplan.dot"));
     assertTrue(containsFileEndingWith(dotfileNames, "split_graph_plan.dot"));
+    assertTrue(containsFileEndingWith(dotfileNames, "split_graph_with_components_plan.dot"));
     assertTrue(containsFileEndingWith(dotfileNames, "rt_plan.dot"));
     assertTrue(containsFileEndingWith(dotfileNames, "base_graph_plan.dot"));
     assertTrue(containsFileEndingWith(dotfileNames, "lineage_plan.dot"));

http://git-wip-us.apache.org/repos/asf/crunch/blob/e520d9f6/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DotfileUtil.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DotfileUtil.java b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DotfileUtil.java
index 6475e87..ee854d9 100644
--- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DotfileUtil.java
+++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DotfileUtil.java
@@ -47,6 +47,7 @@ public class DotfileUtil {
   private String rtNodesDotfile = "";
   private String basePlanGraphDotfile = "";
   private String splitGraphPlanDotfile = "";
+  private String splitGraphWithComponentsPlanDotfile = "";
   private String pcollectionLineageDotfile = "";
   private String planDotFile = "";
 
@@ -75,8 +76,22 @@ public class DotfileUtil {
   void buildBaseGraphDotfile(Map<PCollectionImpl<?>, Set<Target>> outputs,
Graph graph) {
     if (isDebugDotfilesEnabled(conf)) {
       try {
-        basePlanGraphDotfile = new DotfileWriterGraph(graph, outputs, null).buildDiagram("Base
Graph ("
-            + jarClass.getSimpleName() + ")");
+        basePlanGraphDotfile = new DotfileWriterGraph(graph, outputs, null)
+                .buildDiagram("Base Graph (" + jarClass.getSimpleName() + ")");
+      } catch (Exception ex) {
+        LOG.error("Problem creating debug dotfile:", ex);
+      }
+    }
+  }
+
+  /**
+   * Builds the split graph dotfile only if the dotfile-debug mode is enabled.
+   */
+  void buildSplitGraphDotfile(Map<PCollectionImpl<?>, Set<Target>> outputs,
Graph graph) {
+    if (isDebugDotfilesEnabled(conf)) {
+      try {
+        splitGraphPlanDotfile = new DotfileWriterGraph(graph, outputs, null)
+                .buildDiagram("Split Graph (" + jarClass.getSimpleName() + ")");
       } catch (Exception ex) {
         LOG.error("Problem creating debug dotfile:", ex);
       }
@@ -84,13 +99,15 @@ public class DotfileUtil {
   }
 
   /**
-   * Builds a split graph dotfile only if the dotfile-debug mode is enabled.
+   * Builds a split graph with components dotfile only if the dotfile-debug mode is enabled.
    */
-  void buildSplitGraphDotfile(Map<PCollectionImpl<?>, Set<Target>> outputs,
Graph graph, List<List<Vertex>> components) {
+  void buildSplitGraphWithComponentsDotfile(
+          Map<PCollectionImpl<?>, Set<Target>> outputs, Graph graph, List<List<Vertex>>
components
+  ) {
     if (isDebugDotfilesEnabled(conf)) {
       try {
-        splitGraphPlanDotfile = new DotfileWriterGraph(graph, outputs, components)
-            .buildDiagram("Graph With Components (" + jarClass.getSimpleName() + ")");
+        splitGraphWithComponentsPlanDotfile = new DotfileWriterGraph(graph, outputs, components)
+            .buildDiagram("Split Graph With Components (" + jarClass.getSimpleName() + ")");
       } catch (Exception ex) {
         LOG.error("Problem creating debug dotfile:", ex);
       }
@@ -145,6 +162,7 @@ public class DotfileUtil {
         exec.addNamedDotFile("rt_plan", rtNodesDotfile);
         exec.addNamedDotFile("base_graph_plan", basePlanGraphDotfile);
         exec.addNamedDotFile("split_graph_plan", splitGraphPlanDotfile);
+        exec.addNamedDotFile("split_graph_with_components_plan", splitGraphWithComponentsPlanDotfile);
         exec.addNamedDotFile("lineage_plan", pcollectionLineageDotfile);
       }
     } catch (Exception ex) {

http://git-wip-us.apache.org/repos/asf/crunch/blob/e520d9f6/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/MSCRPlanner.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/MSCRPlanner.java b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/MSCRPlanner.java
index eace851..5113a2b 100644
--- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/MSCRPlanner.java
+++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/MSCRPlanner.java
@@ -137,8 +137,9 @@ public class MSCRPlanner {
       List<List<Vertex>> components = graph.connectedComponents();
 
       // Generate the debug graph dotfiles (if configuration is enabled)
-      dotfileUtil.buildBaseGraphDotfile(outputs, graph);
-      dotfileUtil.buildSplitGraphDotfile(outputs, graph, components);
+      dotfileUtil.buildBaseGraphDotfile(outputs, baseGraph);
+      dotfileUtil.buildSplitGraphDotfile(outputs, graph);
+      dotfileUtil.buildSplitGraphWithComponentsDotfile(outputs, graph, components);
 
       // For each component, we will create one or more job prototypes,
       // depending on its profile.


Mime
View raw message