giraph-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ikabi...@apache.org
Subject git commit: updated refs/heads/trunk to b9d20ed
Date Tue, 16 Jun 2015 18:09:39 GMT
Repository: giraph
Updated Branches:
  refs/heads/trunk 6dd7d1425 -> b9d20edec


[GIRAPH-1013] Cleanup use of conf for local testing

Summary:
Right now we are creating two immutable conf options, and using them both,
which is unnecessary and confusing.

Change to do it only once, and not need to pass it around (TestGraph has it)

Test Plan: mvn clean install

Reviewers: laxman.dhulipala, maja.kabiljo, dionysis.logothetis, sergey.edunov

Differential Revision: https://reviews.facebook.net/D39987


Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/b9d20ede
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/b9d20ede
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/b9d20ede

Branch: refs/heads/trunk
Commit: b9d20edece212b3898fd61f0e6799c2ebbf99986
Parents: 6dd7d14
Author: Igor Kabiljo <ikabiljo@fb.com>
Authored: Thu Jun 11 16:17:53 2015 -0700
Committer: Igor Kabiljo <ikabiljo@fb.com>
Committed: Tue Jun 16 11:09:03 2015 -0700

----------------------------------------------------------------------
 .../giraph/block_app/framework/BlockUtils.java  | 21 ++++------
 .../framework/api/local/LocalBlockRunner.java   | 42 +++++++++-----------
 .../block_app/framework/BlockExecutionTest.java |  4 +-
 .../java/org/apache/giraph/utils/TestGraph.java |  4 ++
 4 files changed, 33 insertions(+), 38 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/giraph/blob/b9d20ede/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/BlockUtils.java
----------------------------------------------------------------------
diff --git a/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/BlockUtils.java
b/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/BlockUtils.java
index 3175a55..e00909c 100644
--- a/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/BlockUtils.java
+++ b/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/BlockUtils.java
@@ -138,8 +138,7 @@ public class BlockUtils {
     // They will not be used here
     Block executionBlock = blockFactory.createBlock(immConf);
     checkBlockTypes(
-        executionBlock, blockFactory.createExecutionStage(immConf),
-        conf, immConf);
+        executionBlock, blockFactory.createExecutionStage(immConf), immConf);
 
     // check for non 'static final' fields in BlockFactories
     Class<?> bfClass = blockFactory.getClass();
@@ -163,19 +162,15 @@ public class BlockUtils {
 
   public static void checkBlockTypes(
       Block executionBlock, Object executionStage,
-      GiraphConfiguration conf,
-      final ImmutableClassesGiraphConfiguration immConf) {
+      final ImmutableClassesGiraphConfiguration conf) {
     LOG.info("Executing application - " + executionBlock);
 
-    final Class<?> vertexIdClass = GiraphConstants.VERTEX_ID_CLASS.get(conf);
-    final Class<?> vertexValueClass =
-        GiraphConstants.VERTEX_VALUE_CLASS.get(conf);
-    final Class<?> edgeValueClass =
-        GiraphConstants.EDGE_VALUE_CLASS.get(conf);
+    final Class<?> vertexIdClass = conf.getVertexIdClass();
+    final Class<?> vertexValueClass = conf.getVertexValueClass();
+    final Class<?> edgeValueClass = conf.getEdgeValueClass();
     final Class<?> workerContextValueClass =
-        BlockUtils.BLOCK_WORKER_CONTEXT_VALUE_CLASS.get(conf);
-    final Class<?> executionStageClass =
-        executionStage.getClass();
+        BLOCK_WORKER_CONTEXT_VALUE_CLASS.get(conf);
+    final Class<?> executionStageClass = executionStage.getClass();
 
     // Check for type inconsistencies
     executionBlock.forAllPossiblePieces(new Consumer<AbstractPiece>() {
@@ -193,7 +188,7 @@ public class BlockUtils {
           ReflectionUtils.verifyTypes(
               edgeValueClass, classList[2], "edgeValue", piece.getClass());
 
-          MessageClasses classes = piece.getMessageClasses(immConf);
+          MessageClasses classes = piece.getMessageClasses(conf);
           Class<?> messageType = classes.getMessageClass();
           if (messageType == null) {
             messageType = NoMessage.class;

http://git-wip-us.apache.org/repos/asf/giraph/blob/b9d20ede/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/api/local/LocalBlockRunner.java
----------------------------------------------------------------------
diff --git a/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/api/local/LocalBlockRunner.java
b/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/api/local/LocalBlockRunner.java
index ea6817f..ad6ca15 100644
--- a/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/api/local/LocalBlockRunner.java
+++ b/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/api/local/LocalBlockRunner.java
@@ -37,7 +37,6 @@ import org.apache.giraph.block_app.framework.internal.BlockWorkerLogic;
 import org.apache.giraph.block_app.framework.internal.BlockWorkerPieces;
 import org.apache.giraph.block_app.framework.output.BlockOutputHandle;
 import org.apache.giraph.conf.BooleanConfOption;
-import org.apache.giraph.conf.GiraphConfiguration;
 import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration;
 import org.apache.giraph.conf.IntConfOption;
 import org.apache.giraph.graph.Vertex;
@@ -86,12 +85,11 @@ public class LocalBlockRunner {
   public static
   <I extends WritableComparable, V extends Writable, E extends Writable>
   TestGraph<I, V, E> runApp(
-      TestGraph<I, V, E> graph, GiraphConfiguration conf,
-      boolean useFullDigraphTests) throws Exception {
+      TestGraph<I, V, E> graph, boolean useFullDigraphTests) throws Exception {
     if (useFullDigraphTests) {
-      return InternalVertexRunner.runWithInMemoryOutput(conf, graph);
+      return InternalVertexRunner.runWithInMemoryOutput(graph.getConf(), graph);
     } else {
-      runApp(graph, conf);
+      runApp(graph);
       return graph;
     }
   }
@@ -102,9 +100,9 @@ public class LocalBlockRunner {
    */
   public static
   <I extends WritableComparable, V extends Writable, E extends Writable>
-  void runApp(TestGraph<I, V, E> graph, GiraphConfiguration conf) {
+  void runApp(TestGraph<I, V, E> graph) {
     VertexSaver<I, V, E> noOpVertexSaver = noOpVertexSaver();
-    runAppWithVertexOutput(graph, noOpVertexSaver, conf);
+    runAppWithVertexOutput(graph, noOpVertexSaver);
   }
 
   /**
@@ -114,11 +112,10 @@ public class LocalBlockRunner {
   public static
   <I extends WritableComparable, V extends Writable, E extends Writable>
   void runBlock(
-      TestGraph<I, V, E> graph, Block block, Object executionStage,
-      GiraphConfiguration conf) {
+      TestGraph<I, V, E> graph, Block block, Object executionStage) {
     VertexSaver<I, V, E> noOpVertexSaver = noOpVertexSaver();
     runBlockWithVertexOutput(
-        block, executionStage, graph, noOpVertexSaver, conf);
+        block, executionStage, graph, noOpVertexSaver);
   }
 
 
@@ -129,12 +126,12 @@ public class LocalBlockRunner {
   public static
   <I extends WritableComparable, V extends Writable, E extends Writable>
   void runAppWithVertexOutput(
-      TestGraph<I, V, E> graph, final VertexSaver<I, V, E> vertexSaver,
-      GiraphConfiguration conf) {
-    BlockFactory<?> factory = BlockUtils.createBlockFactory(conf);
+      TestGraph<I, V, E> graph, final VertexSaver<I, V, E> vertexSaver) {
+    BlockFactory<?> factory = BlockUtils.createBlockFactory(graph.getConf());
     runBlockWithVertexOutput(
-        factory.createBlock(conf), factory.createExecutionStage(conf),
-        graph, vertexSaver, conf);
+        factory.createBlock(graph.getConf()),
+        factory.createExecutionStage(graph.getConf()),
+        graph, vertexSaver);
   }
 
   /**
@@ -145,20 +142,19 @@ public class LocalBlockRunner {
   <I extends WritableComparable, V extends Writable, E extends Writable>
   void runBlockWithVertexOutput(
       Block block, Object executionStage, TestGraph<I, V, E> graph,
-      final VertexSaver<I, V, E> vertexSaver, GiraphConfiguration conf
+      final VertexSaver<I, V, E> vertexSaver
   ) {
+    ImmutableClassesGiraphConfiguration<I, V, E> conf = graph.getConf();
     int numWorkers = NUM_THREADS.get(conf);
     boolean runAllChecks = RUN_ALL_CHECKS.get(conf);
     boolean serializeMaster = SERIALIZE_MASTER.get(conf);
     final boolean doOutputDuringComputation = conf.doOutputDuringComputation();
 
-    ImmutableClassesGiraphConfiguration<I, V, E> immConf =
-        new ImmutableClassesGiraphConfiguration(conf);
     final InternalApi internalApi =
-        new InternalApi(graph, immConf, runAllChecks);
+        new InternalApi(graph, conf, runAllChecks);
     final InternalWorkerApi internalWorkerApi = internalApi.getWorkerApi();
 
-    BlockUtils.checkBlockTypes(block, executionStage, conf, immConf);
+    BlockUtils.checkBlockTypes(block, executionStage, conf);
 
     BlockMasterLogic<Object> blockMasterLogic = new BlockMasterLogic<>();
     blockMasterLogic.initialize(block, executionStage, internalApi);
@@ -177,12 +173,12 @@ public class LocalBlockRunner {
 
     if (runAllChecks) {
       for (Vertex<I, V, E> vertex : graph) {
-        V value = immConf.createVertexValue();
+        V value = conf.createVertexValue();
         WritableUtils.copyInto(vertex.getValue(), value);
         vertex.setValue(value);
 
         vertex.setEdges((Iterable) WritableUtils.createCopy(
-            (Writable) vertex.getEdges(), immConf.getOutEdgesClass(), immConf));
+            (Writable) vertex.getEdges(), conf.getOutEdgesClass(), conf));
       }
     }
 
@@ -194,7 +190,7 @@ public class LocalBlockRunner {
         blockMasterLogic = (BlockMasterLogic) WritableUtils.createCopy(
             new KryoWritableWrapper<>(blockMasterLogic),
             KryoWritableWrapper.class,
-            immConf).get();
+            conf).get();
         blockMasterLogic.initializeAfterRead(internalApi);
       }
 

http://git-wip-us.apache.org/repos/asf/giraph/blob/b9d20ede/giraph-block-app/src/test/java/org/apache/giraph/block_app/framework/BlockExecutionTest.java
----------------------------------------------------------------------
diff --git a/giraph-block-app/src/test/java/org/apache/giraph/block_app/framework/BlockExecutionTest.java
b/giraph-block-app/src/test/java/org/apache/giraph/block_app/framework/BlockExecutionTest.java
index b77c797..170f307 100644
--- a/giraph-block-app/src/test/java/org/apache/giraph/block_app/framework/BlockExecutionTest.java
+++ b/giraph-block-app/src/test/java/org/apache/giraph/block_app/framework/BlockExecutionTest.java
@@ -109,7 +109,7 @@ public class BlockExecutionTest {
       protected Class<BooleanWritable> getMessageClass() {
         return BooleanWritable.class;
       }
-    }, new Object(), conf);
+    }, new Object());
 
     Assert.assertEquals(1, graph.getVertex(new LongWritable(1)).getValue().get());
     Assert.assertEquals(2, graph.getVertex(new LongWritable(2)).getValue().get());
@@ -149,7 +149,7 @@ public class BlockExecutionTest {
       public void masterCompute(BlockMasterApi masterApi, Object executionStage) {
         value.set(numVertices.getReducedValue(masterApi).get());
       }
-    }, new Object(), conf);
+    }, new Object());
 
     Assert.assertEquals(4, value.get());
   }

http://git-wip-us.apache.org/repos/asf/giraph/blob/b9d20ede/giraph-core/src/main/java/org/apache/giraph/utils/TestGraph.java
----------------------------------------------------------------------
diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/TestGraph.java b/giraph-core/src/main/java/org/apache/giraph/utils/TestGraph.java
index 7a37167..6bdd4b9 100644
--- a/giraph-core/src/main/java/org/apache/giraph/utils/TestGraph.java
+++ b/giraph-core/src/main/java/org/apache/giraph/utils/TestGraph.java
@@ -64,6 +64,10 @@ public class TestGraph<I extends WritableComparable,
     return vertices;
   }
 
+  public ImmutableClassesGiraphConfiguration<I, V, E> getConf() {
+    return conf;
+  }
+
   /**
    * Clear all data
    *


Mime
View raw message