tinkerpop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ok...@apache.org
Subject [08/13] incubator-tinkerpop git commit: GroovyProcessCompiuterSuite was missing GroovyFlatMapTest. Added it. Added HadoopPool registration to ToyGraphInputRDD so it doesn't give a WARN message. Also I tweaked BulkLoaderVertexProgramTest to use target/tes
Date Thu, 03 Dec 2015 19:49:27 GMT
GroovyProcessCompiuterSuite was missing GroovyFlatMapTest. Added it. Added HadoopPool registration
to ToyGraphInputRDD so it doesn't give a WARN message. Also I tweaked BulkLoaderVertexProgramTest
to use target/test-output/ for its intermediary data.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/f8209353
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/f8209353
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/f8209353

Branch: refs/heads/master
Commit: f820935303799b8dd661e4fa3686a24a36baad67
Parents: 26dcf3c
Author: Marko A. Rodriguez <okrammarko@gmail.com>
Authored: Wed Dec 2 15:44:50 2015 -0700
Committer: Marko A. Rodriguez <okrammarko@gmail.com>
Committed: Thu Dec 3 12:47:53 2015 -0700

----------------------------------------------------------------------
 .../process/GroovyProcessComputerSuite.java     |  2 ++
 .../BulkLoaderVertexProgramTest.java            |  2 +-
 .../computer/SparkHadoopGraphProvider.java      | 16 +++++----
 .../spark/structure/io/ToyGraphInputRDD.java    | 38 +++++++++++---------
 4 files changed, 33 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f8209353/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
index 9df7902..d07d52f 100644
--- a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
+++ b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
@@ -47,6 +47,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyAddEdgeTest
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCoalesceTest;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyConstantTest;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCountTest;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyFlatMapTest;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyFoldTest;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyGraphTest;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyLoopsTest;
@@ -128,6 +129,7 @@ public class GroovyProcessComputerSuite extends ProcessComputerSuite {
             GroovyCoalesceTest.Traversals.class,
             GroovyConstantTest.Traversals.class,
             GroovyCountTest.Traversals.class,
+            GroovyFlatMapTest.Traversals.class,
             GroovyFoldTest.Traversals.class,
             GroovyGraphTest.Traversals.class,
             GroovyLoopsTest.Traversals.class,

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f8209353/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
index 61a267f..992bfa4 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
@@ -52,7 +52,7 @@ import static org.junit.Assert.assertTrue;
  */
 public class BulkLoaderVertexProgramTest extends AbstractGremlinProcessTest {
 
-    final static String TINKERGRAPH_LOCATION = "/tmp/tinkertest.kryo";
+    final static String TINKERGRAPH_LOCATION = "target/test-output/tinkertest.kryo";
 
     private BulkLoader getBulkLoader(final BulkLoaderVertexProgram blvp) throws Exception
{
         final Field field = BulkLoaderVertexProgram.class.getDeclaredField("bulkLoader");

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f8209353/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index 2cfeea3..de5a60d 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -22,6 +22,8 @@ import org.apache.tinkerpop.gremlin.GraphProvider;
 import org.apache.tinkerpop.gremlin.LoadGraphWith;
 import org.apache.tinkerpop.gremlin.hadoop.Constants;
 import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.process.computer.bulkloading.BulkLoaderVertexProgramTest;
+import org.apache.tinkerpop.gremlin.process.computer.ranking.PageRankVertexProgramTest;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
 import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
 import org.apache.tinkerpop.gremlin.spark.structure.io.InputRDDFormat;
@@ -43,13 +45,13 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider
{
     @Override
     public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?>
test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
         final Map<String, Object> config = super.getBaseConfiguration(graphName, test,
testMethodName, loadGraphWith);
-        if (null != loadGraphWith) {
-            if (RANDOM.nextBoolean()) {
-                config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
-                config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ToyGraphInputRDD.class.getCanonicalName());
-                config.put(ToyGraphInputRDD.GREMLIN_SPARK_TOY_GRAPH, loadGraphWith.toString());
-                config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
-            }
+        config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);  // this makes the test
suite go really fast
+        if (null != loadGraphWith &&
+                !test.equals(BulkLoaderVertexProgramTest.class) &&
+                !test.equals(PageRankVertexProgramTest.class) &&
+                RANDOM.nextBoolean()) {
+            config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ToyGraphInputRDD.class.getCanonicalName());
+            config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
         }
         /// spark configuration
         config.put("spark.master", "local[4]");

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f8209353/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
index ed97c04..ea3636f 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
@@ -22,11 +22,13 @@ package org.apache.tinkerpop.gremlin.spark.structure.io;
 import org.apache.commons.configuration.Configuration;
 import org.apache.spark.api.java.JavaPairRDD;
 import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
 import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
 import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
 import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
 import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
 import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
 import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
@@ -34,36 +36,38 @@ import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
 import scala.Tuple2;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.util.List;
-import java.util.stream.Collectors;
 
 /**
  * @author Marko A. Rodriguez (http://markorodriguez.com)
  */
 public final class ToyGraphInputRDD implements InputRDD {
 
-    public static final String GREMLIN_SPARK_TOY_GRAPH = "gremlin.spark.toyGraph";
-
     @Override
     public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration,
final JavaSparkContext sparkContext) {
-        final List<Vertex> vertices;
-        if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.MODERN.toString()))
-            vertices = IteratorUtils.list(TinkerFactory.createModern().vertices());
-        else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.CLASSIC.toString()))
-            vertices = IteratorUtils.list(TinkerFactory.createClassic().vertices());
-        else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.CREW.toString()))
-            vertices = IteratorUtils.list(TinkerFactory.createTheCrew().vertices());
-        else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.GRATEFUL.toString()))
{
+        HadoopPools.initialize(TinkerGraph.open().configuration());
+        final List<VertexWritable> vertices;
+        if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("modern"))
+            vertices = IteratorUtils.list(IteratorUtils.map(TinkerFactory.createModern().vertices(),
VertexWritable::new));
+        else if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("classic"))
+            vertices = IteratorUtils.list(IteratorUtils.map(TinkerFactory.createClassic().vertices(),
VertexWritable::new));
+        else if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("crew"))
+            vertices = IteratorUtils.list(IteratorUtils.map(TinkerFactory.createTheCrew().vertices(),
VertexWritable::new));
+        else if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("grateful"))
{
             try {
                 final Graph graph = TinkerGraph.open();
-                graph.io(GryoIo.build()).readGraph(GryoResourceAccess.class.getResource("grateful-dead.kryo").getFile());
-                vertices = IteratorUtils.list(graph.vertices());
+                final GraphReader reader = GryoReader.build().mapper(graph.io(GryoIo.build()).mapper().create()).create();
+                try (final InputStream stream = GryoResourceAccess.class.getResourceAsStream("grateful-dead.kryo"))
{
+                    reader.readGraph(stream, graph);
+                }
+                vertices = IteratorUtils.list(IteratorUtils.map(graph.vertices(), VertexWritable::new));
             } catch (final IOException e) {
                 throw new IllegalStateException(e.getMessage(), e);
             }
         } else
-            throw new IllegalArgumentException("No legal toy graph was provided to load:
" + configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH));
+            throw new IllegalArgumentException("No legal toy graph was provided to load:
" + configuration.getProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION));
 
-        return sparkContext.parallelize(vertices.stream().map(VertexWritable::new).collect(Collectors.toList())).mapToPair(vertex
-> new Tuple2<>(vertex.get().id(), vertex));
+        return sparkContext.parallelize(vertices).mapToPair(vertex -> new Tuple2<>(vertex.get().id(),
vertex));
     }
 }


Mime
View raw message