Repository: incubator-tinkerpop
Updated Branches:
refs/heads/TINKERPOP3-1011 e20ff9199 -> 91efb28df
GroovyProcessCompiuterSuite was missing GroovyFlatMapTest. Added it. Added HadoopPool registration
to ToyGraphInputRDD so it doesn't give a WARN message. Also I tweaked BulkLoaderVertexProgramTest
to use target/test-output/ for its intermediary data.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/91efb28d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/91efb28d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/91efb28d
Branch: refs/heads/TINKERPOP3-1011
Commit: 91efb28df23fdc0dc99c78bd72159f0478614df1
Parents: e20ff91
Author: Marko A. Rodriguez <okrammarko@gmail.com>
Authored: Wed Dec 2 15:44:50 2015 -0700
Committer: Marko A. Rodriguez <okrammarko@gmail.com>
Committed: Wed Dec 2 15:44:50 2015 -0700
----------------------------------------------------------------------
.../process/GroovyProcessComputerSuite.java | 2 ++
.../BulkLoaderVertexProgramTest.java | 2 +-
.../computer/SparkHadoopGraphProvider.java | 16 +++++----
.../spark/structure/io/ToyGraphInputRDD.java | 38 +++++++++++---------
4 files changed, 33 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/91efb28d/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
index 9df7902..d07d52f 100644
--- a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
+++ b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
@@ -47,6 +47,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyAddEdgeTest
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCoalesceTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyConstantTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCountTest;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyFlatMapTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyFoldTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyGraphTest;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyLoopsTest;
@@ -128,6 +129,7 @@ public class GroovyProcessComputerSuite extends ProcessComputerSuite {
GroovyCoalesceTest.Traversals.class,
GroovyConstantTest.Traversals.class,
GroovyCountTest.Traversals.class,
+ GroovyFlatMapTest.Traversals.class,
GroovyFoldTest.Traversals.class,
GroovyGraphTest.Traversals.class,
GroovyLoopsTest.Traversals.class,
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/91efb28d/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
index 61a267f..992bfa4 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java
@@ -52,7 +52,7 @@ import static org.junit.Assert.assertTrue;
*/
public class BulkLoaderVertexProgramTest extends AbstractGremlinProcessTest {
- final static String TINKERGRAPH_LOCATION = "/tmp/tinkertest.kryo";
+ final static String TINKERGRAPH_LOCATION = "target/test-output/tinkertest.kryo";
private BulkLoader getBulkLoader(final BulkLoaderVertexProgram blvp) throws Exception
{
final Field field = BulkLoaderVertexProgram.class.getDeclaredField("bulkLoader");
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/91efb28d/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
index 2cfeea3..de5a60d 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java
@@ -22,6 +22,8 @@ import org.apache.tinkerpop.gremlin.GraphProvider;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider;
+import org.apache.tinkerpop.gremlin.process.computer.bulkloading.BulkLoaderVertexProgramTest;
+import org.apache.tinkerpop.gremlin.process.computer.ranking.PageRankVertexProgramTest;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine;
import org.apache.tinkerpop.gremlin.spark.structure.io.InputRDDFormat;
@@ -43,13 +45,13 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider
{
@Override
public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?>
test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) {
final Map<String, Object> config = super.getBaseConfiguration(graphName, test,
testMethodName, loadGraphWith);
- if (null != loadGraphWith) {
- if (RANDOM.nextBoolean()) {
- config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION);
- config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ToyGraphInputRDD.class.getCanonicalName());
- config.put(ToyGraphInputRDD.GREMLIN_SPARK_TOY_GRAPH, loadGraphWith.toString());
- config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
- }
+ config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); // this makes the test
suite go really fast
+ if (null != loadGraphWith &&
+ !test.equals(BulkLoaderVertexProgramTest.class) &&
+ !test.equals(PageRankVertexProgramTest.class) &&
+ RANDOM.nextBoolean()) {
+ config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ToyGraphInputRDD.class.getCanonicalName());
+ config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName());
}
/// spark configuration
config.put("spark.master", "local[4]");
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/91efb28d/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
index ed97c04..ea3636f 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java
@@ -22,11 +22,13 @@ package org.apache.tinkerpop.gremlin.spark.structure.io;
import org.apache.commons.configuration.Configuration;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;
-import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.hadoop.Constants;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
@@ -34,36 +36,38 @@ import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import scala.Tuple2;
import java.io.IOException;
+import java.io.InputStream;
import java.util.List;
-import java.util.stream.Collectors;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
public final class ToyGraphInputRDD implements InputRDD {
- public static final String GREMLIN_SPARK_TOY_GRAPH = "gremlin.spark.toyGraph";
-
@Override
public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration,
final JavaSparkContext sparkContext) {
- final List<Vertex> vertices;
- if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.MODERN.toString()))
- vertices = IteratorUtils.list(TinkerFactory.createModern().vertices());
- else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.CLASSIC.toString()))
- vertices = IteratorUtils.list(TinkerFactory.createClassic().vertices());
- else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.CREW.toString()))
- vertices = IteratorUtils.list(TinkerFactory.createTheCrew().vertices());
- else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.GRATEFUL.toString()))
{
+ HadoopPools.initialize(TinkerGraph.open().configuration());
+ final List<VertexWritable> vertices;
+ if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("modern"))
+ vertices = IteratorUtils.list(IteratorUtils.map(TinkerFactory.createModern().vertices(),
VertexWritable::new));
+ else if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("classic"))
+ vertices = IteratorUtils.list(IteratorUtils.map(TinkerFactory.createClassic().vertices(),
VertexWritable::new));
+ else if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("crew"))
+ vertices = IteratorUtils.list(IteratorUtils.map(TinkerFactory.createTheCrew().vertices(),
VertexWritable::new));
+ else if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("grateful"))
{
try {
final Graph graph = TinkerGraph.open();
- graph.io(GryoIo.build()).readGraph(GryoResourceAccess.class.getResource("grateful-dead.kryo").getFile());
- vertices = IteratorUtils.list(graph.vertices());
+ final GraphReader reader = GryoReader.build().mapper(graph.io(GryoIo.build()).mapper().create()).create();
+ try (final InputStream stream = GryoResourceAccess.class.getResourceAsStream("grateful-dead.kryo"))
{
+ reader.readGraph(stream, graph);
+ }
+ vertices = IteratorUtils.list(IteratorUtils.map(graph.vertices(), VertexWritable::new));
} catch (final IOException e) {
throw new IllegalStateException(e.getMessage(), e);
}
} else
- throw new IllegalArgumentException("No legal toy graph was provided to load:
" + configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH));
+ throw new IllegalArgumentException("No legal toy graph was provided to load:
" + configuration.getProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION));
- return sparkContext.parallelize(vertices.stream().map(VertexWritable::new).collect(Collectors.toList())).mapToPair(vertex
-> new Tuple2<>(vertex.get().id(), vertex));
+ return sparkContext.parallelize(vertices).mapToPair(vertex -> new Tuple2<>(vertex.get().id(),
vertex));
}
}
|