Return-Path: X-Original-To: apmail-tinkerpop-commits-archive@minotaur.apache.org Delivered-To: apmail-tinkerpop-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id C735818474 for ; Wed, 2 Dec 2015 22:45:03 +0000 (UTC) Received: (qmail 89393 invoked by uid 500); 2 Dec 2015 22:45:03 -0000 Delivered-To: apmail-tinkerpop-commits-archive@tinkerpop.apache.org Received: (qmail 89369 invoked by uid 500); 2 Dec 2015 22:45:03 -0000 Mailing-List: contact commits-help@tinkerpop.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tinkerpop.incubator.apache.org Delivered-To: mailing list commits@tinkerpop.incubator.apache.org Received: (qmail 89359 invoked by uid 99); 2 Dec 2015 22:45:03 -0000 Received: from Unknown (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 Dec 2015 22:45:03 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id 373A4CC184 for ; Wed, 2 Dec 2015 22:45:03 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 1.771 X-Spam-Level: * X-Spam-Status: No, score=1.771 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, T_RP_MATCHES_RCVD=-0.01, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-us-west.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id 7AzeXw0LESIo for ; Wed, 2 Dec 2015 22:44:58 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-us-west.apache.org (ASF Mail Server at mx1-us-west.apache.org) with SMTP id BE6FA2021C for ; Wed, 2 Dec 2015 22:44:57 +0000 (UTC) Received: (qmail 89320 invoked by uid 99); 2 Dec 2015 22:44:57 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 02 Dec 2015 22:44:57 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 6C47FE60D4; Wed, 2 Dec 2015 22:44:57 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: okram@apache.org To: commits@tinkerpop.incubator.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: incubator-tinkerpop git commit: GroovyProcessCompiuterSuite was missing GroovyFlatMapTest. Added it. Added HadoopPool registration to ToyGraphInputRDD so it doesn't give a WARN message. Also I tweaked BulkLoaderVertexProgramTest to use target/test-output Date: Wed, 2 Dec 2015 22:44:57 +0000 (UTC) Repository: incubator-tinkerpop Updated Branches: refs/heads/TINKERPOP3-1011 e20ff9199 -> 91efb28df GroovyProcessCompiuterSuite was missing GroovyFlatMapTest. Added it. Added HadoopPool registration to ToyGraphInputRDD so it doesn't give a WARN message. Also I tweaked BulkLoaderVertexProgramTest to use target/test-output/ for its intermediary data. Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/91efb28d Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/91efb28d Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/91efb28d Branch: refs/heads/TINKERPOP3-1011 Commit: 91efb28df23fdc0dc99c78bd72159f0478614df1 Parents: e20ff91 Author: Marko A. Rodriguez Authored: Wed Dec 2 15:44:50 2015 -0700 Committer: Marko A. Rodriguez Committed: Wed Dec 2 15:44:50 2015 -0700 ---------------------------------------------------------------------- .../process/GroovyProcessComputerSuite.java | 2 ++ .../BulkLoaderVertexProgramTest.java | 2 +- .../computer/SparkHadoopGraphProvider.java | 16 +++++---- .../spark/structure/io/ToyGraphInputRDD.java | 38 +++++++++++--------- 4 files changed, 33 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/91efb28d/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java ---------------------------------------------------------------------- diff --git a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java index 9df7902..d07d52f 100644 --- a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java +++ b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java @@ -47,6 +47,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyAddEdgeTest import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCoalesceTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyConstantTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCountTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyFlatMapTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyFoldTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyGraphTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyLoopsTest; @@ -128,6 +129,7 @@ public class GroovyProcessComputerSuite extends ProcessComputerSuite { GroovyCoalesceTest.Traversals.class, GroovyConstantTest.Traversals.class, GroovyCountTest.Traversals.class, + GroovyFlatMapTest.Traversals.class, GroovyFoldTest.Traversals.class, GroovyGraphTest.Traversals.class, GroovyLoopsTest.Traversals.class, http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/91efb28d/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java ---------------------------------------------------------------------- diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java index 61a267f..992bfa4 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgramTest.java @@ -52,7 +52,7 @@ import static org.junit.Assert.assertTrue; */ public class BulkLoaderVertexProgramTest extends AbstractGremlinProcessTest { - final static String TINKERGRAPH_LOCATION = "/tmp/tinkertest.kryo"; + final static String TINKERGRAPH_LOCATION = "target/test-output/tinkertest.kryo"; private BulkLoader getBulkLoader(final BulkLoaderVertexProgram blvp) throws Exception { final Field field = BulkLoaderVertexProgram.class.getDeclaredField("bulkLoader"); http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/91efb28d/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java ---------------------------------------------------------------------- diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java index 2cfeea3..de5a60d 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java @@ -22,6 +22,8 @@ import org.apache.tinkerpop.gremlin.GraphProvider; import org.apache.tinkerpop.gremlin.LoadGraphWith; import org.apache.tinkerpop.gremlin.hadoop.Constants; import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider; +import org.apache.tinkerpop.gremlin.process.computer.bulkloading.BulkLoaderVertexProgramTest; +import org.apache.tinkerpop.gremlin.process.computer.ranking.PageRankVertexProgramTest; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.process.traversal.engine.ComputerTraversalEngine; import org.apache.tinkerpop.gremlin.spark.structure.io.InputRDDFormat; @@ -43,13 +45,13 @@ public final class SparkHadoopGraphProvider extends HadoopGraphProvider { @Override public Map getBaseConfiguration(final String graphName, final Class test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) { final Map config = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith); - if (null != loadGraphWith) { - if (RANDOM.nextBoolean()) { - config.remove(Constants.GREMLIN_HADOOP_INPUT_LOCATION); - config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ToyGraphInputRDD.class.getCanonicalName()); - config.put(ToyGraphInputRDD.GREMLIN_SPARK_TOY_GRAPH, loadGraphWith.toString()); - config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName()); - } + config.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); // this makes the test suite go really fast + if (null != loadGraphWith && + !test.equals(BulkLoaderVertexProgramTest.class) && + !test.equals(PageRankVertexProgramTest.class) && + RANDOM.nextBoolean()) { + config.put(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ToyGraphInputRDD.class.getCanonicalName()); + config.put(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputRDDFormat.class.getCanonicalName()); } /// spark configuration config.put("spark.master", "local[4]"); http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/91efb28d/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java ---------------------------------------------------------------------- diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java index ed97c04..ea3636f 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ToyGraphInputRDD.java @@ -22,11 +22,13 @@ package org.apache.tinkerpop.gremlin.spark.structure.io; import org.apache.commons.configuration.Configuration; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.tinkerpop.gremlin.LoadGraphWith; +import org.apache.tinkerpop.gremlin.hadoop.Constants; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools; import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable; import org.apache.tinkerpop.gremlin.structure.Graph; -import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.io.GraphReader; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess; import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerFactory; import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph; @@ -34,36 +36,38 @@ import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils; import scala.Tuple2; import java.io.IOException; +import java.io.InputStream; import java.util.List; -import java.util.stream.Collectors; /** * @author Marko A. Rodriguez (http://markorodriguez.com) */ public final class ToyGraphInputRDD implements InputRDD { - public static final String GREMLIN_SPARK_TOY_GRAPH = "gremlin.spark.toyGraph"; - @Override public JavaPairRDD readGraphRDD(final Configuration configuration, final JavaSparkContext sparkContext) { - final List vertices; - if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.MODERN.toString())) - vertices = IteratorUtils.list(TinkerFactory.createModern().vertices()); - else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.CLASSIC.toString())) - vertices = IteratorUtils.list(TinkerFactory.createClassic().vertices()); - else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.CREW.toString())) - vertices = IteratorUtils.list(TinkerFactory.createTheCrew().vertices()); - else if (configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH).equals(LoadGraphWith.GraphData.GRATEFUL.toString())) { + HadoopPools.initialize(TinkerGraph.open().configuration()); + final List vertices; + if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("modern")) + vertices = IteratorUtils.list(IteratorUtils.map(TinkerFactory.createModern().vertices(), VertexWritable::new)); + else if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("classic")) + vertices = IteratorUtils.list(IteratorUtils.map(TinkerFactory.createClassic().vertices(), VertexWritable::new)); + else if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("crew")) + vertices = IteratorUtils.list(IteratorUtils.map(TinkerFactory.createTheCrew().vertices(), VertexWritable::new)); + else if (configuration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).contains("grateful")) { try { final Graph graph = TinkerGraph.open(); - graph.io(GryoIo.build()).readGraph(GryoResourceAccess.class.getResource("grateful-dead.kryo").getFile()); - vertices = IteratorUtils.list(graph.vertices()); + final GraphReader reader = GryoReader.build().mapper(graph.io(GryoIo.build()).mapper().create()).create(); + try (final InputStream stream = GryoResourceAccess.class.getResourceAsStream("grateful-dead.kryo")) { + reader.readGraph(stream, graph); + } + vertices = IteratorUtils.list(IteratorUtils.map(graph.vertices(), VertexWritable::new)); } catch (final IOException e) { throw new IllegalStateException(e.getMessage(), e); } } else - throw new IllegalArgumentException("No legal toy graph was provided to load: " + configuration.getProperty(GREMLIN_SPARK_TOY_GRAPH)); + throw new IllegalArgumentException("No legal toy graph was provided to load: " + configuration.getProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION)); - return sparkContext.parallelize(vertices.stream().map(VertexWritable::new).collect(Collectors.toList())).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex)); + return sparkContext.parallelize(vertices).mapToPair(vertex -> new Tuple2<>(vertex.get().id(), vertex)); } }