tinkerpop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From spmalle...@apache.org
Subject [12/50] [abbrv] incubator-tinkerpop git commit: Testing to make sure InputOutputHelper translator works for registered RDDs.
Date Thu, 05 Nov 2015 01:56:30 GMT
Testing to make sure InputOutputHelper translator works for registered RDDs.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/cf0c4562
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/cf0c4562
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/cf0c4562

Branch: refs/heads/TINKERPOP3-923
Commit: cf0c4562d4b992343080e9b5cb5f38fae33c0c3e
Parents: cb97238
Author: Marko A. Rodriguez <okrammarko@gmail.com>
Authored: Fri Oct 30 14:37:43 2015 -0600
Committer: Marko A. Rodriguez <okrammarko@gmail.com>
Committed: Fri Oct 30 14:37:43 2015 -0600

----------------------------------------------------------------------
 .../gremlin/spark/structure/io/InputOutputHelper.java  |  2 +-
 .../gremlin/spark/structure/io/ExampleInputRDD.java    | 13 ++++++++-----
 .../gremlin/spark/structure/io/ExampleOutputRDD.java   |  6 +++++-
 3 files changed, 14 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/cf0c4562/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputHelper.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputHelper.java
b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputHelper.java
index b1e1b0a..1e279de 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputHelper.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputOutputHelper.java
@@ -68,7 +68,7 @@ public final class InputOutputHelper {
             final BaseConfiguration newConfiguration = new BaseConfiguration();
             newConfiguration.copy(org.apache.tinkerpop.gremlin.hadoop.structure.io.InputOutputHelper.getOutputGraph(configuration,
resultGraph, persist).configuration());
             if (resultGraph.equals(GraphComputer.ResultGraph.NEW) && hadoopConfiguration.containsKey(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD))
{
-                newConfiguration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, InputOutputHelper.getInputFormat((Class)
Class.forName(hadoopConfiguration.getProperty(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD).toString())).getCanonicalName());
+                newConfiguration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, InputOutputHelper.getInputFormat((Class)
Class.forName(hadoopConfiguration.getString(Constants.GREMLIN_SPARK_GRAPH_OUTPUT_RDD))).getCanonicalName());
                 if (newConfiguration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION, "").endsWith("/"
+ Constants.HIDDEN_G)) {  // Spark RDDs are not namespaced the same as Hadoop
                     newConfiguration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION,
newConfiguration.getString(Constants.GREMLIN_HADOOP_INPUT_LOCATION).replace("/" + Constants.HIDDEN_G,
""));
                 }

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/cf0c4562/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleInputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleInputRDD.java
b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleInputRDD.java
index 8417d1c..86c7610 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleInputRDD.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleInputRDD.java
@@ -22,7 +22,6 @@ import org.apache.commons.configuration.Configuration;
 import org.apache.spark.api.java.JavaPairRDD;
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.spark.structure.io.InputRDD;
 import org.apache.tinkerpop.gremlin.structure.T;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.apache.tinkerpop.gremlin.structure.util.star.StarGraph;
@@ -36,13 +35,17 @@ import java.util.List;
  */
 public final class ExampleInputRDD implements InputRDD {
 
+    static {
+        InputOutputHelper.registerInputOutputPair(ExampleInputRDD.class, ExampleOutputRDD.class);
+    }
+
     @Override
     public JavaPairRDD<Object, VertexWritable> readGraphRDD(final Configuration configuration,
final JavaSparkContext sparkContext) {
         final List<Vertex> list = new ArrayList<>();
-        list.add(StarGraph.open().addVertex(T.id, 1l, T.label,"person","age", 29));
-        list.add(StarGraph.open().addVertex(T.id, 2l, T.label,"person","age", 27));
-        list.add(StarGraph.open().addVertex(T.id, 4l, T.label,"person","age", 32));
-        list.add(StarGraph.open().addVertex(T.id, 6l, T.label,"person","age", 35));
+        list.add(StarGraph.open().addVertex(T.id, 1l, T.label, "person", "age", 29));
+        list.add(StarGraph.open().addVertex(T.id, 2l, T.label, "person", "age", 27));
+        list.add(StarGraph.open().addVertex(T.id, 4l, T.label, "person", "age", 32));
+        list.add(StarGraph.open().addVertex(T.id, 6l, T.label, "person", "age", 35));
         return sparkContext.parallelize(list).mapToPair(vertex -> new Tuple2<>(vertex.id(),
new VertexWritable(vertex)));
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/cf0c4562/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleOutputRDD.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleOutputRDD.java
b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleOutputRDD.java
index 9a9e5ef..103ec20 100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleOutputRDD.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/ExampleOutputRDD.java
@@ -21,7 +21,6 @@ package org.apache.tinkerpop.gremlin.spark.structure.io;
 import org.apache.commons.configuration.Configuration;
 import org.apache.spark.api.java.JavaPairRDD;
 import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
-import org.apache.tinkerpop.gremlin.spark.structure.io.OutputRDD;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 
 import java.util.Iterator;
@@ -32,6 +31,11 @@ import static org.junit.Assert.assertEquals;
  * @author Marko A. Rodriguez (http://markorodriguez.com)
  */
 public final class ExampleOutputRDD implements OutputRDD {
+
+    static {
+        InputOutputHelper.registerInputOutputPair(ExampleInputRDD.class, ExampleOutputRDD.class);
+    }
+
     @Override
     public void writeGraphRDD(final Configuration configuration, final JavaPairRDD<Object,
VertexWritable> graphRDD) {
         int totalAge = 0;


Mime
View raw message