tinkerpop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dkupp...@apache.org
Subject [14/14] incubator-tinkerpop git commit: Merge branch 'master' into TINKERPOP3-909
Date Tue, 03 Nov 2015 16:49:01 GMT
Merge branch 'master' into TINKERPOP3-909


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/558c04e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/558c04e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/558c04e7

Branch: refs/heads/TINKERPOP3-909
Commit: 558c04e7f3d51b12ddd06cd11fe4bbea5298843d
Parents: af4f5dc 06d747f
Author: Daniel Kuppitz <daniel_kuppitz@hotmail.com>
Authored: Tue Nov 3 17:47:37 2015 +0100
Committer: Daniel Kuppitz <daniel_kuppitz@hotmail.com>
Committed: Tue Nov 3 17:47:37 2015 +0100

----------------------------------------------------------------------
 CHANGELOG.asciidoc                              |   3 +-
 docs/src/implementations.asciidoc               |  36 +--
 docs/src/the-traversal.asciidoc                 |  11 +-
 .../upgrade-release-3.1.x-incubating.asciidoc   |  18 +-
 .../process/computer/GiraphGraphComputer.java   |  11 +-
 .../computer/io/GiraphVertexInputFormat.java    |  70 ------
 .../computer/io/GiraphVertexOutputFormat.java   |  65 ------
 .../process/computer/io/GiraphVertexReader.java |  67 ------
 .../process/computer/io/GiraphVertexWriter.java |  57 -----
 .../structure/io/GiraphVertexInputFormat.java   |  70 ++++++
 .../structure/io/GiraphVertexOutputFormat.java  |  65 ++++++
 .../giraph/structure/io/GiraphVertexReader.java |  67 ++++++
 .../giraph/structure/io/GiraphVertexWriter.java |  57 +++++
 .../gremlin/process/computer/GraphComputer.java |   5 +-
 .../process/computer/GraphComputerTest.java     |   2 +-
 .../tinkerpop/gremlin/hadoop/Constants.java     |   1 -
 .../structure/hdfs/HadoopElementIterator.java   |   4 +-
 .../hadoop/structure/io/InputOutputHelper.java  |  22 ++
 .../hadoop/structure/util/HadoopHelper.java     |  50 -----
 .../process/computer/SparkGraphComputer.java    |  75 ++++---
 .../process/computer/io/InputFormatRDD.java     |  47 ----
 .../spark/process/computer/io/InputRDD.java     |  41 ----
 .../process/computer/io/OutputFormatRDD.java    |  49 -----
 .../spark/process/computer/io/OutputRDD.java    |  31 ---
 .../spark/structure/io/InputFormatRDD.java      |  47 ++++
 .../spark/structure/io/InputOutputHelper.java   |  81 +++++++
 .../gremlin/spark/structure/io/InputRDD.java    |  41 ++++
 .../spark/structure/io/OutputFormatRDD.java     |  49 +++++
 .../gremlin/spark/structure/io/OutputRDD.java   |  31 +++
 .../spark/structure/io/PersistedInputRDD.java   |  60 +++++
 .../spark/structure/io/PersistedOutputRDD.java  |  41 ++++
 .../process/computer/io/ExampleInputRDD.java    |  47 ----
 .../process/computer/io/ExampleOutputRDD.java   |  45 ----
 .../process/computer/io/InputOutputRDDTest.java |  59 -----
 .../spark/process/computer/io/InputRDDTest.java |  54 -----
 .../process/computer/io/OutputRDDTest.java      |  62 ------
 .../spark/structure/io/ExampleInputRDD.java     |  51 +++++
 .../spark/structure/io/ExampleOutputRDD.java    |  50 +++++
 .../spark/structure/io/InputOutputRDDTest.java  |  59 +++++
 .../spark/structure/io/InputRDDTest.java        |  54 +++++
 .../spark/structure/io/OutputRDDTest.java       |  62 ++++++
 .../io/PersistedInputOutputRDDTest.java         | 217 +++++++++++++++++++
 .../process/computer/TinkerGraphComputer.java   |   5 -
 43 files changed, 1228 insertions(+), 811 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/558c04e7/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --cc CHANGELOG.asciidoc
index 84cb9f5,e79a0a0..5ad89e5
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@@ -25,10 -25,10 +25,11 @@@ image::https://raw.githubusercontent.co
  TinkerPop 3.1.0 (NOT OFFICIALLY RELEASED YET)
  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  
 +* Integrated `NumberHelper` in `SumStep`, `MinStep`, `MaxStep` and `MeanStep` (local and
global step variants).
  * Bumped to Neo4j 2.3.0.
+ * Added `PersistedInputRDD` and `PersistedOutputRDD` which enables `SparkGraphComputer`
to store the graph RDD in the context between jobs (no HDFS serialization required).
  * Renamed the `public static String` configuration variable names of TinkerGraph (deprecated
old variables).
- * Added `GraphComputer.config(key,value)` to allow engine-specific configurations.
+ * Added `GraphComputer.configure(key,value)` to allow engine-specific configurations.
  * `GraphStep` is no longer in the `sideEffect`-package and is now in `map`-package (breaking
change).
  * Added support for mid-traversal `V()`-steps (`GraphStep` semantics updated).
  * Fixed `Number` handling in `Operator` enums. Prior this change a lot of operations on
mixed `Number` types returned a wrong result (wrong data type).

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/558c04e7/docs/src/upgrade-release-3.1.x-incubating.asciidoc
----------------------------------------------------------------------
diff --cc docs/src/upgrade-release-3.1.x-incubating.asciidoc
index 00be151,9e5b50d..ada54f9
--- a/docs/src/upgrade-release-3.1.x-incubating.asciidoc
+++ b/docs/src/upgrade-release-3.1.x-incubating.asciidoc
@@@ -165,3 -172,11 +173,11 @@@ The `VendorOptimizationStrategy` has be
  with revised terminology for what were formerly referred to as "vendors".
  
  See link:https://issues.apache.org/jira/browse/TINKERPOP3-876[TINKERPOP3-876] for more information.
+ 
+ GraphComputer Updates
+ +++++++++++++++++++++
+ 
+ `GraphComputer.configure(String key, Object value)` is now a method (with default implementation).
+ This allows the user to specify engine-specific parameters to the underlying OLAP system.
These parameters are not intended
+ to be cross engine supported. Moreover, if there are not parameters that can be altered
(beyond the standard `GraphComputer`
 -methods), then the provider's `GraphComputer` implementation should simply return and do
nothing.
++methods), then the provider's `GraphComputer` implementation should simply return and do
nothing.

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/558c04e7/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/558c04e7/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
----------------------------------------------------------------------
diff --cc spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
index 0000000,8d6c06e..e8bd44a
mode 000000,100644..100644
--- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
+++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/InputRDDTest.java
@@@ -1,0 -1,54 +1,54 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  * http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing,
+  * software distributed under the License is distributed on an
+  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  * KIND, either express or implied.  See the License for the
+  * specific language governing permissions and limitations
+  * under the License.
+  */
+ package org.apache.tinkerpop.gremlin.spark.structure.io;
+ 
+ import org.apache.commons.configuration.BaseConfiguration;
+ import org.apache.commons.configuration.Configuration;
+ import org.apache.tinkerpop.gremlin.hadoop.Constants;
+ import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+ import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat;
+ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
+ import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer;
+ import org.apache.tinkerpop.gremlin.structure.Graph;
+ import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
+ import org.junit.Test;
+ 
+ import static org.junit.Assert.assertEquals;
+ 
+ /**
+  * @author Marko A. Rodriguez (http://markorodriguez.com)
+  */
+ public class InputRDDTest {
+ 
+     @Test
+     public void shouldReadFromArbitraryRDD() {
+         final Configuration configuration = new BaseConfiguration();
+         configuration.setProperty("spark.master", "local[4]");
+         configuration.setProperty("spark.serializer", "org.apache.spark.serializer.KryoSerializer");
+         configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
+         configuration.setProperty(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD, ExampleInputRDD.class.getCanonicalName());
+         configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_OUTPUT_FORMAT, GryoOutputFormat.class.getCanonicalName());
+         configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, "target/test-output");
+         configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
+         ////////
+         Graph graph = GraphFactory.open(configuration);
 -        assertEquals(Double.valueOf(123.0d), graph.traversal(GraphTraversalSource.computer(SparkGraphComputer.class)).V().values("age").sum().next());
++        assertEquals(123l, graph.traversal(GraphTraversalSource.computer(SparkGraphComputer.class)).V().values("age").sum().next());
+         assertEquals(Long.valueOf(4l), graph.traversal(GraphTraversalSource.computer(SparkGraphComputer.class)).V().count().next());
+     }
+ }


Mime
View raw message