tinkerpop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From spmalle...@apache.org
Subject [26/38] tinkerpop git commit: TINKERPOP-1996 Pass configurations from with() through to Hadoop
Date Tue, 31 Jul 2018 12:17:43 GMT
TINKERPOP-1996 Pass configurations from with() through to Hadoop

This will allow users to override or add to the Hadoop/Spark/OLAP configuration as needed


Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/4d979cf8
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/4d979cf8
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/4d979cf8

Branch: refs/heads/master
Commit: 4d979cf8dc4482d574191a76a0c60e281f887563
Parents: 51dc821
Author: Stephen Mallette <spmva@genoprime.com>
Authored: Fri Jul 20 07:35:48 2018 -0400
Committer: Stephen Mallette <spmva@genoprime.com>
Committed: Fri Jul 20 07:35:48 2018 -0400

----------------------------------------------------------------------
 .../traversal/step/sideEffect/HadoopIoStep.java | 29 ++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/4d979cf8/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java
b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java
index ca369b6..2e96276 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java
@@ -29,6 +29,7 @@ import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram;
 import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep;
 import org.apache.tinkerpop.gremlin.process.traversal.IO;
 import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
 import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
 import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting;
 import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters;
@@ -37,7 +38,9 @@ import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
 
 /**
  * An OLAP oriented step for doing IO operations with {@link GraphTraversalSource#io(String)}
which uses the
- * {@link CloneVertexProgram} for its implementation.
+ * {@link CloneVertexProgram} for its implementation. Standard Hadoop OLAP configurations
can be passed using the
+ * {@link GraphTraversal#with(String, Object)} step modulator as all options aside from those
in {@link IO} will be
+ * transferred.
  *
  * @author Stephen Mallette (http://stephen.genoprime.com)
  */
@@ -69,7 +72,6 @@ public class HadoopIoStep extends VertexProgramStep implements ReadWriting
{
 
     @Override
     public void configure(final Object... keyValues) {
-        // TODO: probably should write to the Configuration selectively - no need for actual
Parameters?????????
         this.parameters.set(null, keyValues);
     }
 
@@ -121,6 +123,8 @@ public class HadoopIoStep extends VertexProgramStep implements ReadWriting
{
 
         graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, inputFormatClassName);
         graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, file);
+
+        addParametersToConfiguration(graph);
     }
 
     private void configureForWrite(final Graph graph) {
@@ -137,6 +141,27 @@ public class HadoopIoStep extends VertexProgramStep implements ReadWriting
{
         
         graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, outputFormatClassName);
         graph.configuration().setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, file);
+
+        addParametersToConfiguration(graph);
+    }
+
+    /**
+     * Overwrites all configurations from values passed using {@link GraphTraversal#with(String,
Object)}.
+     */
+    private void addParametersToConfiguration(final Graph graph) {
+        parameters.getRaw(IO.writer, IO.writer, IO.registry).entrySet().forEach(kv ->
{
+            if (kv.getValue().size() == 1)
+                graph.configuration().setProperty(kv.getKey().toString(), kv.getValue().get(0));
+            else {
+                // reset the default configuration with the first option then add to that
for List options
+                for (int ix = 0; ix < kv.getValue().size(); ix++) {
+                    if (ix == 0)
+                        graph.configuration().setProperty(kv.getKey().toString(), kv.getValue().get(ix));
+                    else
+                        graph.configuration().addProperty(kv.getKey().toString(), kv.getValue().get(ix));
+                }
+            }
+        });
     }
 
     private String detectReader() {


Mime
View raw message