tinkerpop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ok...@apache.org
Subject [13/14] incubator-tinkerpop git commit: worker regulation based on coalesce.
Date Wed, 09 Dec 2015 13:51:34 GMT
worker regulation based on coalesce.


Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/357bc442
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/357bc442
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/357bc442

Branch: refs/heads/master
Commit: 357bc4423166f7bcdba74ff306dcaaa6ca2b2feb
Parents: aef9528
Author: Marko A. Rodriguez <okrammarko@gmail.com>
Authored: Wed Dec 9 06:40:09 2015 -0700
Committer: Marko A. Rodriguez <okrammarko@gmail.com>
Committed: Wed Dec 9 06:40:09 2015 -0700

----------------------------------------------------------------------
 .../gremlin/spark/process/computer/SparkGraphComputer.java       | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/357bc442/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
index 9219211..56364eb 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java
@@ -67,6 +67,7 @@ import java.util.stream.Stream;
 public final class SparkGraphComputer extends AbstractHadoopGraphComputer {
 
     private final org.apache.commons.configuration.Configuration sparkConfiguration;
+    private boolean workersSet = false;
 
     public SparkGraphComputer(final HadoopGraph hadoopGraph) {
         super(hadoopGraph);
@@ -80,6 +81,7 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer
{
         if (this.sparkConfiguration.getString(SparkLauncher.SPARK_MASTER).startsWith("local"))
{
             this.sparkConfiguration.setProperty(SparkLauncher.SPARK_MASTER, "local[" + this.workers
+ "]");
         }
+        this.workersSet = true;
         return this;
     }
 
@@ -153,7 +155,7 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer
{
                     graphRDD = hadoopConfiguration.getClass(Constants.GREMLIN_SPARK_GRAPH_INPUT_RDD,
InputFormatRDD.class, InputRDD.class)
                             .newInstance()
                             .readGraphRDD(apacheConfiguration, sparkContext);
-                    if (graphRDD.partitions().size() > this.workers) // ensures that the
graphRDD does not have more partitions than workers
+                    if (this.workersSet && graphRDD.partitions().size() > this.workers)
// ensures that the graphRDD does not have more partitions than workers
                         graphRDD = graphRDD.coalesce(this.workers);
                     graphRDD = graphRDD.cache();
                 } catch (final InstantiationException | IllegalAccessException e) {


Mime
View raw message