crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject git commit: CRUNCH-394: Ensure Broadcast variable w/serialized Configuration is never null in SparkRuntimeContext
Date Thu, 15 May 2014 01:06:28 GMT
Repository: crunch
Updated Branches:
  refs/heads/apache-crunch-0.8 5c433c8f4 -> 1a8685ae1


CRUNCH-394: Ensure Broadcast variable w/serialized Configuration is never null in SparkRuntimeContext


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/1a8685ae
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/1a8685ae
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/1a8685ae

Branch: refs/heads/apache-crunch-0.8
Commit: 1a8685ae1f51f3722ec7a6954c836f9b023422a6
Parents: 5c433c8
Author: Josh Wills <jwills@apache.org>
Authored: Wed May 14 17:11:56 2014 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Wed May 14 17:26:11 2014 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/crunch/impl/spark/SparkRuntime.java    | 2 +-
 .../java/org/apache/crunch/impl/spark/SparkRuntimeContext.java  | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/1a8685ae/crunch-spark/src/main/java/org/apache/crunch/impl/spark/SparkRuntime.java
----------------------------------------------------------------------
diff --git a/crunch-spark/src/main/java/org/apache/crunch/impl/spark/SparkRuntime.java b/crunch-spark/src/main/java/org/apache/crunch/impl/spark/SparkRuntime.java
index 2cb2fb3..22375ee 100644
--- a/crunch-spark/src/main/java/org/apache/crunch/impl/spark/SparkRuntime.java
+++ b/crunch-spark/src/main/java/org/apache/crunch/impl/spark/SparkRuntime.java
@@ -110,7 +110,7 @@ public class SparkRuntime extends AbstractFuture<PipelineResult>
implements Pipe
     this.conf = conf;
     this.counters = sparkContext.accumulator(Maps.<String, Map<String, Long>>newHashMap(),
         new CounterAccumulatorParam());
-    this.ctxt = new SparkRuntimeContext(counters);
+    this.ctxt = new SparkRuntimeContext(counters, sparkContext.broadcast(WritableUtils.toByteArray(conf)));
     this.outputTargets = Maps.newTreeMap(DEPTH_COMPARATOR);
     this.outputTargets.putAll(outputTargets);
     this.toMaterialize = toMaterialize;

http://git-wip-us.apache.org/repos/asf/crunch/blob/1a8685ae/crunch-spark/src/main/java/org/apache/crunch/impl/spark/SparkRuntimeContext.java
----------------------------------------------------------------------
diff --git a/crunch-spark/src/main/java/org/apache/crunch/impl/spark/SparkRuntimeContext.java
b/crunch-spark/src/main/java/org/apache/crunch/impl/spark/SparkRuntimeContext.java
index 102ad4a..cea317c 100644
--- a/crunch-spark/src/main/java/org/apache/crunch/impl/spark/SparkRuntimeContext.java
+++ b/crunch-spark/src/main/java/org/apache/crunch/impl/spark/SparkRuntimeContext.java
@@ -50,8 +50,11 @@ public class SparkRuntimeContext implements Serializable {
   private transient Configuration conf;
   private transient TaskInputOutputContext context;
 
-  public SparkRuntimeContext(Accumulator<Map<String, Map<String, Long>>>
counters) {
+  public SparkRuntimeContext(
+      Accumulator<Map<String, Map<String, Long>>> counters,
+      Broadcast<byte[]> broadConf) {
     this.counters = counters;
+    this.broadConf = broadConf;
   }
 
   public void setConf(Broadcast<byte[]> broadConf) {


Mime
View raw message