Return-Path: 
 <commits-return-708-apmail-crunch-commits-archive=crunch.apache.org@crunch.apache.org>
X-Original-To: apmail-crunch-commits-archive@www.apache.org
Delivered-To: apmail-crunch-commits-archive@www.apache.org
Received: from mail.apache.org (hermes.apache.org [140.211.11.3])
	by minotaur.apache.org (Postfix) with SMTP id C49271042D
	for <apmail-crunch-commits-archive@www.apache.org>;
 Thu, 12 Dec 2013 03:29:42 +0000 (UTC)
Received: (qmail 80796 invoked by uid 500); 12 Dec 2013 03:29:38 -0000
Delivered-To: apmail-crunch-commits-archive@crunch.apache.org
Received: (qmail 80725 invoked by uid 500); 12 Dec 2013 03:29:36 -0000
Mailing-List: contact commits-help@crunch.apache.org; run by ezmlm
Precedence: bulk
List-Help: <mailto:commits-help@crunch.apache.org>
List-Unsubscribe: <mailto:commits-unsubscribe@crunch.apache.org>
List-Post: <mailto:commits@crunch.apache.org>
List-Id: <commits.crunch.apache.org>
Reply-To: dev@crunch.apache.org
Delivered-To: mailing list commits@crunch.apache.org
Received: (qmail 80697 invoked by uid 99); 12 Dec 2013 03:29:30 -0000
Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org)
 (140.211.11.114)
    by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 12 Dec 2013 03:29:30 +0000
Received: by tyr.zones.apache.org (Postfix, from userid 65534)
	id 8BECF815D6B; Thu, 12 Dec 2013 03:29:29 +0000 (UTC)
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: jwills@apache.org
To: commits@crunch.apache.org
Date: Thu, 12 Dec 2013 03:29:30 -0000
Message-Id: <8f27cd7169184dc4b82db174b394b869@git.apache.org>
In-Reply-To: <661ebcb6fcab4fdd884d4411c34c96b4@git.apache.org>
References: <661ebcb6fcab4fdd884d4411c34c96b4@git.apache.org>
X-Mailer: ASF-Git Admin Mailer
Subject: [2/2] git commit: Get spark distributed cache working on hadoop2

Get spark distributed cache working on hadoop2


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/e5a36051
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/e5a36051
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/e5a36051

Branch: refs/heads/master
Commit: e5a360512c61d368358130b4a37ef7fa19f6a4ab
Parents: 8932f2a
Author: Josh Wills <jwills@apache.org>
Authored: Wed Dec 11 16:30:23 2013 -0800
Committer: Josh Wills <jwills@apache.org>
Committed: Wed Dec 11 19:26:54 2013 -0800

----------------------------------------------------------------------
 .../java/org/apache/crunch/impl/spark/SparkRuntimeContext.java | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/e5a36051/crunch-spark/src/main/java/org/apache/crunch/impl/spark/SparkRuntimeContext.java
----------------------------------------------------------------------
diff --git a/crunch-spark/src/main/java/org/apache/crunch/impl/spark/SparkRuntimeContext.java b/crunch-spark/src/main/java/org/apache/crunch/impl/spark/SparkRuntimeContext.java
index 92336a3..78436c2 100644
--- a/crunch-spark/src/main/java/org/apache/crunch/impl/spark/SparkRuntimeContext.java
+++ b/crunch-spark/src/main/java/org/apache/crunch/impl/spark/SparkRuntimeContext.java
@@ -76,9 +76,13 @@ public class SparkRuntimeContext implements Serializable {
         List<String> allFiles = Lists.newArrayList();
         for (URI uri : uris) {
           File f = new File(uri.getPath());
+          String sparkFile = SparkFiles.get(f.getName());
           allFiles.add(SparkFiles.get(f.getName()));
         }
-        DistributedCache.setLocalFiles(getConfiguration(), Joiner.on(',').join(allFiles));
+        String sparkFiles = Joiner.on(',').join(allFiles);
+        // Hacking this for Hadoop1 and Hadoop2
+        getConfiguration().set("mapreduce.job.cache.local.files", sparkFiles);
+        getConfiguration().set("mapred.cache.localFiles", sparkFiles);
       }
     } catch (IOException e) {
       throw new CrunchRuntimeException(e);