incubator-crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject git commit: CRUNCH-160: CrunchInputFormat needs to create a new Configuration object for each delegate InputFormat
Date Mon, 04 Feb 2013 16:12:26 GMT
Updated Branches:
  refs/heads/master 41f01c037 -> 6536f4c90


CRUNCH-160: CrunchInputFormat needs to create a new Configuration object for each delegate
InputFormat


Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/6536f4c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/6536f4c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/6536f4c9

Branch: refs/heads/master
Commit: 6536f4c909a1b8d952f1c4398195177659d1423d
Parents: 41f01c0
Author: Josh Wills <jwills@apache.org>
Authored: Sun Feb 3 17:51:55 2013 -0800
Committer: Josh Wills <jwills@apache.org>
Committed: Sun Feb 3 17:51:55 2013 -0800

----------------------------------------------------------------------
 .../crunch/impl/mr/run/CrunchInputFormat.java      |    8 +++-----
 1 files changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/6536f4c9/crunch/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputFormat.java
----------------------------------------------------------------------
diff --git a/crunch/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputFormat.java b/crunch/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputFormat.java
index d39f67f..eb5dd8a 100644
--- a/crunch/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputFormat.java
+++ b/crunch/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputFormat.java
@@ -42,19 +42,17 @@ public class CrunchInputFormat<K, V> extends InputFormat<K, V>
{
   @Override
   public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException
{
     List<InputSplit> splits = Lists.newArrayList();
-    Configuration conf = job.getConfiguration();
+    Configuration base = job.getConfiguration();
     Map<FormatBundle, Map<Integer, List<Path>>> formatNodeMap = CrunchInputs.getFormatNodeMap(job);
 
     // First, build a map of InputFormats to Paths
     for (Map.Entry<FormatBundle, Map<Integer, List<Path>>> entry : formatNodeMap.entrySet())
{
       FormatBundle inputBundle = entry.getKey();
+      Configuration conf = new Configuration(base);
+      inputBundle.configure(conf);
       Job jobCopy = new Job(conf);
-      inputBundle.configure(jobCopy.getConfiguration());
       InputFormat<?, ?> format = (InputFormat<?, ?>) ReflectionUtils.newInstance(inputBundle.getFormatClass(),
           jobCopy.getConfiguration());
-      if (format instanceof Configurable) {
-        ((Configurable) format).setConf(jobCopy.getConfiguration());
-      }
       for (Map.Entry<Integer, List<Path>> nodeEntry : entry.getValue().entrySet())
{
         Integer nodeIndex = nodeEntry.getKey();
         List<Path> paths = nodeEntry.getValue();


Mime
View raw message