crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject crunch git commit: CRUNCH-551: Make the use of Configuration objects consistent in CrunchInputSplit and CrunchRecordReader
Date Tue, 28 Jul 2015 02:34:54 GMT
Repository: crunch
Updated Branches:
  refs/heads/master 29d1ce4cd -> 2f5b33ead


CRUNCH-551: Make the use of Configuration objects consistent in CrunchInputSplit and CrunchRecordReader


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/2f5b33ea
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/2f5b33ea
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/2f5b33ea

Branch: refs/heads/master
Commit: 2f5b33ead752deb4796989f56f9aebf6e3330286
Parents: 29d1ce4
Author: Josh Wills <jwills@apache.org>
Authored: Mon Jul 27 16:00:49 2015 -0700
Committer: Josh Wills <jwills@apache.org>
Committed: Mon Jul 27 19:21:45 2015 -0700

----------------------------------------------------------------------
 .../crunch/impl/mr/run/CrunchInputSplit.java     |  4 ++--
 .../crunch/impl/mr/run/CrunchRecordReader.java   | 19 +++++--------------
 2 files changed, 7 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/2f5b33ea/crunch-core/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputSplit.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputSplit.java
b/crunch-core/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputSplit.java
index ee54483..5ec5015 100644
--- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputSplit.java
+++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/run/CrunchInputSplit.java
@@ -66,8 +66,8 @@ class CrunchInputSplit extends InputSplit implements Writable, Configurable,
Sup
   @Override
   public void setConf(Configuration conf) {
     this.conf = new Configuration(conf);
-    if (bundle != null && conf != null) {
-      this.bundle.configure(conf);
+    if (bundle != null) {
+      this.bundle.configure(this.conf);
     }
   }
 

http://git-wip-us.apache.org/repos/asf/crunch/blob/2f5b33ea/crunch-core/src/main/java/org/apache/crunch/impl/mr/run/CrunchRecordReader.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/run/CrunchRecordReader.java
b/crunch-core/src/main/java/org/apache/crunch/impl/mr/run/CrunchRecordReader.java
index 2842658..d4175a6 100644
--- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/run/CrunchRecordReader.java
+++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/run/CrunchRecordReader.java
@@ -44,12 +44,8 @@ class CrunchRecordReader<K, V> extends RecordReader<K, V> {
     if (crunchSplit.get() instanceof CombineFileSplit) {
       combineFileSplit = (CombineFileSplit) crunchSplit.get();
     }
-    Configuration conf = crunchSplit.getConf();
-    if (conf == null) {
-      conf = context.getConfiguration();
-      crunchSplit.setConf(conf);
-    }
-    this.context = new TaskAttemptContextImpl(conf, context.getTaskAttemptID());
+    crunchSplit.setConf(context.getConfiguration());
+    this.context = new TaskAttemptContextImpl(crunchSplit.getConf(), context.getTaskAttemptID());
     initNextRecordReader();
   }
 
@@ -71,10 +67,9 @@ class CrunchRecordReader<K, V> extends RecordReader<K, V> {
     }
 
     idx++;
-    Configuration conf = crunchSplit.getConf();
     InputFormat<K, V> inputFormat = (InputFormat<K, V>) ReflectionUtils.newInstance(
         crunchSplit.getInputFormatClass(),
-        conf);
+        crunchSplit.getConf());
     this.curReader = inputFormat.createRecordReader(getDelegateSplit(), context);
     return true;
   }
@@ -136,12 +131,8 @@ class CrunchRecordReader<K, V> extends RecordReader<K, V>
{
   @Override
   public void initialize(InputSplit inputSplit, TaskAttemptContext context) throws IOException,
InterruptedException {
     this.crunchSplit = (CrunchInputSplit) inputSplit;
-    Configuration conf = crunchSplit.getConf();
-    if (conf == null) {
-      conf = context.getConfiguration();
-      crunchSplit.setConf(conf);
-    }
-    this.context = new TaskAttemptContextImpl(conf, context.getTaskAttemptID());
+    crunchSplit.setConf(context.getConfiguration());
+    this.context = new TaskAttemptContextImpl(crunchSplit.getConf(), context.getTaskAttemptID());
     if (crunchSplit.get() instanceof CombineFileSplit) {
       combineFileSplit = (CombineFileSplit) crunchSplit.get();
     }


Mime
View raw message