incubator-crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gr...@apache.org
Subject git commit: CRUNCH-42 - Use short poll interval in local mode
Date Tue, 14 Aug 2012 20:26:26 GMT
Updated Branches:
  refs/heads/master 6a66e1794 -> 92c06fc6c


CRUNCH-42 - Use short poll interval in local mode

Use a short poll interval (500 milliseconds instead of 5000) when
running jobs in the local job runner -- this allows the
integration tests to be run significantly faster.


Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/92c06fc6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/92c06fc6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/92c06fc6

Branch: refs/heads/master
Commit: 92c06fc6c3dc852e24a9bce3c368250d5b4af43b
Parents: 6a66e17
Author: Gabriel Reid <greid@apache.org>
Authored: Tue Aug 14 22:24:06 2012 +0200
Committer: Gabriel Reid <greid@apache.org>
Committed: Tue Aug 14 22:24:06 2012 +0200

----------------------------------------------------------------------
 .../mapreduce/lib/jobcontrol/CrunchJobControl.java |   12 +++++++++++-
 1 files changed, 11 insertions(+), 1 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/92c06fc6/crunch/src/main/java/org/apache/crunch/hadoop/mapreduce/lib/jobcontrol/CrunchJobControl.java
----------------------------------------------------------------------
diff --git a/crunch/src/main/java/org/apache/crunch/hadoop/mapreduce/lib/jobcontrol/CrunchJobControl.java
b/crunch/src/main/java/org/apache/crunch/hadoop/mapreduce/lib/jobcontrol/CrunchJobControl.java
index 9e9af85..3230f90 100644
--- a/crunch/src/main/java/org/apache/crunch/hadoop/mapreduce/lib/jobcontrol/CrunchJobControl.java
+++ b/crunch/src/main/java/org/apache/crunch/hadoop/mapreduce/lib/jobcontrol/CrunchJobControl.java
@@ -25,6 +25,7 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.crunch.hadoop.mapreduce.lib.jobcontrol.CrunchControlledJob.State;
+import org.apache.hadoop.conf.Configuration;
 
 /**
  * This class encapsulates a set of MapReduce jobs and its dependency.
@@ -62,6 +63,7 @@ public class CrunchJobControl implements Runnable {
 
   private long nextJobID;
   private String groupName;
+  private int jobPollInterval;
 
   /**
    * Construct a job control for a group of jobs.
@@ -78,6 +80,7 @@ public class CrunchJobControl implements Runnable {
     this.nextJobID = -1;
     this.groupName = groupName;
     this.runnerState = ThreadState.READY;
+    this.jobPollInterval = isLocalMode() ? 500 : 5000;
   }
 
   private static List<CrunchControlledJob> toList(Map<String, CrunchControlledJob>
jobs) {
@@ -282,7 +285,7 @@ public class CrunchJobControl implements Runnable {
         break;
       }
       try {
-        Thread.sleep(5000);
+        Thread.sleep(jobPollInterval);
       } catch (Exception e) {
 
       }
@@ -293,5 +296,12 @@ public class CrunchJobControl implements Runnable {
     }
     this.runnerState = ThreadState.STOPPED;
   }
+  
+  private boolean isLocalMode() {
+    Configuration conf = new Configuration();
+    // Try to handle MapReduce version 0.20 or 0.22
+    String jobTrackerAddress = conf.get("mapreduce.jobtracker.address", conf.get("mapred.job.tracker",
"local"));
+    return "local".equals(jobTrackerAddress);
+  }
 
 }


Mime
View raw message