hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r807940 - in /hadoop/mapreduce/trunk: CHANGES.txt src/java/mapred-default.xml src/java/org/apache/hadoop/mapred/JobClient.java src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java
Date Wed, 26 Aug 2009 09:27:06 GMT
Author: tomwhite
Date: Wed Aug 26 09:27:04 2009
New Revision: 807940

URL: http://svn.apache.org/viewvc?rev=807940&view=rev
Log:
MAPREDUCE-825. JobClient completion poll interval of 5s causes slow tests in local mode. Contributed
by Aaron Kimball.

Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/java/mapred-default.xml
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobClient.java
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=807940&r1=807939&r2=807940&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Wed Aug 26 09:27:04 2009
@@ -266,6 +266,9 @@
     MAPREDUCE-750. Extensible ConnManager factory API. (Aaron Kimball via
     tomwhite)
 
+    MAPREDUCE-825. JobClient completion poll interval of 5s causes slow tests
+    in local mode. (Aaron Kimball via tomwhite)
+
   BUG FIXES
 
     MAPREDUCE-878. Rename fair scheduler design doc to 

Modified: hadoop/mapreduce/trunk/src/java/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/mapred-default.xml?rev=807940&r1=807939&r2=807940&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/mapred-default.xml (original)
+++ hadoop/mapreduce/trunk/src/java/mapred-default.xml Wed Aug 26 09:27:04 2009
@@ -715,6 +715,26 @@
 </property>
 
   <property>
+    <name>jobclient.completion.poll.interval</name>
+    <value>5000</value>
+    <description>The interval (in milliseconds) between which the JobClient
+    polls the JobTracker for updates about job status. You may want to set this
+    to a lower value to make tests run faster on a single node system. Adjusting
+    this value in production may lead to unwanted client-server traffic.
+    </description>
+  </property>
+
+  <property>
+    <name>jobclient.progress.monitor.poll.interval</name>
+    <value>1000</value>
+    <description>The interval (in milliseconds) between which the JobClient
+    reports status to the console and checks for job completion. You may want to set this
+    to a lower value to make tests run faster on a single node system. Adjusting
+    this value in production may lead to unwanted client-server traffic.
+    </description>
+  </property>
+
+  <property>
     <name>mapred.job.tracker.persist.jobstatus.active</name>
     <value>false</value>
     <description>Indicates if persistency of job status information is

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobClient.java?rev=807940&r1=807939&r2=807940&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobClient.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobClient.java Wed Aug 26 09:27:04
2009
@@ -168,6 +168,17 @@
     Configuration.addDefaultResource("mapred-site.xml");
   }
 
+  /** The interval at which monitorAndPrintJob() prints status */
+  private int progMonitorPollIntervalMillis;
+
+  /** Default progMonitorPollIntervalMillis is 1000 ms. */
+  private static final int DEFAULT_MONITOR_POLL_INTERVAL = 1000;
+
+  /** Key in mapred-*.xml that sets progMonitorPollIntervalMillis */
+  static final String PROGRESS_MONITOR_POLL_INTERVAL_KEY =
+      "jobclient.progress.monitor.poll.interval";
+
+
   /**
    * A NetworkedJob is an implementation of RunningJob.  It holds
    * a JobProfile object to provide some info, and interacts with the
@@ -177,6 +188,15 @@
     JobStatus status;
     long statustime;
 
+    /** The interval at which NetworkedJob.waitForCompletion() should check. */
+    private int completionPollIntervalMillis;
+
+    /** Default completionPollIntervalMillis is 5000 ms. */
+    private static final int DEFAULT_COMPLETION_POLL_INTERVAL = 5000;
+
+    /** Key in mapred-*.xml that sets completionPollInvervalMillis */
+    static final String COMPLETION_POLL_INTERVAL_KEY = "jobclient.completion.poll.interval";
+
     /**
      * We store a JobProfile and a timestamp for when we last
      * acquired the job profile.  If the job is null, then we cannot
@@ -187,6 +207,17 @@
     public NetworkedJob(JobStatus job) throws IOException {
       this.status = job;
       this.statustime = System.currentTimeMillis();
+
+      // Set the completion poll interval from the configuration.
+      // Default is 5 seconds.
+      Configuration conf = JobClient.this.getConf();
+      this.completionPollIntervalMillis = conf.getInt(COMPLETION_POLL_INTERVAL_KEY,
+          DEFAULT_COMPLETION_POLL_INTERVAL);
+      if (this.completionPollIntervalMillis < 1) {
+        LOG.warn(COMPLETION_POLL_INTERVAL_KEY + " has been set to an invalid value; "
+            + "replacing with " + DEFAULT_COMPLETION_POLL_INTERVAL);
+        this.completionPollIntervalMillis = DEFAULT_COMPLETION_POLL_INTERVAL;
+      }
     }
 
     /**
@@ -306,7 +337,7 @@
     public void waitForCompletion() throws IOException {
       while (!isComplete()) {
         try {
-          Thread.sleep(5000);
+          Thread.sleep(this.completionPollIntervalMillis);
         } catch (InterruptedException ie) {
         }
       }
@@ -412,6 +443,7 @@
    * Create a job client.
    */
   public JobClient() {
+    this.progMonitorPollIntervalMillis = DEFAULT_MONITOR_POLL_INTERVAL;
   }
     
   /**
@@ -450,6 +482,15 @@
     } else {
       this.jobSubmitClient = createRPCProxy(JobTracker.getAddress(conf), conf);
     }        
+
+    // Read progress monitor poll interval from config. Default is 1 second.
+    this.progMonitorPollIntervalMillis = conf.getInt(PROGRESS_MONITOR_POLL_INTERVAL_KEY,
+        DEFAULT_MONITOR_POLL_INTERVAL);
+    if (this.progMonitorPollIntervalMillis < 1) {
+      LOG.warn(PROGRESS_MONITOR_POLL_INTERVAL_KEY + " has been set to an invalid value; "
+          + " replacing with " + DEFAULT_MONITOR_POLL_INTERVAL);
+      this.progMonitorPollIntervalMillis = DEFAULT_MONITOR_POLL_INTERVAL;
+    }
   }
 
   private JobSubmissionProtocol createRPCProxy(InetSocketAddress addr,
@@ -1292,7 +1333,7 @@
     Configuration.IntegerRanges reduceRanges = conf.getProfileTaskRange(false);
 
     while (!job.isComplete()) {
-      Thread.sleep(1000);
+      Thread.sleep(this.progMonitorPollIntervalMillis);
       String report = 
         (" map " + StringUtils.formatPercent(job.mapProgress(), 0)+
             " reduce " + 

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java?rev=807940&r1=807939&r2=807940&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskFail.java Wed
Aug 26 09:27:04 2009
@@ -165,6 +165,9 @@
       String input = "The quick brown fox\nhas many silly\nred fox sox\n";
       // launch job with fail tasks
       JobConf jobConf = mr.createJobConf();
+      // turn down the completion poll interval from the 5 second default
+      // for better test performance.
+      jobConf.set(JobClient.NetworkedJob.COMPLETION_POLL_INTERVAL_KEY, "50");
       jobConf.setOutputCommitter(CommitterWithLogs.class);
       RunningJob rJob = launchJob(jobConf, inDir, outDir, input);
       rJob.waitForCompletion();



Mime
View raw message