hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject svn commit: r809051 - in /hadoop/mapreduce/branches/MAPREDUCE-233/src: java/org/apache/hadoop/mapred/ test/mapred/org/apache/hadoop/mapred/
Date Fri, 28 Aug 2009 22:29:24 GMT
Author: stevel
Date: Fri Aug 28 22:29:24 2009
New Revision: 809051

URL: http://svn.apache.org/viewvc?rev=809051&view=rev
Log:
MAPREDUCE-233 add some useful toString() ops, increase the test timeout, this makes the problem
go away. I think the test failing is now fixed

Modified:
    hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapred/JobTracker.java
    hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapred/ReduceTask.java
    hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapred/TaskTracker.java
    hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=809051&r1=809050&r2=809051&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapred/JobTracker.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapred/JobTracker.java
Fri Aug 28 22:29:24 2009
@@ -133,7 +133,6 @@
   // in a second
   private int NUM_HEARTBEATS_IN_SECOND = 100;
   public static enum State { INITIALIZING, RUNNING }
-  State state = State.INITIALIZING;
   private static final int FS_ACCESS_RETRY_PERIOD = 10000;
 
   private DNSToSwitchMapping dnsToSwitchMapping;
@@ -2162,10 +2161,10 @@
    * @throws IOException IO Problems
    */
   public void offerService() throws InterruptedException, IOException {
-    if(!enterLiveState()) {
+    if(getServiceState() == ServiceState.LIVE) {
       //catch re-entrancy by returning early
       return;
-    };
+    }
     // Prepare for recovery. This is done irrespective of the status of restart
     // flag.
     while (true) {
@@ -2180,6 +2179,8 @@
       }
     }
 
+    //now we are live
+    enterLiveState();
     taskScheduler.start();
     
     //  Start the recovery after starting the scheduler
@@ -2327,13 +2328,27 @@
   }
 
   /**
-   * Get the current number of workers
-   *
+   * Get the current number of trackers.
+   * This includes blacklisted trackers
    * @return the number of task trackers
    */
   @Override
   public int getLiveWorkerCount() {
-    return getNumResolvedTaskTrackers();
+    return taskTrackers.size();
+  }
+
+  /**
+   * Return a string that is useful in logs and debugging
+   * @return state of the job tracker
+   */
+  @Override
+  public String toString() {
+    return super.toString()
+        + " http://" + conf.get("mapred.job.tracker.http.address") + "/ "
+        + (interTrackerServer != null
+          ? ("ipc://" + interTrackerServer.getListenerAddress() + "/ ")
+          : "")
+        + "workers=" + getLiveWorkerCount();
   }
 
   ///////////////////////////////////////////////////////

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapred/ReduceTask.java?rev=809051&r1=809050&r2=809051&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapred/ReduceTask.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapred/ReduceTask.java
Fri Aug 28 22:29:24 2009
@@ -1230,7 +1230,8 @@
               error = CopyOutputErrorType.NO_ERROR;
             } catch (IOException e) {
               LOG.warn(reduceTask.getTaskID() + " copy failed: " +
-                       loc.getTaskAttemptId() + " from " + loc.getHost());
+                       loc.getTaskAttemptId() + " from " + loc.getHost()
+                       + " " + loc.getOutputLocation());
               LOG.warn(StringUtils.stringifyException(e));
               shuffleClientMetrics.failedFetch();
               if (readError) {
@@ -1297,7 +1298,8 @@
         if (mapOutput == null) {
           throw new IOException("Failed to fetch map-output for " + 
                                 loc.getTaskAttemptId() + " from " + 
-                                loc.getHost());
+                                loc.getHost()
+                                + " at " + loc.getOutputLocation());
         }
         
         // The size of the map-output

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=809051&r1=809050&r2=809051&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapred/TaskTracker.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/java/org/apache/hadoop/mapred/TaskTracker.java
Fri Aug 28 22:29:24 2009
@@ -470,7 +470,7 @@
    */
   synchronized void initialize() throws IOException {
     if (LOG.isDebugEnabled()) {
-      LOG.debug("Initializing Task Tracker: " + this);
+      LOG.debug("Initializing Task Tracker: " + toString());
     }
     //check that the server is not already live.                        
 
@@ -1449,6 +1449,7 @@
           systemDirectory = new Path(dir);
           systemFS = systemDirectory.getFileSystem(fConf);
           if(LOG.isDebugEnabled()) {
+            LOG.debug("Starting " + this);
             LOG.debug("System directory is " + systemDirectory);
           }
         }
@@ -3446,6 +3447,23 @@
   }
 
   /**
+   * Return a string that is useful in logs and debugging
+   *
+   * @return state of the job tracker
+   */
+  @Override
+  public String toString() {
+    return super.toString()
+        + " "
+        + (server != null ?
+          (server.toString() + " ") : "")
+        + (taskReportAddress!=null ?
+          ("rpc://" + taskReportAddress + "/ ") : "")
+        + (jobTrackAddr != null ?
+          (" bound to JobTracker " + jobTrackAddr + " ") : "");
+  }
+
+  /**
    * Is the TaskMemoryManager Enabled on this system?
    * @return true if enabled, false otherwise.
    */

Modified: hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java?rev=809051&r1=809050&r2=809051&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java
(original)
+++ hadoop/mapreduce/branches/MAPREDUCE-233/src/test/mapred/org/apache/hadoop/mapred/UtilsForTests.java
Fri Aug 28 22:29:24 2009
@@ -237,7 +237,7 @@
   static void waitForJobHalfDone(RunningJob job)
           throws IOException {
     // wait for 50%
-    long timeout = System.currentTimeMillis() + 30000;
+    long timeout = System.currentTimeMillis() + 60000;
     while (job.mapProgress() < 0.5f) {
       if(System.currentTimeMillis() > timeout) {
         throw new IOException("Timeout waiting for job to get to 50% done");



Mime
View raw message