hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r806152 - in /hadoop/mapreduce/trunk: ./ src/java/org/apache/hadoop/mapred/ src/test/ src/test/mapred/org/apache/hadoop/mapred/
Date Thu, 20 Aug 2009 12:56:25 GMT
Author: ddas
Date: Thu Aug 20 12:56:25 2009
New Revision: 806152

URL: http://svn.apache.org/viewvc?rev=806152&view=rev
Log:
MAPREDUCE-745. Fixes a testcase problem to do with generation of JobTracker IDs. Contributed
by Amar Kamat.

Added:
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerStart.java
Modified:
    hadoop/mapreduce/trunk/CHANGES.txt
    hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
    hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml
    hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java

Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=806152&r1=806151&r2=806152&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Thu Aug 20 12:56:25 2009
@@ -413,3 +413,6 @@
     MAPREDUCE-832. Reduce number of warning messages printed when
     deprecated memory variables are used. (Rahul Kumar Singh via yhemanth)
 
+    MAPREDUCE-745. Fixes a testcase problem to do with generation of JobTracker
+    IDs. (Amar Kamat via ddas)
+

Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=806152&r1=806151&r2=806152&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Thu Aug 20 12:56:25
2009
@@ -150,7 +150,9 @@
   final static FsPermission SYSTEM_FILE_PERMISSION =
     FsPermission.createImmutable((short) 0700); // rwx------
   
-  private static Clock clock;
+  private static Clock clock = null;
+  
+  static final Clock DEFAULT_CLOCK = new Clock();
 
   /**
    * A client tried to submit a job before the Job Tracker was ready.
@@ -181,8 +183,13 @@
 
   public static final Log LOG = LogFactory.getLog(JobTracker.class);
     
+  /**
+   * Returns JobTracker's clock. Note that the correct clock implementation will
+   * be obtained only when the JobTracker is initialized. If the JobTracker is
+   * not initialized then the default clock i.e {@link Clock} is returned. 
+   */
   static Clock getClock() {
-    return clock;
+    return clock == null ? DEFAULT_CLOCK : clock;
   }
   
   /**
@@ -198,15 +205,20 @@
   public static JobTracker startTracker(JobConf conf
                                         ) throws IOException,
                                                  InterruptedException {
-    return startTracker(conf, new Clock());
+    return startTracker(conf, DEFAULT_CLOCK);
   }
 
   static JobTracker startTracker(JobConf conf, Clock clock) 
   throws IOException, InterruptedException {
+    return startTracker(conf, clock, generateNewIdentifier());
+  }
+
+  static JobTracker startTracker(JobConf conf, Clock clock, String identifier) 
+  throws IOException, InterruptedException {
     JobTracker result = null;
     while (true) {
       try {
-        result = new JobTracker(conf, clock);
+        result = new JobTracker(conf, clock, identifier);
         result.taskScheduler.setTaskTrackerManager(result);
         break;
       } catch (VersionMismatch e) {
@@ -1104,6 +1116,9 @@
           hasUpdates = true;
           LOG.info("Calling init from RM for job " + jip.getJobID().toString());
           initJob(jip);
+          if (!jip.inited()) {
+            throw new IOException("Failed to initialize job " + jip.getJobID());
+          }
         }
       }
       
@@ -1832,8 +1847,13 @@
    * Start the JobTracker process, listen on the indicated port
    */
   JobTracker(JobConf conf, Clock clock) throws IOException, InterruptedException {
+    this(conf, clock, generateNewIdentifier());
+  }
+
+  JobTracker(JobConf conf, Clock newClock, String jobtrackerIndentifier) 
+  throws IOException, InterruptedException {
     // find the owner of the process
-    this.clock = clock;
+    clock = newClock;
     try {
       mrOwner = UnixUserGroupInformation.login(conf);
     } catch (LoginException e) {
@@ -1928,7 +1948,7 @@
     infoServer.addServlet("reducegraph", "/taskgraph", TaskGraphServlet.class);
     infoServer.start();
     
-    trackerIdentifier = getDateFormat().format(new Date());
+    this.trackerIdentifier = jobtrackerIndentifier;
 
     // Initialize instrumentation
     JobTrackerInstrumentation tmp;
@@ -2037,6 +2057,10 @@
     return new SimpleDateFormat("yyyyMMddHHmm");
   }
 
+  private static String generateNewIdentifier() {
+    return getDateFormat().format(new Date());
+  }
+  
   static boolean validateIdentifier(String id) {
     try {
       // the jobtracker id should be 'date' parseable

Modified: hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml?rev=806152&r1=806151&r2=806152&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml (original)
+++ hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml Thu Aug 20 12:56:25 2009
@@ -80,6 +80,11 @@
        <Bug pattern="DM_EXIT" />
      </Match>
      <Match>
+       <Class name="org.apache.hadoop.mapred.JobTracker" />
+       <Field name="clock" />
+       <Bug pattern="ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD" />
+     </Match>
+     <Match>
        <Class name="org.apache.hadoop.mapred.Task$TaskReporter" />
        <Method name="run" />
        <Bug pattern="DM_EXIT" />

Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java?rev=806152&r1=806151&r2=806152&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/MiniMRCluster.java Thu
Aug 20 12:56:25 2009
@@ -19,7 +19,9 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.text.SimpleDateFormat;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
 
@@ -67,7 +69,7 @@
     private volatile boolean isActive = true;
     
     JobConf jc = null;
-    Clock clock = null;
+    Clock clock = JobTracker.DEFAULT_CLOCK;
         
     public JobTrackerRunner(JobConf conf) {
       jc = conf;
@@ -108,11 +110,9 @@
         jc.set("mapred.local.dir",f.getAbsolutePath());
         jc.setClass("topology.node.switch.mapping.impl", 
             StaticMapping.class, DNSToSwitchMapping.class);
-        if (clock == null) {
-          tracker = JobTracker.startTracker(jc);
-        } else {
-          tracker = JobTracker.startTracker(jc, clock);
-        }
+        String id = 
+          new SimpleDateFormat("yyyyMMddHHmmssSSS").format(new Date());
+        tracker = JobTracker.startTracker(jc, clock, id);
         tracker.offerService();
       } catch (Throwable e) {
         LOG.error("Job tracker crashed", e);
@@ -328,6 +328,13 @@
     if(conf == null) {
       conf = new JobConf();
     }
+    return configureJobConf(conf, namenode, jobTrackerPort, jobTrackerInfoPort, 
+                            ugi);
+  }
+  
+  static JobConf configureJobConf(JobConf conf, String namenode, 
+                                  int jobTrackerPort, int jobTrackerInfoPort, 
+                                  UnixUserGroupInformation ugi) {
     JobConf result = new JobConf(conf);
     FileSystem.setDefaultUri(result, namenode);
     result.set("mapred.job.tracker", "localhost:"+jobTrackerPort);

Added: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerStart.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerStart.java?rev=806152&view=auto
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerStart.java
(added)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobTrackerStart.java
Thu Aug 20 12:56:25 2009
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import org.apache.hadoop.mapred.UtilsForTests.FakeClock;
+
+import junit.framework.TestCase;
+
+/**
+ * Test {@link JobTracker} w.r.t config parameters.
+ */
+public class TestJobTrackerStart extends TestCase {
+  
+  public void testJobTrackerStartConfig() throws Exception {
+    JobConf conf = new JobConf();
+    conf = MiniMRCluster.configureJobConf(conf, "file:///", 0, 0, null);
+    
+    // test JobTracker's default clock
+    Clock c = JobTracker.getClock();
+    assertNotNull(c);
+    assertEquals(c, JobTracker.DEFAULT_CLOCK);
+    
+    // test with default values
+    JobTracker jt = JobTracker.startTracker(conf);
+    c = JobTracker.getClock();
+    // test clock
+    assertNotNull(c);
+    assertEquals(c, JobTracker.DEFAULT_CLOCK);
+    // test identifier
+    assertEquals(12, jt.getTrackerIdentifier().length()); // correct upto mins
+    jt.stopTracker();
+    
+    // test with special clock
+    FakeClock myClock = new FakeClock();
+    jt = JobTracker.startTracker(conf, myClock);
+    c = JobTracker.getClock();
+    assertNotNull(c);
+    assertEquals(c, myClock);
+    jt.stopTracker();
+    
+    // test with special identifier
+    String identifier = "test-identifier";
+    jt = JobTracker.startTracker(conf, JobTracker.DEFAULT_CLOCK, identifier);
+    assertEquals(identifier, jt.getTrackerIdentifier());
+    jt.stopTracker();
+  }
+}



Mime
View raw message