hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1077066 - in /hadoop/common/branches/branch-0.20-security-patches/src: mapred/org/apache/hadoop/mapred/JobHistory.java test/findbugsExcludeFile.xml test/org/apache/hadoop/mapred/TestJobRetire.java webapps/job/jobdetails.jsp
Date Fri, 04 Mar 2011 03:37:08 GMT
Author: omalley
Date: Fri Mar  4 03:37:07 2011
New Revision: 1077066

URL: http://svn.apache.org/viewvc?rev=1077066&view=rev
Log:
commit d43124eb4216ad175cfd9312c98736fb91ca51a8
Author: Hemanth Yamijala <yhemanth@yahoo-inc.com>
Date:   Tue Dec 8 11:27:55 2009 +0530

    MAPREDUCE:1185 from https://issues.apache.org/jira/secure/attachment/12426630/patch-1185-3-ydist.txt
    
    +++ b/YAHOO-CHANGES.txt
    +    MAPREDUCE-1185. Redirect running job url to history url if job is already
    +    retired. (Amareshwari Sriramadasu and Sharad Agarwal via sharad)
    +

Added:
    hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobRetire.java
Modified:
    hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
    hadoop/common/branches/branch-0.20-security-patches/src/test/findbugsExcludeFile.xml
    hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobdetails.jsp

Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=1077066&r1=1077065&r2=1077066&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
(original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
Fri Mar  4 03:37:07 2011
@@ -29,10 +29,14 @@ import java.io.UnsupportedEncodingExcept
 import java.net.URLDecoder;
 import java.net.URLEncoder;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
+import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -119,6 +123,30 @@ public class JobHistory {
     }
   };
 
+  private static Map<JobID, MovedFileInfo> jobHistoryFileMap =
+    Collections.<JobID,MovedFileInfo>synchronizedMap(
+        new LinkedHashMap<JobID, MovedFileInfo>());
+
+  private static class MovedFileInfo {
+    private final String historyFile;
+    private final long timestamp;
+    public MovedFileInfo(String historyFile, long timestamp) {
+      this.historyFile = historyFile;
+      this.timestamp = timestamp;
+    }
+  }
+
+  /**
+   * Given the job id, return the history file path from the cache
+   */
+  public static String getHistoryFilePath(JobID jobId) {
+    MovedFileInfo info = jobHistoryFileMap.get(jobId);
+    if (info == null) {
+      return null;
+    }
+    return info.historyFile;
+  }
+
   /**
    * A class that manages all the files related to a job. For now 
    *   - writers : list of open files
@@ -238,6 +266,9 @@ public class JobHistory {
             historyFileDonePath = new Path(DONE, 
                 historyFile.getName()).toString();
           }
+
+          jobHistoryFileMap.put(id, new MovedFileInfo(historyFileDonePath,
+              System.currentTimeMillis()));
           jobTracker.historyFileCopied(id, historyFileDonePath);
           
           //purge the job from the cache
@@ -2036,6 +2067,22 @@ public class JobHistory {
             }
           }
         }
+
+        //walking over the map to purge entries from jobHistoryFileMap
+        synchronized (jobHistoryFileMap) {
+          Iterator<Entry<JobID, MovedFileInfo>> it =
+            jobHistoryFileMap.entrySet().iterator();
+          while (it.hasNext()) {
+            MovedFileInfo info = it.next().getValue();
+            if (now - info.timestamp > THIRTY_DAYS_IN_MS) {
+              it.remove();
+            } else {
+              //since entries are in sorted timestamp order, no more entries
+              //are required to be checked
+              break;
+            }
+          }
+        }
       } catch (IOException ie) {
         LOG.info("Error cleaning up history directory" + 
                  StringUtils.stringifyException(ie));

Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/findbugsExcludeFile.xml?rev=1077066&r1=1077065&r2=1077066&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/findbugsExcludeFile.xml (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/findbugsExcludeFile.xml Fri
Mar  4 03:37:07 2011
@@ -16,6 +16,10 @@
        <Bug pattern="DLS_DEAD_LOCAL_STORE" />
      </Match>
      <Match>
+       <Class name="org.apache.hadoop.mapred.jobdetails_jsp"/>
+       <Bug pattern="HRS_REQUEST_PARAMETER_TO_HTTP_HEADER"/>
+     </Match>
+     <Match>
        <Field name="_jspx_dependants" />
        <Bug pattern="UWF_UNWRITTEN_FIELD" />
      </Match>

Added: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobRetire.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobRetire.java?rev=1077066&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobRetire.java
(added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobRetire.java
Fri Mar  4 03:37:07 2011
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobTracker.RetireJobInfo;
+
+/**
+ * Test if the job retire works fine.
+ */
+public class TestJobRetire extends TestCase {
+  static final Path testDir =
+    new Path(System.getProperty("test.build.data","/tmp"),
+             "job-expiry-testing");
+
+  public void testJobRetire() throws Exception {
+    MiniMRCluster mr = null;
+    try {
+      JobConf conf = new JobConf();
+
+      conf.setLong("mapred.job.tracker.retiredjobs.cache.size", 1);
+      conf.setLong("mapred.jobtracker.retirejob.interval", 0);
+      conf.setLong("mapred.jobtracker.retirejob.check", 0);
+      conf.getLong("mapred.jobtracker.completeuserjobs.maximum", 0);
+      mr = new MiniMRCluster(0, 0, 1, "file:///", 1, null, null, null, conf, 0);
+      JobConf jobConf = mr.createJobConf();
+      JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker();
+
+      Path inDir = new Path(testDir, "input1");
+      Path outDir = new Path(testDir, "output1");
+
+      JobID id1 = validateJobRetire(jobConf, inDir, outDir, jobtracker);
+
+      outDir = new Path(testDir, "output2");
+      JobID id2 = validateJobRetire(jobConf, inDir, outDir, jobtracker);
+
+      assertNull("Job not removed from cache", jobtracker.getJobStatus(id1));
+
+      assertEquals("Total job in cache not correct",
+          1, jobtracker.getAllJobs().length);
+    } finally {
+      if (mr != null) { mr.shutdown();}
+    }
+  }
+
+  private JobID validateJobRetire(JobConf jobConf, Path inDir, Path outDir,
+      JobTracker jobtracker) throws IOException {
+
+    RunningJob rj = UtilsForTests.runJob(jobConf, inDir, outDir, 0, 0);
+    rj.waitForCompletion();
+    assertTrue(rj.isSuccessful());
+    JobID id = rj.getID();
+
+    JobInProgress job = jobtracker.getJob(id);
+    //wait for job to get retired
+    for (int i = 0; i < 10 && job != null; i++) {
+      UtilsForTests.waitFor(1000);
+      job = jobtracker.getJob(id);
+    }
+    assertNull("Job did not retire", job);
+    RetireJobInfo retired = jobtracker.retireJobs.get(id);
+    assertTrue("History url not set", retired.getHistoryFile() != null &&
+      retired.getHistoryFile().length() > 0);
+    assertNotNull("Job is not in cache", jobtracker.getJobStatus(id));
+
+    // get the job conf filename
+    String name = jobtracker.getLocalJobFilePath(id);
+    File file = new File(name);
+
+    assertFalse("JobConf file not deleted", file.exists());
+    //test redirection
+    URL jobUrl = new URL(rj.getTrackingURL());
+    HttpURLConnection conn = (HttpURLConnection) jobUrl.openConnection();
+    conn.setInstanceFollowRedirects(false);
+    conn.connect();
+    assertEquals(HttpURLConnection.HTTP_MOVED_TEMP, conn.getResponseCode());
+    conn.disconnect();
+
+    URL redirectedUrl = new URL(conn.getHeaderField("Location"));
+    conn = (HttpURLConnection) redirectedUrl.openConnection();
+    conn.connect();
+    assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+    conn.disconnect();
+
+    return id;
+  }
+
+}

Modified: hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobdetails.jsp
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobdetails.jsp?rev=1077066&r1=1077065&r2=1077066&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobdetails.jsp (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobdetails.jsp Fri
Mar  4 03:37:07 2011
@@ -197,7 +197,14 @@
 
 <% 
     if (job == null) {
-      out.print("<b>Job " + jobId + " not found.</b><br>\n");
+      String historyFile = JobHistory.getHistoryFilePath(jobIdObj);
+      if (historyFile == null) {
+        out.println("<h2>Job " + jobId + " not known!</h2>");
+        return;
+      }
+      String historyUrl = "/jobdetailshistory.jsp?jobid=" + jobId +
+      "&logFile=" + JobHistory.JobInfo.encodeJobHistoryFilePath(historyFile);
+      response.sendRedirect(response.encodeRedirectURL(historyUrl));
       return;
     }
     JobProfile profile = job.getProfile();



Mime
View raw message