hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From deva...@apache.org
Subject hadoop git commit: MAPREDUCE-6350. JobHistory doesn't support fully-functional search. Contributed by Siqi Li.
Date Wed, 10 Jun 2015 07:14:31 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 48ca23def -> 6785661e5


MAPREDUCE-6350. JobHistory doesn't support fully-functional search.
Contributed by Siqi Li.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6785661e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6785661e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6785661e

Branch: refs/heads/trunk
Commit: 6785661e554114a4613b5fe7dabec9bfa80c41d9
Parents: 48ca23d
Author: Devaraj K <devaraj@apache.org>
Authored: Wed Jun 10 12:40:29 2015 +0530
Committer: Devaraj K <devaraj@apache.org>
Committed: Wed Jun 10 12:40:29 2015 +0530

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt            |  3 +++
 .../jobhistory/JobHistoryEventHandler.java      |  6 ++++-
 .../v2/jobhistory/FileNameIndexUtils.java       | 17 ++++++++-----
 .../mapreduce/v2/jobhistory/JHAdminConfig.java  |  7 ++++++
 .../v2/jobhistory/TestFileNameIndexUtils.java   | 26 ++++++++++++++++++++
 .../src/main/resources/mapred-default.xml       |  8 ++++++
 6 files changed, 60 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 115fc1b..3c1884d 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -465,6 +465,9 @@ Release 2.8.0 - UNRELEASED
     MAPREDUCE-6388. Remove deprecation warnings from JobHistoryServer classes
     (Ray Chiang via ozawa).
 
+    MAPREDUCE-6350. JobHistory doesn't support fully-functional search. 
+    (Siqi Li via devaraj)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index bf32888..35556a6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.mapreduce.v2.app.AppContext;
 import org.apache.hadoop.mapreduce.v2.app.job.Job;
 import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal;
 import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
 import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
 import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -1107,9 +1108,12 @@ public class JobHistoryEventHandler extends AbstractService
       if (mi.getHistoryFile() != null) {
         Path historyFile = mi.getHistoryFile();
         Path qualifiedLogFile = stagingDirFS.makeQualified(historyFile);
+        int jobNameLimit =
+            getConfig().getInt(JHAdminConfig.MR_HS_JOBNAME_LIMIT,
+            JHAdminConfig.DEFAULT_MR_HS_JOBNAME_LIMIT);
         String doneJobHistoryFileName =
             getTempFileName(FileNameIndexUtils.getDoneFileName(mi
-                .getJobIndexInfo()));
+                .getJobIndexInfo(), jobNameLimit));
         qualifiedDoneFile =
             doneDirFS.makeQualified(new Path(doneDirPrefixPath,
                 doneJobHistoryFileName));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
index 741da11..eb0c54c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
@@ -32,8 +32,6 @@ import org.apache.hadoop.mapreduce.v2.api.records.JobId;
 
 public class FileNameIndexUtils {
 
-  static final int JOB_NAME_TRIM_LENGTH = 50;
-  
   // Sanitize job history file for predictable parsing
   static final String DELIMITER = "-";
   static final String DELIMITER_ESCAPE = "%2D";
@@ -60,6 +58,12 @@ public class FileNameIndexUtils {
    * @return the done job history filename.
    */
   public static String getDoneFileName(JobIndexInfo indexInfo) throws IOException {
+    return getDoneFileName(indexInfo,
+        JHAdminConfig.DEFAULT_MR_HS_JOBNAME_LIMIT);
+  }
+
+  public static String getDoneFileName(JobIndexInfo indexInfo,
+      int jobNameLimit) throws IOException {
     StringBuilder sb = new StringBuilder();
     //JobId
     sb.append(escapeDelimiters(TypeConverter.fromYarn(indexInfo.getJobId()).toString()));
@@ -74,7 +78,8 @@ public class FileNameIndexUtils {
     sb.append(DELIMITER);
     
     //JobName
-    sb.append(escapeDelimiters(trimJobName(getJobName(indexInfo))));
+    sb.append(escapeDelimiters(trimJobName(
+        getJobName(indexInfo), jobNameLimit)));
     sb.append(DELIMITER);
     
     //FinishTime
@@ -286,9 +291,9 @@ public class FileNameIndexUtils {
   /**
    * Trims the job-name if required
    */
-  private static String trimJobName(String jobName) {
-    if (jobName.length() > JOB_NAME_TRIM_LENGTH) {
-      jobName = jobName.substring(0, JOB_NAME_TRIM_LENGTH);
+  private static String trimJobName(String jobName, int jobNameLimit) {
+    if (jobName.length() > jobNameLimit) {
+      jobName = jobName.substring(0, jobNameLimit);
     }
     return jobName;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
index f7cba9f..a97c2ca 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
@@ -214,4 +214,11 @@ public class JHAdminConfig {
    */
   public static boolean DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS = false;
 
+  /**
+   * Number of characters allowed for job name in Job History Server web page.
+   */
+  public static final String MR_HS_JOBNAME_LIMIT = MR_HISTORY_PREFIX
+      + "jobname.limit";
+  public static final int DEFAULT_MR_HS_JOBNAME_LIMIT = 50;
+
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
index a0d5fce..3d722e0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
@@ -148,6 +148,32 @@ public class TestFileNameIndexUtils {
   }
 
   @Test
+  public void testTrimJobName() throws IOException {
+    int jobNameTrimLength = 5;
+    JobIndexInfo info = new JobIndexInfo();
+    JobID oldJobId = JobID.forName(JOB_ID);
+    JobId jobId = TypeConverter.toYarn(oldJobId);
+    info.setJobId(jobId);
+    info.setSubmitTime(Long.parseLong(SUBMIT_TIME));
+    info.setUser(USER_NAME);
+    info.setJobName(JOB_NAME);
+    info.setFinishTime(Long.parseLong(FINISH_TIME));
+    info.setNumMaps(Integer.parseInt(NUM_MAPS));
+    info.setNumReduces(Integer.parseInt(NUM_REDUCES));
+    info.setJobStatus(JOB_STATUS);
+    info.setQueueName(QUEUE_NAME);
+    info.setJobStartTime(Long.parseLong(JOB_START_TIME));
+
+    String jobHistoryFile =
+         FileNameIndexUtils.getDoneFileName(info, jobNameTrimLength);
+    JobIndexInfo parsedInfo = FileNameIndexUtils.getIndexInfo(jobHistoryFile);
+
+    Assert.assertEquals("Job name did not get trimmed correctly",
+        info.getJobName().substring(0, jobNameTrimLength),
+        parsedInfo.getJobName());
+  }
+
+  @Test
   public void testUserNamePercentDecoding() throws IOException {
     String jobHistoryFile = String.format(JOB_HISTORY_FILE_FORMATTER,
         JOB_ID,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 4a2dd77..3789429 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1706,6 +1706,14 @@
 </property>
 
 <property>
+  <name>mapreduce.jobhistory.jobname.limit</name>
+  <value>50</value>
+  <description>
+     Number of characters allowed for job name in Job History Server web page.
+  </description>
+</property>
+
+<property>
   <name>mapreduce.job.heap.memory-mb.ratio</name>
   <value>0.8</value>
   <description>The ratio of heap-size to container-size. If no -Xmx is


Mime
View raw message