hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject svn commit: r1582127 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/ hadoop-mapreduce-client/hadoop-mapreduce-client-comm...
Date Wed, 26 Mar 2014 23:44:18 GMT
Author: vinodkv
Date: Wed Mar 26 23:44:17 2014
New Revision: 1582127

URL: http://svn.apache.org/r1582127
Log:
MAPREDUCE-5805. Fixed MapReduce JobHistory encoding of queue-name to escape hyphens and thus
avoid parsing errors. Contributed by Akira Ajisaka.
svn merge --ignore-ancestry -c 1582125 ../../trunk/

Modified:
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
    hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1582127&r1=1582126&r2=1582127&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Wed Mar 26 23:44:17
2014
@@ -124,6 +124,9 @@ Release 2.4.0 - UNRELEASED
     MAPREDUCE-5795. Fixed MRAppMaster to record the correct job-state after it
     recovers from a commit during a previous attempt. (Xuan Gong via vinodkv)
 
+    MAPREDUCE-5805. Fixed MapReduce JobHistory encoding of queue-name to escape
+    hyphens and thus avoid parsing errors. (Akira AJISAKA via vinodkv)
+
 Release 2.3.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java?rev=1582127&r1=1582126&r2=1582127&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
Wed Mar 26 23:44:17 2014
@@ -94,7 +94,7 @@ public class FileNameIndexUtils {
     sb.append(DELIMITER);
     
     //QueueName
-    sb.append(indexInfo.getQueueName());
+    sb.append(escapeDelimiters(indexInfo.getQueueName()));
     sb.append(DELIMITER);
 
     //JobStartTime

Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java?rev=1582127&r1=1582126&r2=1582127&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java
Wed Mar 26 23:44:17 2014
@@ -68,6 +68,10 @@ public class TestFileNameIndexUtils {
   private static final String NUM_REDUCES = "1";
   private static final String JOB_STATUS = "SUCCEEDED";
   private static final String QUEUE_NAME = "default";
+  private static final String QUEUE_NAME_WITH_DELIMITER = "test"
+      + FileNameIndexUtils.DELIMITER + "queue";
+  private static final String QUEUE_NAME_WITH_DELIMITER_ESCAPE = "test"
+      + FileNameIndexUtils.DELIMITER_ESCAPE + "queue";
   private static final String JOB_START_TIME = "1317928742060";
 
   @Test
@@ -192,6 +196,46 @@ public class TestFileNameIndexUtils {
   }
 
   @Test
+  public void testQueueNamePercentEncoding() throws IOException {
+    JobIndexInfo info = new JobIndexInfo();
+    JobID oldJobId = JobID.forName(JOB_ID);
+    JobId jobId = TypeConverter.toYarn(oldJobId);
+    info.setJobId(jobId);
+    info.setSubmitTime(Long.parseLong(SUBMIT_TIME));
+    info.setUser(USER_NAME);
+    info.setJobName(JOB_NAME);
+    info.setFinishTime(Long.parseLong(FINISH_TIME));
+    info.setNumMaps(Integer.parseInt(NUM_MAPS));
+    info.setNumReduces(Integer.parseInt(NUM_REDUCES));
+    info.setJobStatus(JOB_STATUS);
+    info.setQueueName(QUEUE_NAME_WITH_DELIMITER);
+    info.setJobStartTime(Long.parseLong(JOB_START_TIME));
+
+    String jobHistoryFile = FileNameIndexUtils.getDoneFileName(info);
+    Assert.assertTrue("Queue name not encoded correctly into job history file",
+        jobHistoryFile.contains(QUEUE_NAME_WITH_DELIMITER_ESCAPE));
+  }
+
+  @Test
+  public void testQueueNamePercentDecoding() throws IOException {
+    String jobHistoryFile = String.format(JOB_HISTORY_FILE_FORMATTER,
+        JOB_ID,
+        SUBMIT_TIME,
+        USER_NAME,
+        JOB_NAME,
+        FINISH_TIME,
+        NUM_MAPS,
+        NUM_REDUCES,
+        JOB_STATUS,
+        QUEUE_NAME_WITH_DELIMITER_ESCAPE,
+        JOB_START_TIME );
+
+    JobIndexInfo info = FileNameIndexUtils.getIndexInfo(jobHistoryFile);
+    Assert.assertEquals("Queue name doesn't match",
+        QUEUE_NAME_WITH_DELIMITER, info.getQueueName());
+  }
+
+  @Test
   public void testJobHistoryFileNameBackwardsCompatible() throws IOException {
     JobID oldJobId = JobID.forName(JOB_ID);
     JobId jobId = TypeConverter.toYarn(oldJobId);



Mime
View raw message