Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 492AE106D2 for ; Wed, 26 Mar 2014 23:44:18 +0000 (UTC) Received: (qmail 96023 invoked by uid 500); 26 Mar 2014 23:44:16 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 95739 invoked by uid 500); 26 Mar 2014 23:44:15 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 95717 invoked by uid 99); 26 Mar 2014 23:44:14 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 26 Mar 2014 23:44:14 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 26 Mar 2014 23:44:12 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id EFF98238890D; Wed, 26 Mar 2014 23:43:49 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1582125 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/... Date: Wed, 26 Mar 2014 23:43:49 -0000 To: mapreduce-commits@hadoop.apache.org From: vinodkv@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20140326234349.EFF98238890D@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: vinodkv Date: Wed Mar 26 23:43:49 2014 New Revision: 1582125 URL: http://svn.apache.org/r1582125 Log: MAPREDUCE-5805. Fixed MapReduce JobHistory encoding of queue-name to escape hyphens and thus avoid parsing errors. Contributed by Akira Ajisaka. Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1582125&r1=1582124&r2=1582125&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Mar 26 23:43:49 2014 @@ -266,6 +266,9 @@ Release 2.4.0 - UNRELEASED MAPREDUCE-5795. Fixed MRAppMaster to record the correct job-state after it recovers from a commit during a previous attempt. (Xuan Gong via vinodkv) + MAPREDUCE-5805. Fixed MapReduce JobHistory encoding of queue-name to escape + hyphens and thus avoid parsing errors. (Akira AJISAKA via vinodkv) + Release 2.3.1 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java?rev=1582125&r1=1582124&r2=1582125&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java Wed Mar 26 23:43:49 2014 @@ -94,7 +94,7 @@ public class FileNameIndexUtils { sb.append(DELIMITER); //QueueName - sb.append(indexInfo.getQueueName()); + sb.append(escapeDelimiters(indexInfo.getQueueName())); sb.append(DELIMITER); //JobStartTime Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java?rev=1582125&r1=1582124&r2=1582125&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/jobhistory/TestFileNameIndexUtils.java Wed Mar 26 23:43:49 2014 @@ -68,6 +68,10 @@ public class TestFileNameIndexUtils { private static final String NUM_REDUCES = "1"; private static final String JOB_STATUS = "SUCCEEDED"; private static final String QUEUE_NAME = "default"; + private static final String QUEUE_NAME_WITH_DELIMITER = "test" + + FileNameIndexUtils.DELIMITER + "queue"; + private static final String QUEUE_NAME_WITH_DELIMITER_ESCAPE = "test" + + FileNameIndexUtils.DELIMITER_ESCAPE + "queue"; private static final String JOB_START_TIME = "1317928742060"; @Test @@ -192,6 +196,46 @@ public class TestFileNameIndexUtils { } @Test + public void testQueueNamePercentEncoding() throws IOException { + JobIndexInfo info = new JobIndexInfo(); + JobID oldJobId = JobID.forName(JOB_ID); + JobId jobId = TypeConverter.toYarn(oldJobId); + info.setJobId(jobId); + info.setSubmitTime(Long.parseLong(SUBMIT_TIME)); + info.setUser(USER_NAME); + info.setJobName(JOB_NAME); + info.setFinishTime(Long.parseLong(FINISH_TIME)); + info.setNumMaps(Integer.parseInt(NUM_MAPS)); + info.setNumReduces(Integer.parseInt(NUM_REDUCES)); + info.setJobStatus(JOB_STATUS); + info.setQueueName(QUEUE_NAME_WITH_DELIMITER); + info.setJobStartTime(Long.parseLong(JOB_START_TIME)); + + String jobHistoryFile = FileNameIndexUtils.getDoneFileName(info); + Assert.assertTrue("Queue name not encoded correctly into job history file", + jobHistoryFile.contains(QUEUE_NAME_WITH_DELIMITER_ESCAPE)); + } + + @Test + public void testQueueNamePercentDecoding() throws IOException { + String jobHistoryFile = String.format(JOB_HISTORY_FILE_FORMATTER, + JOB_ID, + SUBMIT_TIME, + USER_NAME, + JOB_NAME, + FINISH_TIME, + NUM_MAPS, + NUM_REDUCES, + JOB_STATUS, + QUEUE_NAME_WITH_DELIMITER_ESCAPE, + JOB_START_TIME ); + + JobIndexInfo info = FileNameIndexUtils.getIndexInfo(jobHistoryFile); + Assert.assertEquals("Queue name doesn't match", + QUEUE_NAME_WITH_DELIMITER, info.getQueueName()); + } + + @Test public void testJobHistoryFileNameBackwardsCompatible() throws IOException { JobID oldJobId = JobID.forName(JOB_ID); JobId jobId = TypeConverter.toYarn(oldJobId);