Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id ED69ED3E9 for ; Thu, 28 Jun 2012 19:29:06 +0000 (UTC) Received: (qmail 98895 invoked by uid 500); 28 Jun 2012 19:29:06 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 98838 invoked by uid 500); 28 Jun 2012 19:29:06 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 98821 invoked by uid 99); 28 Jun 2012 19:29:06 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Jun 2012 19:29:06 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 Jun 2012 19:29:04 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 818A923889E1; Thu, 28 Jun 2012 19:28:44 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1355124 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java... Date: Thu, 28 Jun 2012 19:28:43 -0000 To: mapreduce-commits@hadoop.apache.org From: bobby@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120628192844.818A923889E1@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: bobby Date: Thu Jun 28 19:28:42 2012 New Revision: 1355124 URL: http://svn.apache.org/viewvc?rev=1355124&view=rev Log: MAPREDUCE-4376. TestClusterMRNotification times out (Kihwal Lee via bobby) Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1355124&r1=1355123&r2=1355124&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Thu Jun 28 19:28:42 2012 @@ -194,6 +194,8 @@ Branch-2 ( Unreleased changes ) MAPREDUCE-4372. Deadlock in Resource Manager (Devaraj K via bobby) + MAPREDUCE-4376. TestClusterMRNotification times out (Kihwal Lee via bobby) + Release 2.0.0-alpha - 05-23-2012 INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1355124&r1=1355123&r2=1355124&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Thu Jun 28 19:28:42 2012 @@ -1097,10 +1097,17 @@ public class RMContainerAllocator extend } ContainerId get(TaskAttemptId tId) { + Container taskContainer; if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) { - return maps.get(tId).getId(); + taskContainer = maps.get(tId); } else { - return reduces.get(tId).getId(); + taskContainer = reduces.get(tId); + } + + if (taskContainer == null) { + return null; + } else { + return taskContainer.getId(); } } } Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java?rev=1355124&r1=1355123&r2=1355124&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/NotificationTestCase.java Thu Jun 28 19:28:42 2012 @@ -91,26 +91,22 @@ public abstract class NotificationTestCa public static class NotificationServlet extends HttpServlet { public static int counter = 0; + public static int failureCounter = 0; private static final long serialVersionUID = 1L; protected void doGet(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException { + String queryString = req.getQueryString(); switch (counter) { case 0: - { - assertTrue(req.getQueryString().contains("SUCCEEDED")); - } - break; + verifyQuery(queryString, "SUCCEEDED"); + break; case 2: - { - assertTrue(req.getQueryString().contains("KILLED")); - } - break; + verifyQuery(queryString, "KILLED"); + break; case 4: - { - assertTrue(req.getQueryString().contains("FAILED")); - } - break; + verifyQuery(queryString, "FAILED"); + break; } if (counter % 2 == 0) { res.sendError(HttpServletResponse.SC_BAD_REQUEST, "forcing error"); @@ -120,6 +116,15 @@ public abstract class NotificationTestCa } counter++; } + + protected void verifyQuery(String query, String expected) + throws IOException { + if (query.contains(expected)) { + return; + } + failureCounter++; + assertTrue("The request (" + query + ") does not contain " + expected, false); + } } private String getNotificationUrlTemplate() { @@ -147,10 +152,12 @@ public abstract class NotificationTestCa } public void testMR() throws Exception { + System.out.println(launchWordCount(this.createJobConf(), "a b c d e f g h", 1, 1)); Thread.sleep(2000); assertEquals(2, NotificationServlet.counter); + assertEquals(0, NotificationServlet.failureCounter); Path inDir = new Path("notificationjob/input"); Path outDir = new Path("notificationjob/output"); @@ -168,12 +175,14 @@ public abstract class NotificationTestCa outDir).getID()); Thread.sleep(2000); assertEquals(4, NotificationServlet.counter); + assertEquals(0, NotificationServlet.failureCounter); // run a job with FAILED status System.out.println(UtilsForTests.runJobFail(this.createJobConf(), inDir, outDir).getID()); Thread.sleep(2000); assertEquals(6, NotificationServlet.counter); + assertEquals(0, NotificationServlet.failureCounter); } private String launchWordCount(JobConf conf, Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java?rev=1355124&r1=1355123&r2=1355124&view=diff ============================================================================== --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java Thu Jun 28 19:28:42 2012 @@ -606,9 +606,14 @@ public class UtilsForTests { conf.setReducerClass(IdentityReducer.class); RunningJob job = UtilsForTests.runJob(conf, inDir, outDir); + long sleepCount = 0; while (!job.isComplete()) { try { + if (sleepCount > 300) { // 30 seconds + throw new IOException("Job didn't finish in 30 seconds"); + } Thread.sleep(100); + sleepCount++; } catch (InterruptedException e) { break; } @@ -626,9 +631,14 @@ public class UtilsForTests { conf.setMaxMapAttempts(1); RunningJob job = UtilsForTests.runJob(conf, inDir, outDir); + long sleepCount = 0; while (!job.isComplete()) { try { + if (sleepCount > 300) { // 30 seconds + throw new IOException("Job didn't finish in 30 seconds"); + } Thread.sleep(100); + sleepCount++; } catch (InterruptedException e) { break; } @@ -646,17 +656,27 @@ public class UtilsForTests { conf.setReducerClass(IdentityReducer.class); RunningJob job = UtilsForTests.runJob(conf, inDir, outDir); + long sleepCount = 0; while (job.getJobState() != JobStatus.RUNNING) { try { + if (sleepCount > 300) { // 30 seconds + throw new IOException("Job didn't finish in 30 seconds"); + } Thread.sleep(100); + sleepCount++; } catch (InterruptedException e) { break; } } job.killJob(); + sleepCount = 0; while (job.cleanupProgress() == 0.0f) { try { + if (sleepCount > 2000) { // 20 seconds + throw new IOException("Job cleanup didn't start in 20 seconds"); + } Thread.sleep(10); + sleepCount++; } catch (InterruptedException ie) { break; }