Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0963E94DB for ; Fri, 13 Jul 2012 20:47:23 +0000 (UTC) Received: (qmail 17972 invoked by uid 500); 13 Jul 2012 20:47:22 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 17913 invoked by uid 500); 13 Jul 2012 20:47:22 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 17904 invoked by uid 99); 13 Jul 2012 20:47:22 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 13 Jul 2012 20:47:22 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 13 Jul 2012 20:47:21 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id B96252388860; Fri, 13 Jul 2012 20:47:01 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1361392 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/sr... Date: Fri, 13 Jul 2012 20:47:01 -0000 To: mapreduce-commits@hadoop.apache.org From: bobby@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120713204701.B96252388860@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: bobby Date: Fri Jul 13 20:47:00 2012 New Revision: 1361392 URL: http://svn.apache.org/viewvc?rev=1361392&view=rev Log: svn merge -c 1361389 FIXES: MAPREDUCE-4419. ./mapred queue -info -showJobs displays all the jobs irrespective of (Devaraj K via bobby) Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1361392&r1=1361391&r2=1361392&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri Jul 13 20:47:00 2012 @@ -322,6 +322,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-3348. Fixed a bug in MR client to redirect to JobHistoryServer correctly when RM forgets the app. (Devaraj K via vinodkv) + MAPREDUCE-4419. ./mapred queue -info -showJobs displays all + the jobs irrespective of (Devaraj K via bobby) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java?rev=1361392&r1=1361391&r2=1361392&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobQueueClient.java Fri Jul 13 20:47:00 2012 @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Arrays; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; @@ -184,7 +185,7 @@ class JobQueueClient extends Configured printJobQueueInfo(jobQueueInfo, new PrintWriter(System.out)); if (showJobs && (jobQueueInfo.getChildren() == null || jobQueueInfo.getChildren().size() == 0)) { - JobStatus[] jobs = jc.getJobsFromQueue(queue); + JobStatus[] jobs = jobQueueInfo.getJobStatuses(); if (jobs == null) jobs = new JobStatus[0]; jc.displayJobList(jobs); Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java?rev=1361392&r1=1361391&r2=1361392&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobStatus.java Fri Jul 13 20:47:00 2012 @@ -238,7 +238,7 @@ public class JobStatus extends org.apach stat.getSetupProgress(), stat.getMapProgress(), stat.getReduceProgress(), stat.getCleanupProgress(), stat.getState().getValue(), JobPriority.valueOf(stat.getPriority().name()), - stat.getUsername(), stat.getJobName(), stat.getJobFile(), + stat.getUsername(), stat.getJobName(), stat.getQueue(), stat.getJobFile(), stat.getTrackingUrl(), stat.isUber()); old.setStartTime(stat.getStartTime()); old.setFinishTime(stat.getFinishTime()); Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1361392&r1=1361391&r2=1361392&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Fri Jul 13 20:47:00 2012 @@ -389,7 +389,9 @@ public class ClientRMService extends Abs appReports = new ArrayList( apps.size()); for (RMApp app : apps) { - appReports.add(app.createAndGetApplicationReport(true)); + if (app.getQueue().equals(queueInfo.getQueueName())) { + appReports.add(app.createAndGetApplicationReport(true)); + } } } queueInfo.setApplications(appReports); Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java?rev=1361392&r1=1361391&r2=1361392&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java Fri Jul 13 20:47:00 2012 @@ -20,7 +20,10 @@ package org.apache.hadoop.yarn.server.re import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyString; +import java.io.IOException; import java.net.InetSocketAddress; import java.util.List; import java.util.concurrent.ConcurrentHashMap; @@ -34,13 +37,21 @@ import org.apache.hadoop.yarn.api.Client import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse; import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest; +import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.NodeReport; +import org.apache.hadoop.yarn.api.records.QueueInfo; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.util.Records; import org.junit.Test; @@ -49,6 +60,9 @@ public class TestClientRMService { private static final Log LOG = LogFactory.getLog(TestClientRMService.class); + private RecordFactory recordFactory = RecordFactoryProvider + .getRecordFactory(null); + @Test public void testGetClusterNodes() throws Exception { MockRM rm = new MockRM() { @@ -109,4 +123,66 @@ public class TestClientRMService { Assert.assertNull("It should return null as application report for absent application.", applicationReport.getApplicationReport()); } + + @Test + public void testGetQueueInfo() throws Exception { + YarnScheduler yarnScheduler = mock(YarnScheduler.class); + RMContext rmContext = mock(RMContext.class); + mockRMContext(yarnScheduler, rmContext); + ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, + null, null, null); + GetQueueInfoRequest request = recordFactory + .newRecordInstance(GetQueueInfoRequest.class); + request.setQueueName("testqueue"); + request.setIncludeApplications(true); + GetQueueInfoResponse queueInfo = rmService.getQueueInfo(request); + List applications = queueInfo.getQueueInfo() + .getApplications(); + Assert.assertEquals(2, applications.size()); + } + + private void mockRMContext(YarnScheduler yarnScheduler, RMContext rmContext) + throws IOException { + Dispatcher dispatcher = mock(Dispatcher.class); + when(rmContext.getDispatcher()).thenReturn(dispatcher); + QueueInfo queInfo = recordFactory.newRecordInstance(QueueInfo.class); + queInfo.setQueueName("testqueue"); + when(yarnScheduler.getQueueInfo(anyString(), anyBoolean(), anyBoolean())) + .thenReturn(queInfo); + ConcurrentHashMap apps = getRMApps(rmContext, + yarnScheduler); + when(rmContext.getRMApps()).thenReturn(apps); + } + + private ConcurrentHashMap getRMApps( + RMContext rmContext, YarnScheduler yarnScheduler) { + ConcurrentHashMap apps = + new ConcurrentHashMap(); + ApplicationId applicationId1 = getApplicationId(1); + ApplicationId applicationId2 = getApplicationId(2); + ApplicationId applicationId3 = getApplicationId(3); + YarnConfiguration config = new YarnConfiguration(); + apps.put(applicationId1, getRMApp(rmContext, yarnScheduler, applicationId1, + config, "testqueue")); + apps.put(applicationId2, getRMApp(rmContext, yarnScheduler, applicationId2, + config, "a")); + apps.put(applicationId3, getRMApp(rmContext, yarnScheduler, applicationId3, + config, "testqueue")); + return apps; + } + + private ApplicationId getApplicationId(int id) { + ApplicationId applicationId = recordFactory + .newRecordInstance(ApplicationId.class); + applicationId.setClusterTimestamp(123456); + applicationId.setId(id); + return applicationId; + } + + private RMAppImpl getRMApp(RMContext rmContext, YarnScheduler yarnScheduler, + ApplicationId applicationId3, YarnConfiguration config, String queueName) { + return new RMAppImpl(applicationId3, rmContext, config, null, null, + queueName, null, null, null, yarnScheduler, null, System + .currentTimeMillis()); + } }