Return-Path: X-Original-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9FC809007 for ; Thu, 6 Oct 2011 01:17:53 +0000 (UTC) Received: (qmail 81316 invoked by uid 500); 6 Oct 2011 01:17:53 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 81271 invoked by uid 500); 6 Oct 2011 01:17:53 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 81263 invoked by uid 99); 6 Oct 2011 01:17:53 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 06 Oct 2011 01:17:53 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 06 Oct 2011 01:17:39 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id E76CC2388ACC; Thu, 6 Oct 2011 01:17:05 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1179484 [3/6] - in /hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ conf/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/ hadoop-mapreduce-client/hadoop-mapreduce-client-a... Date: Thu, 06 Oct 2011 01:16:57 -0000 To: mapreduce-commits@hadoop.apache.org From: atm@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20111006011705.E76CC2388ACC@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Thu Oct 6 01:16:48 2011 @@ -62,7 +62,6 @@ import org.apache.hadoop.yarn.api.Applic import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationState; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.LocalResource; @@ -70,6 +69,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -99,7 +99,7 @@ public class YARNRunner implements Clien } /** - * Similar to {@link #YARNRunner(Configuration)} but allowing injecting + * Similar to {@link #YARNRunner(Configuration)} but allowing injecting * {@link ResourceMgrDelegate}. Enables mocking and testing. * @param conf the configuration object for the client * @param resMgrDelegate the resourcemanager client handle. @@ -107,12 +107,12 @@ public class YARNRunner implements Clien public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate) { this(conf, resMgrDelegate, new ClientCache(conf, resMgrDelegate)); } - + /** - * Similar to {@link YARNRunner#YARNRunner(Configuration, ResourceMgrDelegate)} + * Similar to {@link YARNRunner#YARNRunner(Configuration, ResourceMgrDelegate)} * but allowing injecting {@link ClientCache}. Enable mocking and testing. * @param conf the configuration object - * @param resMgrDelegate the resource manager delegate + * @param resMgrDelegate the resource manager delegate * @param clientCache the client cache object. */ public YARNRunner(Configuration conf, ResourceMgrDelegate resMgrDelegate, @@ -126,7 +126,7 @@ public class YARNRunner implements Clien throw new RuntimeException("Error in instantiating YarnClient", ufe); } } - + @Override public void cancelDelegationToken(Token arg0) throws IOException, InterruptedException { @@ -152,7 +152,7 @@ public class YARNRunner implements Clien @Override public ClusterMetrics getClusterMetrics() throws IOException, - InterruptedException { + InterruptedException { return resMgrDelegate.getClusterMetrics(); } @@ -209,13 +209,13 @@ public class YARNRunner implements Clien public String getSystemDir() throws IOException, InterruptedException { return resMgrDelegate.getSystemDir(); } - + @Override public long getTaskTrackerExpiryInterval() throws IOException, InterruptedException { return resMgrDelegate.getTaskTrackerExpiryInterval(); } - + @Override public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts) throws IOException, InterruptedException { @@ -230,20 +230,20 @@ public class YARNRunner implements Clien } // Construct necessary information to start the MR AM - ApplicationSubmissionContext appContext = + ApplicationSubmissionContext appContext = createApplicationSubmissionContext(conf, jobSubmitDir, ts); - + // Submit to ResourceManager ApplicationId applicationId = resMgrDelegate.submitApplication(appContext); - + ApplicationReport appMaster = resMgrDelegate .getApplicationReport(applicationId); - String diagnostics = - (appMaster == null ? + String diagnostics = + (appMaster == null ? "application report is null" : appMaster.getDiagnostics()); - if (appMaster == null || appMaster.getState() == ApplicationState.FAILED - || appMaster.getState() == ApplicationState.KILLED) { - throw new IOException("Failed to run job : " + + if (appMaster == null || appMaster.getYarnApplicationState() == YarnApplicationState.FAILED + || appMaster.getYarnApplicationState() == YarnApplicationState.KILLED) { + throw new IOException("Failed to run job : " + diagnostics); } return clientCache.getClient(jobId).getJobStatus(jobId); @@ -266,7 +266,7 @@ public class YARNRunner implements Clien Configuration jobConf, String jobSubmitDir, Credentials ts) throws IOException { ApplicationId applicationId = resMgrDelegate.getApplicationId(); - + // Setup resource requirements Resource capability = recordFactory.newRecordInstance(Resource.class); capability.setMemory(conf.getInt(MRJobConfig.MR_AM_VMEM_MB, @@ -276,9 +276,9 @@ public class YARNRunner implements Clien // Setup LocalResources Map localResources = new HashMap(); - + Path jobConfPath = new Path(jobSubmitDir, MRJobConfig.JOB_CONF_FILE); - + URL yarnUrlForJobSubmitDir = ConverterUtils .getYarnUrlFromPath(defaultFileContext.getDefaultFileSystem() .resolvePath( @@ -299,18 +299,18 @@ public class YARNRunner implements Clien LOG.info("Job jar is not present. " + "Not adding any jar to the list of resources."); } - + // TODO gross hack - for (String s : new String[] { - MRJobConfig.JOB_SPLIT, + for (String s : new String[] { + MRJobConfig.JOB_SPLIT, MRJobConfig.JOB_SPLIT_METAINFO, MRJobConfig.APPLICATION_TOKENS_FILE }) { localResources.put( MRJobConfig.JOB_SUBMIT_DIR + "/" + s, - createApplicationResource(defaultFileContext, + createApplicationResource(defaultFileContext, new Path(jobSubmitDir, s))); } - + // Setup security tokens ByteBuffer securityTokens = null; if (UserGroupInformation.isSecurityEnabled()) { @@ -322,20 +322,20 @@ public class YARNRunner implements Clien // Setup the command to run the AM Vector vargs = new Vector(8); vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); - + long logSize = TaskLog.getTaskLogLength(new JobConf(conf)); vargs.add("-Dlog4j.configuration=container-log4j.properties"); vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR); vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + logSize); - + vargs.add(conf.get(MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS)); vargs.add(MRJobConfig.APPLICATION_MASTER_CLASS); - vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + + vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR + ApplicationConstants.STDOUT); - vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + + vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR + ApplicationConstants.STDERR); @@ -349,12 +349,12 @@ public class YARNRunner implements Clien LOG.info("Command to launch container for ApplicationMaster is : " + mergedCommand); - - // Setup the CLASSPATH in environment + + // Setup the CLASSPATH in environment // i.e. add { job jar, CWD, Hadoop jars} to classpath. Map environment = new HashMap(); MRApps.setClasspath(environment); - + // Parse distributed cache MRApps.setupDistributedCache(jobConf, localResources); @@ -374,12 +374,12 @@ public class YARNRunner implements Clien appContext.setUser( // User name UserGroupInformation.getCurrentUser().getShortUserName()); appContext.setQueue( // Queue name - jobConf.get(JobContext.QUEUE_NAME, + jobConf.get(JobContext.QUEUE_NAME, YarnConfiguration.DEFAULT_QUEUE_NAME)); appContext.setApplicationName( // Job name - jobConf.get(JobContext.JOB_NAME, - YarnConfiguration.DEFAULT_APPLICATION_NAME)); - appContext.setAMContainerSpec(amContainer); // AM Container + jobConf.get(JobContext.JOB_NAME, + YarnConfiguration.DEFAULT_APPLICATION_NAME)); + appContext.setAMContainerSpec(amContainer); // AM Container return appContext; } @@ -394,14 +394,14 @@ public class YARNRunner implements Clien public long getProtocolVersion(String arg0, long arg1) throws IOException { return resMgrDelegate.getProtocolVersion(arg0, arg1); } - + @Override public long renewDelegationToken(Token arg0) throws IOException, InterruptedException { return resMgrDelegate.renewDelegationToken(arg0); } - + @Override public Counters getJobCounters(JobID arg0) throws IOException, InterruptedException { @@ -419,7 +419,7 @@ public class YARNRunner implements Clien JobStatus status = clientCache.getClient(jobID).getJobStatus(jobID); return status; } - + @Override public TaskCompletionEvent[] getTaskCompletionEvents(JobID arg0, int arg1, int arg2) throws IOException, InterruptedException { @@ -446,8 +446,8 @@ public class YARNRunner implements Clien if (status.getState() != JobStatus.State.RUNNING) { resMgrDelegate.killApplication(TypeConverter.toYarn(arg0).getAppId()); return; - } - + } + try { /* send a kill to the AM */ clientCache.getClient(arg0).killJob(arg0); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java Thu Oct 6 01:16:48 2011 @@ -88,7 +88,8 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationState; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -107,17 +108,17 @@ public class TestClientRedirect { private static final Log LOG = LogFactory.getLog(TestClientRedirect.class); private static final String RMADDRESS = "0.0.0.0:8054"; private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - + private static final String AMHOSTADDRESS = "0.0.0.0:10020"; private static final String HSHOSTADDRESS = "0.0.0.0:10021"; - private volatile boolean amContact = false; + private volatile boolean amContact = false; private volatile boolean hsContact = false; private volatile boolean amRunning = false; private volatile boolean amRestarting = false; @Test public void testRedirect() throws Exception { - + Configuration conf = new YarnConfiguration(); conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); conf.set(YarnConfiguration.RM_ADDRESS, RMADDRESS); @@ -125,7 +126,7 @@ public class TestClientRedirect { RMService rmService = new RMService("test"); rmService.init(conf); rmService.start(); - + AMService amService = new AMService(); amService.init(conf); amService.start(conf); @@ -134,16 +135,16 @@ public class TestClientRedirect { HistoryService historyService = new HistoryService(); historyService.init(conf); historyService.start(conf); - + LOG.info("services started"); Cluster cluster = new Cluster(conf); org.apache.hadoop.mapreduce.JobID jobID = new org.apache.hadoop.mapred.JobID("201103121733", 1); - org.apache.hadoop.mapreduce.Counters counters = + org.apache.hadoop.mapreduce.Counters counters = cluster.getJob(jobID).getCounters(); validateCounters(counters); Assert.assertTrue(amContact); - + LOG.info("Sleeping for 5 seconds before stop for" + " the client socket to not get EOF immediately.."); Thread.sleep(5000); @@ -155,17 +156,17 @@ public class TestClientRedirect { LOG.info("Sleeping for 5 seconds after stop for" + " the server to exit cleanly.."); Thread.sleep(5000); - + amRestarting = true; // Same client //results are returned from fake (not started job) counters = cluster.getJob(jobID).getCounters(); Assert.assertEquals(0, counters.countCounters()); Job job = cluster.getJob(jobID); - org.apache.hadoop.mapreduce.TaskID taskId = + org.apache.hadoop.mapreduce.TaskID taskId = new org.apache.hadoop.mapreduce.TaskID(jobID, TaskType.MAP, 0); TaskAttemptID tId = new TaskAttemptID(taskId, 0); - + //invoke all methods to check that no exception is thrown job.killJob(); job.killTask(tId); @@ -175,25 +176,25 @@ public class TestClientRedirect { job.getTaskDiagnostics(tId); job.getTaskReports(TaskType.MAP); job.getTrackingURL(); - + amRestarting = false; amService = new AMService(); amService.init(conf); amService.start(conf); amRunning = true; amContact = false; //reset - + counters = cluster.getJob(jobID).getCounters(); validateCounters(counters); Assert.assertTrue(amContact); - + amRunning = false; // Same client counters = cluster.getJob(jobID).getCounters(); validateCounters(counters); Assert.assertTrue(hsContact); - + rmService.stop(); historyService.stop(); } @@ -248,7 +249,7 @@ public class TestClientRedirect { public GetNewApplicationResponse getNewApplication(GetNewApplicationRequest request) throws YarnRemoteException { return null; } - + @Override public GetApplicationReportResponse getApplicationReport( GetApplicationReportRequest request) throws YarnRemoteException { @@ -256,12 +257,14 @@ public class TestClientRedirect { ApplicationReport application = recordFactory .newRecordInstance(ApplicationReport.class); application.setApplicationId(applicationId); + application.setFinalApplicationStatus(FinalApplicationStatus.UNDEFINED); if (amRunning) { - application.setState(ApplicationState.RUNNING); + application.setYarnApplicationState(YarnApplicationState.RUNNING); } else if (amRestarting) { - application.setState(ApplicationState.SUBMITTED); + application.setYarnApplicationState(YarnApplicationState.SUBMITTED); } else { - application.setState(ApplicationState.SUCCEEDED); + application.setYarnApplicationState(YarnApplicationState.FINISHED); + application.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED); } String[] split = AMHOSTADDRESS.split(":"); application.setHost(split[0]); @@ -339,7 +342,7 @@ public class TestClientRedirect { } } - class AMService extends AbstractService + class AMService extends AbstractService implements MRClientProtocol { private InetSocketAddress bindAddress; private Server server; @@ -347,7 +350,7 @@ public class TestClientRedirect { public AMService() { this(AMHOSTADDRESS); } - + public AMService(String hostAddress) { super("AMService"); this.hostAddress = hostAddress; Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java Thu Oct 6 01:16:48 2011 @@ -32,8 +32,9 @@ import org.apache.hadoop.mapreduce.v2.ap import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportResponse; import org.apache.hadoop.mapreduce.v2.api.records.JobReport; import org.apache.hadoop.mapreduce.v2.api.records.JobState; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationState; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRemoteException; import org.apache.hadoop.yarn.ipc.RPCUtil; @@ -124,6 +125,26 @@ public class TestClientServiceDelegate { Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState()); } + + @Test + public void testJobReportFromHistoryServer() throws Exception { + MRClientProtocol historyServerProxy = mock(MRClientProtocol.class); + when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn( + getJobReportResponseFromHistoryServer()); + ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class); + when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())) + .thenReturn(null); + ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate( + historyServerProxy, rm); + + JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId); + Assert.assertNotNull(jobStatus); + Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile()); + Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl()); + Assert.assertEquals(1.0f, jobStatus.getMapProgress()); + Assert.assertEquals(1.0f, jobStatus.getReduceProgress()); + } + private GetJobReportRequest getJobReportRequest() { GetJobReportRequest request = Records.newRecord(GetJobReportRequest.class); request.setJobId(jobId); @@ -143,7 +164,7 @@ public class TestClientServiceDelegate { private ApplicationReport getApplicationReport() { ApplicationReport applicationReport = Records .newRecord(ApplicationReport.class); - applicationReport.setState(ApplicationState.SUCCEEDED); + applicationReport.setYarnApplicationState(YarnApplicationState.FINISHED); applicationReport.setUser("root"); applicationReport.setHost("N/A"); applicationReport.setName("N/A"); @@ -152,6 +173,7 @@ public class TestClientServiceDelegate { applicationReport.setFinishTime(0); applicationReport.setTrackingUrl("N/A"); applicationReport.setDiagnostics("N/A"); + applicationReport.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED); return applicationReport; } @@ -170,4 +192,17 @@ public class TestClientServiceDelegate { return clientServiceDelegate; } + private GetJobReportResponse getJobReportResponseFromHistoryServer() { + GetJobReportResponse jobReportResponse = Records + .newRecord(GetJobReportResponse.class); + JobReport jobReport = Records.newRecord(JobReport.class); + jobReport.setJobId(jobId); + jobReport.setJobState(JobState.SUCCEEDED); + jobReport.setMapProgress(1.0f); + jobReport.setReduceProgress(1.0f); + jobReport.setJobFile("TestJobFilePath"); + jobReport.setTrackingUrl("TestTrackingUrl"); + jobReportResponse.setJobReport(jobReport); + return jobReportResponse; + } } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java Thu Oct 6 01:16:48 2011 @@ -273,9 +273,9 @@ public class TestMRJobs { TaskCompletionEvent[] events = job.getTaskCompletionEvents(0, 2); Assert.assertEquals(TaskCompletionEvent.Status.FAILED, - events[0].getStatus().FAILED); - Assert.assertEquals(TaskCompletionEvent.Status.FAILED, - events[1].getStatus().FAILED); + events[0].getStatus()); + Assert.assertEquals(TaskCompletionEvent.Status.TIPFAILED, + events[1].getStatus()); Assert.assertEquals(JobStatus.State.FAILED, job.getJobState()); verifyFailingMapperCounters(job); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobsWithHistoryService.java Thu Oct 6 01:16:48 2011 @@ -36,7 +36,6 @@ import org.apache.hadoop.mapreduce.Count import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.TypeConverter; import org.apache.hadoop.yarn.api.records.ApplicationId; -import org.apache.hadoop.yarn.api.records.ApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.junit.Before; import org.junit.After; Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestYARNRunner.java Thu Oct 6 01:16:48 2011 @@ -64,10 +64,10 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoResponse; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationState; import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; import org.apache.hadoop.yarn.api.records.QueueInfo; import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -77,25 +77,25 @@ import org.mockito.invocation.Invocation import org.mockito.stubbing.Answer; /** - * Test YarnRunner and make sure the client side plugin works + * Test YarnRunner and make sure the client side plugin works * fine */ public class TestYARNRunner extends TestCase { private static final Log LOG = LogFactory.getLog(TestYARNRunner.class); private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); - + private YARNRunner yarnRunner; private ResourceMgrDelegate resourceMgrDelegate; private YarnConfiguration conf; private ClientCache clientCache; private ApplicationId appId; private JobID jobId; - private File testWorkDir = + private File testWorkDir = new File("target", TestYARNRunner.class.getName()); private ApplicationSubmissionContext submissionContext; private ClientServiceDelegate clientDelegate; private static final String failString = "Rejected job"; - + @Before public void setUp() throws Exception { resourceMgrDelegate = mock(ResourceMgrDelegate.class); @@ -115,7 +115,7 @@ public class TestYARNRunner extends Test } ).when(yarnRunner).createApplicationSubmissionContext(any(Configuration.class), any(String.class), any(Credentials.class)); - + appId = recordFactory.newRecordInstance(ApplicationId.class); appId.setClusterTimestamp(System.currentTimeMillis()); appId.setId(1); @@ -125,13 +125,13 @@ public class TestYARNRunner extends Test } testWorkDir.mkdirs(); } - - + + @Test public void testJobKill() throws Exception { clientDelegate = mock(ClientServiceDelegate.class); - when(clientDelegate.getJobStatus(any(JobID.class))).thenReturn(new - org.apache.hadoop.mapreduce.JobStatus(jobId, 0f, 0f, 0f, 0f, + when(clientDelegate.getJobStatus(any(JobID.class))).thenReturn(new + org.apache.hadoop.mapreduce.JobStatus(jobId, 0f, 0f, 0f, 0f, State.PREP, JobPriority.HIGH, "tmp", "tmp", "tmp", "tmp")); when(clientDelegate.killJob(any(JobID.class))).thenReturn(true); doAnswer( @@ -145,13 +145,13 @@ public class TestYARNRunner extends Test ).when(clientCache).getClient(any(JobID.class)); yarnRunner.killJob(jobId); verify(resourceMgrDelegate).killApplication(appId); - when(clientDelegate.getJobStatus(any(JobID.class))).thenReturn(new - org.apache.hadoop.mapreduce.JobStatus(jobId, 0f, 0f, 0f, 0f, + when(clientDelegate.getJobStatus(any(JobID.class))).thenReturn(new + org.apache.hadoop.mapreduce.JobStatus(jobId, 0f, 0f, 0f, 0f, State.RUNNING, JobPriority.HIGH, "tmp", "tmp", "tmp", "tmp")); yarnRunner.killJob(jobId); verify(clientDelegate).killJob(jobId); } - + @Test public void testJobSubmissionFailure() throws Exception { when(resourceMgrDelegate.submitApplication(any(ApplicationSubmissionContext.class))). @@ -159,7 +159,7 @@ public class TestYARNRunner extends Test ApplicationReport report = mock(ApplicationReport.class); when(report.getApplicationId()).thenReturn(appId); when(report.getDiagnostics()).thenReturn(failString); - when(report.getState()).thenReturn(ApplicationState.FAILED); + when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.FAILED); when(resourceMgrDelegate.getApplicationReport(appId)).thenReturn(report); Credentials credentials = new Credentials(); File jobxml = new File(testWorkDir, "job.xml"); @@ -167,13 +167,13 @@ public class TestYARNRunner extends Test conf.writeXml(out); out.close(); try { - yarnRunner.submitJob(jobId, testWorkDir.getAbsolutePath().toString(), credentials); + yarnRunner.submitJob(jobId, testWorkDir.getAbsolutePath().toString(), credentials); } catch(IOException io) { LOG.info("Logging exception:", io); assertTrue(io.getLocalizedMessage().contains(failString)); } } - + @Test public void testResourceMgrDelegate() throws Exception { /* we not want a mock of resourcemgr deleagte */ @@ -184,19 +184,19 @@ public class TestYARNRunner extends Test .thenReturn(null); delegate.killApplication(appId); verify(clientRMProtocol).forceKillApplication(any(KillApplicationRequest.class)); - + /* make sure getalljobs calls get all applications */ when(clientRMProtocol.getAllApplications(any(GetAllApplicationsRequest.class))). thenReturn(recordFactory.newRecordInstance(GetAllApplicationsResponse.class)); delegate.getAllJobs(); verify(clientRMProtocol).getAllApplications(any(GetAllApplicationsRequest.class)); - + /* make sure getapplication report is called */ when(clientRMProtocol.getApplicationReport(any(GetApplicationReportRequest.class))) .thenReturn(recordFactory.newRecordInstance(GetApplicationReportResponse.class)); delegate.getApplicationReport(appId); verify(clientRMProtocol).getApplicationReport(any(GetApplicationReportRequest.class)); - + /* make sure metrics is called */ GetClusterMetricsResponse clusterMetricsResponse = recordFactory.newRecordInstance (GetClusterMetricsResponse.class); @@ -206,7 +206,7 @@ public class TestYARNRunner extends Test .thenReturn(clusterMetricsResponse); delegate.getClusterMetrics(); verify(clientRMProtocol).getClusterMetrics(any(GetClusterMetricsRequest.class)); - + when(clientRMProtocol.getClusterNodes(any(GetClusterNodesRequest.class))). thenReturn(recordFactory.newRecordInstance(GetClusterNodesResponse.class)); delegate.getActiveTrackers(); @@ -227,7 +227,7 @@ public class TestYARNRunner extends Test thenReturn(queueInfoResponse); delegate.getQueues(); verify(clientRMProtocol).getQueueInfo(any(GetQueueInfoRequest.class)); - + GetQueueUserAclsInfoResponse aclResponse = recordFactory.newRecordInstance( GetQueueUserAclsInfoResponse.class); when(clientRMProtocol.getQueueUserAcls(any(GetQueueUserAclsInfoRequest.class))) Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn Thu Oct 6 01:16:48 2011 @@ -47,7 +47,7 @@ # YARN_ROOT_LOGGER The root appender. Default is INFO,console # -bin=`dirname "$0"` +bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` . "$bin"/yarn-config.sh Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/bin/yarn-daemon.sh Thu Oct 6 01:16:48 2011 @@ -36,7 +36,7 @@ if [ $# -le 1 ]; then exit 1 fi -bin=`dirname "$0"` +bin=`dirname "${BASH_SOURCE-$0}"` bin=`cd "$bin"; pwd` . "$bin"/yarn-config.sh Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java Thu Oct 6 01:16:48 2011 @@ -22,15 +22,16 @@ import org.apache.hadoop.classification. import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.yarn.api.AMRMProtocol; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; /** - *

The finalization request sent by the ApplicationMaster to + *

The finalization request sent by the ApplicationMaster to * inform the ResourceManager about its completion.

- * + * *

The final request includes details such: *

    *
  • - * {@link ApplicationAttemptId} being managed by the + * {@link ApplicationAttemptId} being managed by the * ApplicationMaster *
    • *
  • Final state of the ApplicationMaster
    • @@ -47,19 +48,19 @@ import org.apache.hadoop.yarn.api.record public interface FinishApplicationMasterRequest { /** - * Get the ApplicationAttemptId being managed by the + * Get the ApplicationAttemptId being managed by the * ApplicationMaster. - * @return ApplicationAttemptId being managed by the + * @return ApplicationAttemptId being managed by the * ApplicationMaster */ @Public @Stable ApplicationAttemptId getApplicationAttemptId(); - + /** - * Set the ApplicationAttemptId being managed by the + * Set the ApplicationAttemptId being managed by the * ApplicationMaster. - * @param applicationAttemptId ApplicationAttemptId being managed + * @param applicationAttemptId ApplicationAttemptId being managed * by the ApplicationMaster */ @Public @@ -72,15 +73,15 @@ public interface FinishApplicationMaster */ @Public @Stable - String getFinalState(); - + FinalApplicationStatus getFinalApplicationStatus(); + /** - * Set final state of the ApplicationMaster - * @param finalState final state of the ApplicationMaster + * Set the finish state of the ApplicationMaster + * @param finishState finish state of the ApplicationMaster */ @Public @Stable - void setFinalState(String finalState); + void setFinishApplicationStatus(FinalApplicationStatus finishState); /** * Get diagnostic information on application failure. @@ -89,7 +90,7 @@ public interface FinishApplicationMaster @Public @Stable String getDiagnostics(); - + /** * Set diagnostic information on application failure. * @param diagnostics diagnostic information on application failure @@ -105,10 +106,10 @@ public interface FinishApplicationMaster @Public @Stable String getTrackingUrl(); - + /** * Set the tracking URLfor the ApplicationMaster - * @param url tracking URLfor the + * @param url tracking URLfor the * ApplicationMaster */ @Public Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java Thu Oct 6 01:16:48 2011 @@ -21,23 +21,24 @@ package org.apache.hadoop.yarn.api.proto import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.ProtoBase; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProtoOrBuilder; -import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterRequestProtoOrBuilder; +import org.apache.hadoop.yarn.util.ProtoUtils; - public class FinishApplicationMasterRequestPBImpl extends ProtoBase implements FinishApplicationMasterRequest { FinishApplicationMasterRequestProto proto = FinishApplicationMasterRequestProto.getDefaultInstance(); FinishApplicationMasterRequestProto.Builder builder = null; boolean viaProto = false; - + private ApplicationAttemptId appAttemptId = null; - - + + public FinishApplicationMasterRequestPBImpl() { builder = FinishApplicationMasterRequestProto.newBuilder(); } @@ -46,7 +47,7 @@ public class FinishApplicationMasterRequ this.proto = proto; viaProto = true; } - + public FinishApplicationMasterRequestProto getProto() { mergeLocalToProto(); proto = viaProto ? proto : builder.build(); @@ -61,7 +62,7 @@ public class FinishApplicationMasterRequ } private void mergeLocalToProto() { - if (viaProto) + if (viaProto) maybeInitBuilder(); mergeLocalToBuilder(); proto = builder.build(); @@ -74,8 +75,7 @@ public class FinishApplicationMasterRequ } viaProto = false; } - - + @Override public ApplicationAttemptId getApplicationAttemptId() { FinishApplicationMasterRequestProtoOrBuilder p = viaProto ? proto : builder; @@ -92,7 +92,7 @@ public class FinishApplicationMasterRequ @Override public void setAppAttemptId(ApplicationAttemptId applicationAttemptId) { maybeInitBuilder(); - if (applicationAttemptId == null) + if (applicationAttemptId == null) builder.clearApplicationAttemptId(); this.appAttemptId = applicationAttemptId; } @@ -122,15 +122,22 @@ public class FinishApplicationMasterRequ } @Override - public String getFinalState() { + public FinalApplicationStatus getFinalApplicationStatus() { FinishApplicationMasterRequestProtoOrBuilder p = viaProto ? proto : builder; - return p.getFinalState(); + if (!p.hasFinalApplicationStatus()) { + return null; + } + return convertFromProtoFormat(p.getFinalApplicationStatus()); } @Override - public void setFinalState(String state) { + public void setFinishApplicationStatus(FinalApplicationStatus finishState) { maybeInitBuilder(); - builder.setFinalState(state); + if (finishState == null) { + builder.clearFinalApplicationStatus(); + return; + } + builder.setFinalApplicationStatus(convertToProtoFormat(finishState)); } private ApplicationAttemptIdPBImpl convertFromProtoFormat(ApplicationAttemptIdProto p) { @@ -141,6 +148,13 @@ public class FinishApplicationMasterRequ return ((ApplicationAttemptIdPBImpl)t).getProto(); } + private FinalApplicationStatus convertFromProtoFormat(FinalApplicationStatusProto s) { + return ProtoUtils.convertFromProtoFormat(s); + } + + private FinalApplicationStatusProto convertToProtoFormat(FinalApplicationStatus s) { + return ProtoUtils.convertToProtoFormat(s); + } -} +} Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java Thu Oct 6 01:16:48 2011 @@ -23,24 +23,22 @@ import org.apache.hadoop.yarn.api.protoc import org.apache.hadoop.yarn.api.records.ProtoBase; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.QueueInfoProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProto; import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProtoOrBuilder; - -public class RegisterApplicationMasterResponsePBImpl -extends ProtoBase +public class RegisterApplicationMasterResponsePBImpl +extends ProtoBase implements RegisterApplicationMasterResponse { - RegisterApplicationMasterResponseProto proto = + RegisterApplicationMasterResponseProto proto = RegisterApplicationMasterResponseProto.getDefaultInstance(); RegisterApplicationMasterResponseProto.Builder builder = null; boolean viaProto = false; - + private Resource minimumResourceCapability; private Resource maximumResourceCapability; - + public RegisterApplicationMasterResponsePBImpl() { builder = RegisterApplicationMasterResponseProto.newBuilder(); } @@ -49,16 +47,16 @@ implements RegisterApplicationMasterResp this.proto = proto; viaProto = true; } - + public RegisterApplicationMasterResponseProto getProto() { mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; return proto; } - + private void mergeLocalToProto() { - if (viaProto) + if (viaProto) maybeInitBuilder(); mergeLocalToBuilder(); proto = builder.build(); @@ -94,7 +92,7 @@ implements RegisterApplicationMasterResp if (!p.hasMaximumCapability()) { return null; } - + this.maximumResourceCapability = convertFromProtoFormat(p.getMaximumCapability()); return this.maximumResourceCapability; } @@ -109,7 +107,7 @@ implements RegisterApplicationMasterResp if (!p.hasMinimumCapability()) { return null; } - + this.minimumResourceCapability = convertFromProtoFormat(p.getMinimumCapability()); return this.minimumResourceCapability; } @@ -140,4 +138,4 @@ implements RegisterApplicationMasterResp return ((ResourcePBImpl)resource).getProto(); } -} +} Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMaster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMaster.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMaster.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationMaster.java Thu Oct 6 01:16:48 2011 @@ -22,38 +22,38 @@ import org.apache.hadoop.classification. import org.apache.hadoop.classification.InterfaceStability.Unstable; /** - * For internal use only... + * For internal use only... */ @Private @Unstable public interface ApplicationMaster { ApplicationId getApplicationId(); void setApplicationId(ApplicationId appId); - + String getHost(); void setHost(String host); - + int getRpcPort(); void setRpcPort(int rpcPort); - + String getTrackingUrl(); void setTrackingUrl(String url); - + ApplicationStatus getStatus(); void setStatus(ApplicationStatus status); - - ApplicationState getState(); - void setState(ApplicationState state); - + + YarnApplicationState getState(); + void setState(YarnApplicationState state); + String getClientToken(); void setClientToken(String clientToken); - + int getAMFailCount(); void setAMFailCount(int amFailCount); - + int getContainerCount(); void setContainerCount(int containerCount); - + String getDiagnostics(); void setDiagnostics(String diagnostics); } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java Thu Oct 6 01:16:48 2011 @@ -36,13 +36,13 @@ import org.apache.hadoop.yarn.api.Client *
  • Host on which the ApplicationMasteris running.
    • *
  • RPC port of the ApplicationMaster.
    • *
  • Tracking URL.
    • - *
  • {@link ApplicationState} of the application.
    • + *
  • {@link YarnApplicationState} of the application.
    • *
  • Diagnostic information in case of errors.
    • *
  • Start time of the application.
    • *
  • Client token of the application (if security is enabled).
    • *
*

- * + * * @see ClientRMProtocol#getApplicationReport(org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest) */ @Public @@ -56,7 +56,7 @@ public interface ApplicationReport { @Public @Stable ApplicationId getApplicationId(); - + @Private @Unstable void setApplicationId(ApplicationId applicationId); @@ -68,7 +68,7 @@ public interface ApplicationReport { @Public @Stable String getUser(); - + @Private @Unstable void setUser(String user); @@ -80,7 +80,7 @@ public interface ApplicationReport { @Public @Stable String getQueue(); - + @Private @Unstable void setQueue(String queue); @@ -92,21 +92,21 @@ public interface ApplicationReport { @Public @Stable String getName(); - + @Private @Unstable void setName(String name); /** - * Get the host on which the ApplicationMaster + * Get the host on which the ApplicationMaster * is running. - * @return host on which the ApplicationMaster + * @return host on which the ApplicationMaster * is running */ @Public @Stable String getHost(); - + @Private @Unstable void setHost(String host); @@ -118,47 +118,47 @@ public interface ApplicationReport { @Public @Stable int getRpcPort(); - + @Private @Unstable void setRpcPort(int rpcPort); /** - * Get the client token for communicating with the + * Get the client token for communicating with the * ApplicationMaster. - * @return client token for communicating with the + * @return client token for communicating with the * ApplicationMaster */ @Public @Stable String getClientToken(); - + @Private @Unstable void setClientToken(String clientToken); /** - * Get the ApplicationState of the application. - * @return ApplicationState of the application + * Get the YarnApplicationState of the application. + * @return YarnApplicationState of the application */ @Public @Stable - ApplicationState getState(); - + YarnApplicationState getYarnApplicationState(); + @Private @Unstable - void setState(ApplicationState state); + void setYarnApplicationState(YarnApplicationState state); /** - * Get the diagnositic information of the application in case of + * Get the diagnositic information of the application in case of * errors. - * @return diagnositic information of the application in case + * @return diagnositic information of the application in case * of errors */ @Public @Stable String getDiagnostics(); - + @Private @Unstable void setDiagnostics(String diagnostics); @@ -170,11 +170,11 @@ public interface ApplicationReport { @Public @Stable String getTrackingUrl(); - + @Private @Unstable void setTrackingUrl(String url); - + /** * Get the start time of the application. * @return start time of the application @@ -182,7 +182,7 @@ public interface ApplicationReport { @Public @Stable long getStartTime(); - + @Private @Unstable void setStartTime(long startTime); @@ -194,8 +194,21 @@ public interface ApplicationReport { @Public @Stable long getFinishTime(); - + @Private @Unstable void setFinishTime(long finishTime); + + + /** + * Get the final finish status of the application. + */ + @Public + @Stable + FinalApplicationStatus getFinalApplicationStatus(); + + @Private + @Unstable + void setFinalApplicationStatus(FinalApplicationStatus finishState); + } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMasterPBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMasterPBImpl.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMasterPBImpl.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationMasterPBImpl.java Thu Oct 6 01:16:48 2011 @@ -19,31 +19,28 @@ package org.apache.hadoop.yarn.api.records.impl.pb; -import java.util.List; - import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationMaster; -import org.apache.hadoop.yarn.api.records.ApplicationState; import org.apache.hadoop.yarn.api.records.ApplicationStatus; import org.apache.hadoop.yarn.api.records.ProtoBase; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationMasterProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationMasterProtoOrBuilder; -import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationStateProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationStatusProto; +import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; import org.apache.hadoop.yarn.util.ProtoUtils; - public class ApplicationMasterPBImpl extends ProtoBase implements ApplicationMaster { ApplicationMasterProto proto = ApplicationMasterProto.getDefaultInstance(); ApplicationMasterProto.Builder builder = null; boolean viaProto = false; - + private ApplicationId applicationId = null; private ApplicationStatus applicationStatus = null; - - + + public ApplicationMasterPBImpl() { builder = ApplicationMasterProto.newBuilder(); } @@ -52,9 +49,9 @@ public class ApplicationMasterPBImpl ext this.proto = proto; viaProto = true; } - + public ApplicationMasterProto getProto() { - + mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; @@ -72,24 +69,24 @@ public class ApplicationMasterPBImpl ext } private void mergeLocalToProto() { - if (viaProto) + if (viaProto) maybeInitBuilder(); mergeLocalToBuilder(); proto = builder.build(); - + viaProto = true; } - + private void maybeInitBuilder() { if (viaProto || builder == null) { builder = ApplicationMasterProto.newBuilder(proto); } viaProto = false; } - - + + @Override - public ApplicationState getState() { + public YarnApplicationState getState() { ApplicationMasterProtoOrBuilder p = viaProto ? proto : builder; if (!p.hasState()) { return null; @@ -98,7 +95,7 @@ public class ApplicationMasterPBImpl ext } @Override - public void setState(ApplicationState state) { + public void setState(YarnApplicationState state) { maybeInitBuilder(); if (state == null) { builder.clearState(); @@ -124,7 +121,7 @@ public class ApplicationMasterPBImpl ext } builder.setHost((host)); } - + @Override public ApplicationId getApplicationId() { ApplicationMasterProtoOrBuilder p = viaProto ? proto : builder; @@ -135,7 +132,7 @@ public class ApplicationMasterPBImpl ext return null; } applicationId = convertFromProtoFormat(p.getApplicationId()); - + return applicationId; } @@ -145,7 +142,7 @@ public class ApplicationMasterPBImpl ext if (applicationId == null) builder.clearApplicationId(); this.applicationId = applicationId; - + } @Override public int getRpcPort() { @@ -179,7 +176,7 @@ public class ApplicationMasterPBImpl ext return null; } this.applicationStatus = convertFromProtoFormat(p.getStatus()); - + return this.applicationStatus; } @@ -189,7 +186,7 @@ public class ApplicationMasterPBImpl ext if (status == null) builder.clearStatus(); this.applicationStatus = status; - + } @Override public String getClientToken() { @@ -209,7 +206,7 @@ public class ApplicationMasterPBImpl ext } builder.setClientToken((clientToken)); } - + @Override public int getAMFailCount() { ApplicationMasterProtoOrBuilder p = viaProto ? proto : builder; @@ -250,11 +247,11 @@ public class ApplicationMasterPBImpl ext builder.setDiagnostics(diagnostics); } - private ApplicationStateProto convertToProtoFormat(ApplicationState e) { + private YarnApplicationStateProto convertToProtoFormat(YarnApplicationState e) { return ProtoUtils.convertToProtoFormat(e); } - private ApplicationState convertFromProtoFormat(ApplicationStateProto e) { + private YarnApplicationState convertFromProtoFormat(YarnApplicationStateProto e) { return ProtoUtils.convertFromProtoFormat(e); } @@ -274,4 +271,4 @@ public class ApplicationMasterPBImpl ext return ((ApplicationStatusPBImpl)t).getProto(); } -} +} Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java Thu Oct 6 01:16:48 2011 @@ -18,17 +18,19 @@ package org.apache.hadoop.yarn.api.records.impl.pb; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationState; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.ProtoBase; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProto; import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationReportProtoOrBuilder; -import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationStateProto; +import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto; +import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; import org.apache.hadoop.yarn.util.ProtoUtils; -public class ApplicationReportPBImpl extends ProtoBase +public class ApplicationReportPBImpl extends ProtoBase implements ApplicationReport { ApplicationReportProto proto = ApplicationReportProto.getDefaultInstance(); ApplicationReportProto.Builder builder = null; @@ -39,7 +41,7 @@ implements ApplicationReport { public ApplicationReportPBImpl() { builder = ApplicationReportProto.newBuilder(); } - + public ApplicationReportPBImpl(ApplicationReportProto proto) { this.proto = proto; viaProto = true; @@ -87,12 +89,12 @@ implements ApplicationReport { } @Override - public ApplicationState getState() { + public YarnApplicationState getYarnApplicationState() { ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; - if (!p.hasState()) { + if (!p.hasYarnApplicationState()) { return null; } - return convertFromProtoFormat(p.getState()); + return convertFromProtoFormat(p.getYarnApplicationState()); } @Override @@ -139,6 +141,27 @@ implements ApplicationReport { } @Override + public long getStartTime() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getStartTime(); + } + + @Override + public long getFinishTime() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + return p.getFinishTime(); + } + + @Override + public FinalApplicationStatus getFinalApplicationStatus() { + ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; + if (!p.hasFinalApplicationStatus()) { + return null; + } + return convertFromProtoFormat(p.getFinalApplicationStatus()); + } + + @Override public void setApplicationId(ApplicationId applicationId) { maybeInitBuilder(); if (applicationId == null) @@ -177,13 +200,13 @@ implements ApplicationReport { } @Override - public void setState(ApplicationState state) { + public void setYarnApplicationState(YarnApplicationState state) { maybeInitBuilder(); if (state == null) { - builder.clearState(); + builder.clearYarnApplicationState(); return; } - builder.setState(convertToProtoFormat(state)); + builder.setYarnApplicationState(convertToProtoFormat(state)); } @Override @@ -233,35 +256,33 @@ implements ApplicationReport { } @Override - public ApplicationReportProto getProto() { - mergeLocalToProto(); - proto = viaProto ? proto : builder.build(); - viaProto = true; - return proto; - } - - @Override - public long getStartTime() { - ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; - return p.getStartTime(); - } - - @Override public void setStartTime(long startTime) { maybeInitBuilder(); builder.setStartTime(startTime); } @Override - public long getFinishTime() { - ApplicationReportProtoOrBuilder p = viaProto ? proto : builder; - return p.getFinishTime(); + public void setFinishTime(long finishTime) { + maybeInitBuilder(); + builder.setFinishTime(finishTime); } @Override - public void setFinishTime(long finishTime) { + public void setFinalApplicationStatus(FinalApplicationStatus finishState) { maybeInitBuilder(); - builder.setFinishTime(finishTime); + if (finishState == null) { + builder.clearFinalApplicationStatus(); + return; + } + builder.setFinalApplicationStatus(convertToProtoFormat(finishState)); + } + + @Override + public ApplicationReportProto getProto() { + mergeLocalToProto(); + proto = viaProto ? proto : builder.build(); + viaProto = true; + return proto; } private void mergeLocalToBuilder() { @@ -291,16 +312,25 @@ implements ApplicationReport { return ((ApplicationIdPBImpl) t).getProto(); } - private ApplicationState convertFromProtoFormat(ApplicationStateProto s) { + private ApplicationIdPBImpl convertFromProtoFormat( + ApplicationIdProto applicationId) { + return new ApplicationIdPBImpl(applicationId); + } + + private YarnApplicationState convertFromProtoFormat(YarnApplicationStateProto s) { return ProtoUtils.convertFromProtoFormat(s); } - private ApplicationStateProto convertToProtoFormat(ApplicationState s) { + private YarnApplicationStateProto convertToProtoFormat(YarnApplicationState s) { return ProtoUtils.convertToProtoFormat(s); } - private ApplicationIdPBImpl convertFromProtoFormat( - ApplicationIdProto applicationId) { - return new ApplicationIdPBImpl(applicationId); + private FinalApplicationStatus convertFromProtoFormat(FinalApplicationStatusProto s) { + return ProtoUtils.convertFromProtoFormat(s); } + + private FinalApplicationStatusProto convertToProtoFormat(FinalApplicationStatus s) { + return ProtoUtils.convertToProtoFormat(s); + } + } Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/ProtoUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/ProtoUtils.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/ProtoUtils.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/ProtoUtils.java Thu Oct 6 01:16:48 2011 @@ -20,24 +20,26 @@ package org.apache.hadoop.yarn.util; import java.nio.ByteBuffer; -import org.apache.hadoop.yarn.api.records.ApplicationState; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.QueueState; -import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationStateProto; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto; +import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceTypeProto; import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceVisibilityProto; import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto; import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto; +import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto; import com.google.protobuf.ByteString; public class ProtoUtils { - - + + /* * ContainerState */ @@ -48,18 +50,29 @@ public class ProtoUtils { public static ContainerState convertFromProtoFormat(ContainerStateProto e) { return ContainerState.valueOf(e.name().replace(CONTAINER_STATE_PREFIX, "")); } - + /* - * ApplicationState + * YarnApplicationState */ - public static ApplicationStateProto convertToProtoFormat(ApplicationState e) { - return ApplicationStateProto.valueOf(e.name()); + public static YarnApplicationStateProto convertToProtoFormat(YarnApplicationState e) { + return YarnApplicationStateProto.valueOf(e.name()); } - public static ApplicationState convertFromProtoFormat(ApplicationStateProto e) { - return ApplicationState.valueOf(e.name()); + public static YarnApplicationState convertFromProtoFormat(YarnApplicationStateProto e) { + return YarnApplicationState.valueOf(e.name()); } - + + /* + * FinalApplicationStatus + */ + private static String FINAL_APPLICATION_STATUS_PREFIX = "APP_"; + public static FinalApplicationStatusProto convertToProtoFormat(FinalApplicationStatus e) { + return FinalApplicationStatusProto.valueOf(FINAL_APPLICATION_STATUS_PREFIX + e.name()); + } + public static FinalApplicationStatus convertFromProtoFormat(FinalApplicationStatusProto e) { + return FinalApplicationStatus.valueOf(e.name().replace(FINAL_APPLICATION_STATUS_PREFIX, "")); + } + /* * LocalResourceType */ @@ -69,7 +82,7 @@ public class ProtoUtils { public static LocalResourceType convertFromProtoFormat(LocalResourceTypeProto e) { return LocalResourceType.valueOf(e.name()); } - + /* * LocalResourceVisibility */ @@ -79,7 +92,7 @@ public class ProtoUtils { public static LocalResourceVisibility convertFromProtoFormat(LocalResourceVisibilityProto e) { return LocalResourceVisibility.valueOf(e.name()); } - + /* * ByteBuffer */ @@ -98,7 +111,7 @@ public class ProtoUtils { byteBuffer.position(oldPos); return bs; } - + /* * QueueState */ @@ -109,7 +122,7 @@ public class ProtoUtils { public static QueueState convertFromProtoFormat(QueueStateProto e) { return QueueState.valueOf(e.name().replace(QUEUE_STATE_PREFIX, "")); } - + /* * QueueACL */ Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Thu Oct 6 01:16:48 2011 @@ -76,14 +76,20 @@ message ContainerProto { optional ContainerStatusProto container_status = 8; } -enum ApplicationStateProto { +enum YarnApplicationStateProto { NEW = 1; SUBMITTED = 2; RUNNING = 3; - RESTARTING = 4; - SUCCEEDED = 5; - FAILED = 6; - KILLED = 7; + FINISHED = 4; + FAILED = 5; + KILLED = 6; +} + +enum FinalApplicationStatusProto { + APP_UNDEFINED = 0; + APP_SUCCEEDED = 1; + APP_FAILED = 2; + APP_KILLED = 3; } message ApplicationStatusProto { @@ -98,7 +104,7 @@ message ApplicationMasterProto { optional int32 rpc_port = 3; optional string trackingUrl = 4; optional ApplicationStatusProto status = 5; - optional ApplicationStateProto state = 6; + optional YarnApplicationStateProto state = 6; optional string client_token = 7; optional int32 containerCount = 8; optional int32 amFailCount = 9; @@ -107,7 +113,7 @@ message ApplicationMasterProto { message URLProto { optional string scheme = 1; - optional string host = 2; + optional string host = 2; optional int32 port = 3; optional string file = 4; } @@ -140,12 +146,13 @@ message ApplicationReportProto { optional int32 rpc_port = 6; optional string client_token = 7; optional ApplicationStatusProto status = 8; - optional ApplicationStateProto state = 9; + optional YarnApplicationStateProto yarn_application_state = 9; optional ContainerProto masterContainer = 10; optional string trackingUrl = 11; optional string diagnostics = 12 [default = "N/A"]; optional int64 startTime = 13; optional int64 finishTime = 14; + optional FinalApplicationStatusProto final_application_status = 15; } message NodeIdProto { @@ -195,7 +202,7 @@ message AMResponseProto { message ApplicationSubmissionContextProto { optional ApplicationIdProto application_id = 1; optional string application_name = 2 [default = "N/A"]; - optional string user = 3; + optional string user = 3; optional string queue = 4 [default = "default"]; optional PriorityProto priority = 5; optional ContainerLaunchContextProto am_container_spec = 6; Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto Thu Oct 6 01:16:48 2011 @@ -42,7 +42,7 @@ message FinishApplicationMasterRequestPr optional ApplicationAttemptIdProto application_attempt_id = 1; optional string diagnostics = 2; optional string tracking_url = 3; - optional string final_state = 4; + optional FinalApplicationStatusProto final_application_status = 4; } message FinishApplicationMasterResponseProto { @@ -115,7 +115,7 @@ message GetClusterNodesRequestProto { } message GetClusterNodesResponseProto { - repeated NodeReportProto nodeReports = 1; + repeated NodeReportProto nodeReports = 1; } message GetQueueInfoRequestProto { @@ -133,7 +133,7 @@ message GetQueueUserAclsInfoRequestProto } message GetQueueUserAclsInfoResponseProto { - repeated QueueUserACLInfoProto queueUserAcls = 1; + repeated QueueUserACLInfoProto queueUserAcls = 1; } ////////////////////////////////////////////////////// Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Thu Oct 6 01:16:48 2011 @@ -44,14 +44,7 @@ public class YarnConfiguration extends C /** ACL of who can modify this application.*/ public static final String APPLICATION_ACL_MODIFY_APP = "yarn.app.acl.modify-job"; - - /** - * Security info class This is an internal config set and - * read by YARN itself. - */ - public static final String YARN_SECURITY_INFO = - "yarn.security.info.class"; - + /** Delay before deleting resource to ease debugging of NM issues */ public static final String DEBUG_NM_DELETE_DELAY_SEC = YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec"; Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnProtoRPC.java Thu Oct 6 01:16:48 2011 @@ -45,12 +45,9 @@ public class HadoopYarnProtoRPC extends @Override public Object getProxy(Class protocol, InetSocketAddress addr, Configuration conf) { - Configuration myConf = new Configuration(conf); LOG.info("Creating a HadoopYarnProtoRpc proxy for protocol " + protocol); - LOG.debug("Configured SecurityInfo class name is " - + myConf.get(YarnConfiguration.YARN_SECURITY_INFO)); - - return RpcFactoryProvider.getClientFactory(myConf).getClient(protocol, 1, addr, myConf); + return RpcFactoryProvider.getClientFactory(conf).getClient(protocol, 1, + addr, conf); } @Override @@ -60,8 +57,6 @@ public class HadoopYarnProtoRPC extends int numHandlers) { LOG.info("Creating a HadoopYarnProtoRpc server for protocol " + protocol + " with " + numHandlers + " handlers"); - LOG.info("Configured SecurityInfo class name is " - + conf.get(YarnConfiguration.YARN_SECURITY_INFO)); final RPC.Server hadoopServer; hadoopServer = Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnRPC.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnRPC.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnRPC.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/ipc/HadoopYarnRPC.java Thu Oct 6 01:16:48 2011 @@ -46,13 +46,10 @@ public class HadoopYarnRPC extends YarnR @Override public Object getProxy(Class protocol, InetSocketAddress addr, Configuration conf) { - Configuration myConf = new Configuration(conf); LOG.info("Creating a HadoopYarnRpc proxy for protocol " + protocol); - LOG.debug("Configured SecurityInfo class name is " - + myConf.get(YarnConfiguration.YARN_SECURITY_INFO)); - RPC.setProtocolEngine(myConf, protocol, AvroSpecificRpcEngine.class); + RPC.setProtocolEngine(conf, protocol, AvroSpecificRpcEngine.class); try { - return RPC.getProxy(protocol, 1, addr, myConf); + return RPC.getProxy(protocol, 1, addr, conf); } catch (IOException e) { throw new YarnException(e); } @@ -65,8 +62,6 @@ public class HadoopYarnRPC extends YarnR int numHandlers) { LOG.info("Creating a HadoopYarnRpc server for protocol " + protocol + " with " + numHandlers + " handlers"); - LOG.info("Configured SecurityInfo class name is " - + conf.get(YarnConfiguration.YARN_SECURITY_INFO)); RPC.setProtocolEngine(conf, protocol, AvroSpecificRpcEngine.class); final RPC.Server hadoopServer; try { Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java Thu Oct 6 01:16:48 2011 @@ -44,7 +44,8 @@ public class ContainerTokenIdentifier ex private String nmHostName; private Resource resource; - public ContainerTokenIdentifier(ContainerId containerID, String hostName, Resource r) { + public ContainerTokenIdentifier(ContainerId containerID, String hostName, + Resource r) { this.containerId = containerID; this.nmHostName = hostName; this.resource = r; Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java Thu Oct 6 01:16:48 2011 @@ -24,9 +24,10 @@ import java.util.List; import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; -import org.apache.hadoop.yarn.api.records.ApplicationState; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerState; @@ -69,12 +70,12 @@ public class BuilderUtils { } } - public static class ResourceRequestComparator + public static class ResourceRequestComparator implements java.util.Comparator { @Override public int compare(org.apache.hadoop.yarn.api.records.ResourceRequest r1, org.apache.hadoop.yarn.api.records.ResourceRequest r2) { - + // Compare priority, host and capability int ret = r1.getPriority().compareTo(r2.getPriority()); if (ret == 0) { @@ -198,12 +199,12 @@ public class BuilderUtils { String nodeHttpAddress, Resource resource, Priority priority) { ContainerId containerID = newContainerId(recordFactory, appAttemptId, containerId); - return newContainer(containerID, nodeId, nodeHttpAddress, + return newContainer(containerID, nodeId, nodeHttpAddress, resource, priority); } public static Container newContainer(ContainerId containerId, - NodeId nodeId, String nodeHttpAddress, + NodeId nodeId, String nodeHttpAddress, Resource resource, Priority priority) { Container container = recordFactory.newRecordInstance(Container.class); container.setId(containerId); @@ -242,8 +243,9 @@ public class BuilderUtils { public static ApplicationReport newApplicationReport( ApplicationId applicationId, String user, String queue, String name, - String host, int rpcPort, String clientToken, ApplicationState state, - String diagnostics, String url, long startTime, long finishTime) { + String host, int rpcPort, String clientToken, YarnApplicationState state, + String diagnostics, String url, long startTime, long finishTime, + FinalApplicationStatus finalStatus) { ApplicationReport report = recordFactory .newRecordInstance(ApplicationReport.class); report.setApplicationId(applicationId); @@ -253,20 +255,21 @@ public class BuilderUtils { report.setHost(host); report.setRpcPort(rpcPort); report.setClientToken(clientToken); - report.setState(state); + report.setYarnApplicationState(state); report.setDiagnostics(diagnostics); report.setTrackingUrl(url); report.setStartTime(startTime); report.setFinishTime(finishTime); + report.setFinalApplicationStatus(finalStatus); return report; } - + public static Resource newResource(int memory) { Resource resource = recordFactory.newRecordInstance(Resource.class); resource.setMemory(memory); return resource; } - + public static URL newURL(String scheme, String host, int port, String file) { URL url = recordFactory.newRecordInstance(URL.class); url.setScheme(scheme); Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java?rev=1179484&r1=1179483&r2=1179484&view=diff ============================================================================== --- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java (original) +++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java Thu Oct 6 01:16:48 2011 @@ -33,7 +33,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.URL; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -181,4 +183,5 @@ public class ConverterUtils { + applicationAttmeptIdStr, n); } } + }