Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id C5D1C18365 for ; Wed, 16 Dec 2015 23:12:06 +0000 (UTC) Received: (qmail 85314 invoked by uid 500); 16 Dec 2015 23:12:06 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 85184 invoked by uid 500); 16 Dec 2015 23:12:06 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 85156 invoked by uid 99); 16 Dec 2015 23:12:05 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 16 Dec 2015 23:12:05 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A9188DFFF0; Wed, 16 Dec 2015 23:12:05 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vinodkv@apache.org To: common-commits@hadoop.apache.org Date: Wed, 16 Dec 2015 23:12:05 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/3] hadoop git commit: Revert "MAPREDUCE-6566. Add retry support to mapreduce CLI tool. Contributed by Varun Vasudev" Repository: hadoop Updated Branches: refs/heads/trunk ce16541c6 -> 60fe6d53d http://git-wip-us.apache.org/repos/asf/hadoop/blob/60fe6d53/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java index db87d9d..59f5ada 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/tools/CLI.java @@ -26,7 +26,6 @@ import java.util.Set; import java.util.HashSet; import java.util.Arrays; -import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -44,7 +43,6 @@ import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.JobPriority; import org.apache.hadoop.mapreduce.JobStatus; -import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskCompletionEvent; import org.apache.hadoop.mapreduce.TaskReport; @@ -270,7 +268,7 @@ public class CLI extends Configured implements Tool { System.out.println("Created job " + job.getJobID()); exitCode = 0; } else if (getStatus) { - Job job = getJob(JobID.forName(jobid)); + Job job = cluster.getJob(JobID.forName(jobid)); if (job == null) { System.out.println("Could not find job " + jobid); } else { @@ -285,7 +283,7 @@ public class CLI extends Configured implements Tool { exitCode = 0; } } else if (getCounter) { - Job job = getJob(JobID.forName(jobid)); + Job job = cluster.getJob(JobID.forName(jobid)); if (job == null) { System.out.println("Could not find job " + jobid); } else { @@ -301,7 +299,7 @@ public class CLI extends Configured implements Tool { } } } else if (killJob) { - Job job = getJob(JobID.forName(jobid)); + Job job = cluster.getJob(JobID.forName(jobid)); if (job == null) { System.out.println("Could not find job " + jobid); } else { @@ -325,7 +323,7 @@ public class CLI extends Configured implements Tool { } } } else if (setJobPriority) { - Job job = getJob(JobID.forName(jobid)); + Job job = cluster.getJob(JobID.forName(jobid)); if (job == null) { System.out.println("Could not find job " + jobid); } else { @@ -341,7 +339,7 @@ public class CLI extends Configured implements Tool { viewHistory(historyFile, viewAllHistory); exitCode = 0; } else if (listEvents) { - listEvents(getJob(JobID.forName(jobid)), fromEvent, nEvents); + listEvents(cluster.getJob(JobID.forName(jobid)), fromEvent, nEvents); exitCode = 0; } else if (listJobs) { listJobs(cluster); @@ -356,11 +354,11 @@ public class CLI extends Configured implements Tool { listBlacklistedTrackers(cluster); exitCode = 0; } else if (displayTasks) { - displayTasks(getJob(JobID.forName(jobid)), taskType, taskState); + displayTasks(cluster.getJob(JobID.forName(jobid)), taskType, taskState); exitCode = 0; } else if(killTask) { TaskAttemptID taskID = TaskAttemptID.forName(taskid); - Job job = getJob(taskID.getJobID()); + Job job = cluster.getJob(taskID.getJobID()); if (job == null) { System.out.println("Could not find job " + jobid); } else if (job.killTask(taskID, false)) { @@ -372,7 +370,7 @@ public class CLI extends Configured implements Tool { } } else if(failTask) { TaskAttemptID taskID = TaskAttemptID.forName(taskid); - Job job = getJob(taskID.getJobID()); + Job job = cluster.getJob(taskID.getJobID()); if (job == null) { System.out.println("Could not find job " + jobid); } else if(job.killTask(taskID, true)) { @@ -533,29 +531,6 @@ public class CLI extends Configured implements Tool { protected static String getTaskLogURL(TaskAttemptID taskId, String baseUrl) { return (baseUrl + "/tasklog?plaintext=true&attemptid=" + taskId); } - - @VisibleForTesting - Job getJob(JobID jobid) throws IOException, InterruptedException { - - int maxRetry = getConf().getInt(MRJobConfig.MR_CLIENT_JOB_MAX_RETRIES, - MRJobConfig.DEFAULT_MR_CLIENT_JOB_MAX_RETRIES); - long retryInterval = getConf() - .getLong(MRJobConfig.MR_CLIENT_JOB_RETRY_INTERVAL, - MRJobConfig.DEFAULT_MR_CLIENT_JOB_RETRY_INTERVAL); - Job job = cluster.getJob(jobid); - - for (int i = 0; i < maxRetry; ++i) { - if (job != null) { - return job; - } - LOG.info("Could not obtain job info after " + String.valueOf(i + 1) - + " attempt(s). Sleeping for " + String.valueOf(retryInterval / 1000) - + " seconds and retrying."); - Thread.sleep(retryInterval); - job = cluster.getJob(jobid); - } - return job; - } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/60fe6d53/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/tools/TestCLI.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/tools/TestCLI.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/tools/TestCLI.java index 73f57d5..fdc916e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/tools/TestCLI.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/tools/TestCLI.java @@ -20,19 +20,14 @@ package org.apache.hadoop.mapreduce.tools; import static org.junit.Assert.*; import java.io.IOException; - -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.mapreduce.Cluster; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobID; -import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.TaskReport; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.JobPriority; import org.apache.hadoop.mapreduce.JobStatus; import org.apache.hadoop.mapreduce.JobStatus.State; -import org.apache.hadoop.util.Time; -import org.junit.Assert; import org.junit.Test; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -49,7 +44,7 @@ public class TestCLI { JobID jobId = JobID.forName(jobIdStr); Cluster mockCluster = mock(Cluster.class); Job job = mock(Job.class); - CLI cli = spy(new CLI(new Configuration())); + CLI cli = spy(new CLI()); doReturn(mockCluster).when(cli).createCluster(); when(job.getTaskReports(TaskType.MAP)).thenReturn( @@ -117,7 +112,7 @@ public class TestCLI { @Test public void testJobKIll() throws Exception { Cluster mockCluster = mock(Cluster.class); - CLI cli = spy(new CLI(new Configuration())); + CLI cli = spy(new CLI()); doReturn(mockCluster).when(cli).createCluster(); String jobId1 = "job_1234654654_001"; String jobId2 = "job_1234654654_002"; @@ -154,26 +149,4 @@ public class TestCLI { when(mockJob.getStatus()).thenReturn(status); return mockJob; } - - @Test - public void testGetJob() throws Exception { - Configuration conf = new Configuration(); - long sleepTime = 100; - conf.setLong(MRJobConfig.MR_CLIENT_JOB_RETRY_INTERVAL, sleepTime); - Cluster mockCluster = mock(Cluster.class); - JobID jobId1 = JobID.forName("job_1234654654_001"); - when(mockCluster.getJob(jobId1)).thenReturn(null); - - for (int i = 0; i < 2; ++i) { - conf.setInt(MRJobConfig.MR_CLIENT_JOB_MAX_RETRIES, i); - CLI cli = spy(new CLI(conf)); - cli.cluster = mockCluster; - doReturn(mockCluster).when(cli).createCluster(); - long start = Time.monotonicNow(); - cli.getJob(jobId1); - long end = Time.monotonicNow(); - Assert.assertTrue(end - start > (i * sleepTime)); - Assert.assertTrue(end - start < ((i + 1) * sleepTime)); - } - } }