Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 133C017B03 for ; Tue, 24 Mar 2015 15:58:19 +0000 (UTC) Received: (qmail 94802 invoked by uid 500); 24 Mar 2015 15:57:44 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 94735 invoked by uid 500); 24 Mar 2015 15:57:44 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 94726 invoked by uid 99); 24 Mar 2015 15:57:44 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 24 Mar 2015 15:57:44 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 38F4DE05E1; Tue, 24 Mar 2015 15:57:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ozawa@apache.org To: common-commits@hadoop.apache.org Message-Id: <33704e4a21f04663b2a05cead399315d@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: MAPREDUCE-6285. ClientServiceDelegate should not retry upon AuthenticationException. Contributed by Jonathan Eagles. Date: Tue, 24 Mar 2015 15:57:44 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk 3ca5bd163 -> 4170c9914 MAPREDUCE-6285. ClientServiceDelegate should not retry upon AuthenticationException. Contributed by Jonathan Eagles. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4170c991 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4170c991 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4170c991 Branch: refs/heads/trunk Commit: 4170c99147b0cb6d561ff626cea140e0a061b314 Parents: 3ca5bd1 Author: Tsuyoshi Ozawa Authored: Wed Mar 25 00:56:26 2015 +0900 Committer: Tsuyoshi Ozawa Committed: Wed Mar 25 00:56:26 2015 +0900 ---------------------------------------------------------------------- hadoop-mapreduce-project/CHANGES.txt | 3 ++ .../hadoop/mapred/ClientServiceDelegate.java | 6 +++ .../mapred/TestClientServiceDelegate.java | 44 ++++++++++++++++++++ 3 files changed, 53 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/4170c991/hadoop-mapreduce-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index b8a2a1c..2b16c30 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -496,6 +496,9 @@ Release 2.7.0 - UNRELEASED MAPREDUCE-6275. Race condition in FileOutputCommitter v2 for user-specified task output subdirs (Gera Shegalov and Siqi Li via jlowe) + MAPREDUCE-6285. ClientServiceDelegate should not retry upon + AuthenticationException. (Jonathan Eagles via ozawa) + Release 2.6.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/4170c991/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java index 686fa0c..8517c19 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java @@ -64,6 +64,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport; import org.apache.hadoop.mapreduce.v2.util.MRApps; import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -328,6 +329,11 @@ public class ClientServiceDelegate { // Force reconnection by setting the proxy to null. realProxy = null; // HS/AMS shut down + + if (e.getCause() instanceof AuthorizationException) { + throw new IOException(e.getTargetException()); + } + // if it's AM shut down, do not decrement maxClientRetry as we wait for // AM to be restarted. if (!usingAMProxy.get()) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/4170c991/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java index 7d6b2f3..b85f18d 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java @@ -48,6 +48,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.Counters; import org.apache.hadoop.mapreduce.v2.api.records.JobReport; import org.apache.hadoop.mapreduce.v2.api.records.JobState; import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils; +import org.apache.hadoop.security.authorize.AuthorizationException; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -183,6 +184,49 @@ public class TestClientServiceDelegate { } @Test + public void testNoRetryOnAMAuthorizationException() throws Exception { + if (!isAMReachableFromClient) { + return; + } + + ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class); + when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId())) + .thenReturn(getRunningApplicationReport("am1", 78)); + + // throw authorization exception on first invocation + final MRClientProtocol amProxy = mock(MRClientProtocol.class); + when(amProxy.getJobReport(any(GetJobReportRequest.class))) + .thenThrow(new AuthorizationException("Denied")); + Configuration conf = new YarnConfiguration(); + conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); + conf.setBoolean(MRJobConfig.JOB_AM_ACCESS_DISABLED, + !isAMReachableFromClient); + ClientServiceDelegate clientServiceDelegate = + new ClientServiceDelegate(conf, rm, oldJobId, null) { + @Override + MRClientProtocol instantiateAMProxy( + final InetSocketAddress serviceAddr) throws IOException { + super.instantiateAMProxy(serviceAddr); + return amProxy; + } + }; + + try { + clientServiceDelegate.getJobStatus(oldJobId); + Assert.fail("Exception should be thrown upon AuthorizationException"); + } catch (IOException e) { + Assert.assertEquals(AuthorizationException.class.getName() + ": Denied", + e.getMessage()); + } + + // assert maxClientRetry is not decremented. + Assert.assertEquals(conf.getInt(MRJobConfig.MR_CLIENT_MAX_RETRIES, + MRJobConfig.DEFAULT_MR_CLIENT_MAX_RETRIES), clientServiceDelegate + .getMaxClientRetry()); + verify(amProxy, times(1)).getJobReport(any(GetJobReportRequest.class)); + } + + @Test public void testHistoryServerNotConfigured() throws Exception { //RM doesn't have app report and job History Server is not configured ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(