Return-Path: X-Original-To: apmail-tez-commits-archive@minotaur.apache.org Delivered-To: apmail-tez-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5F1541857B for ; Thu, 20 Aug 2015 18:44:50 +0000 (UTC) Received: (qmail 66411 invoked by uid 500); 20 Aug 2015 18:44:50 -0000 Delivered-To: apmail-tez-commits-archive@tez.apache.org Received: (qmail 66377 invoked by uid 500); 20 Aug 2015 18:44:50 -0000 Mailing-List: contact commits-help@tez.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tez.apache.org Delivered-To: mailing list commits@tez.apache.org Received: (qmail 66368 invoked by uid 99); 20 Aug 2015 18:44:50 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 20 Aug 2015 18:44:50 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 16A3DE7DD9; Thu, 20 Aug 2015 18:44:50 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sseth@apache.org To: commits@tez.apache.org Message-Id: <4cab577ffe5e4127be010551777666c5@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: tez git commit: TEZ-2629. TEZ-2629. LimitExceededException in Tez client when DAG has exceeds the default max counters. (sseth) Date: Thu, 20 Aug 2015 18:44:50 +0000 (UTC) Repository: tez Updated Branches: refs/heads/branch-0.7 a405ce957 -> 9f6ca7bac TEZ-2629. TEZ-2629. LimitExceededException in Tez client when DAG has exceeds the default max counters. (sseth) Project: http://git-wip-us.apache.org/repos/asf/tez/repo Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/9f6ca7ba Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/9f6ca7ba Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/9f6ca7ba Branch: refs/heads/branch-0.7 Commit: 9f6ca7bac7dbff2982fea3ca75c637a1acb6118a Parents: a405ce9 Author: Siddharth Seth Authored: Thu Aug 20 11:44:46 2015 -0700 Committer: Siddharth Seth Committed: Thu Aug 20 11:44:46 2015 -0700 ---------------------------------------------------------------------- CHANGES.txt | 3 + .../java/org/apache/tez/client/TezClient.java | 2 + .../org/apache/tez/common/counters/Limits.java | 10 ++- .../org/apache/tez/client/TestTezClient.java | 69 +++++++++++++++----- 4 files changed, 67 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tez/blob/9f6ca7ba/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 2df2253..86dbbcc 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,6 +6,7 @@ Release 0.7.1: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2629. LimitExceededException in Tez client when DAG has exceeds the default max counters TEZ-2719. Consider reducing logs in unordered fetcher with shared-fetch option TEZ-2540. Create both tez-dist minimal and minimal.tar.gz formats as part of build TEZ-2630. TezChild receives IP address instead of FQDN. @@ -234,6 +235,7 @@ Release 0.6.2: Unreleased INCOMPATIBLE CHANGES ALL CHANGES: + TEZ-2629. LimitExceededException in Tez client when DAG has exceeds the default max counters TEZ-2311. AM can hang if kill received while recovering from previous attempt. TEZ-2623. Fix module dependencies related to hadoop-auth. TEZ-2560. fix tex-ui build for maven 3.3+ @@ -434,6 +436,7 @@ INCOMPATIBLE CHANGES TEZ-2552. CRC errors can cause job to run for very long time in large jobs. ALL CHANGES: + TEZ-2629. LimitExceededException in Tez client when DAG has exceeds the default max counters TEZ-2635. Limit number of attempts being downloaded in unordered fetch. TEZ-2636. MRInput and MultiMRInput should work for cases when there are 0 physical inputs. TEZ-2600. When used with HDFS federation(viewfs) ,tez will throw a error http://git-wip-us.apache.org/repos/asf/tez/blob/9f6ca7ba/tez-api/src/main/java/org/apache/tez/client/TezClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java index aad6e76..45837fe 100644 --- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java +++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java @@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceType; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.tez.common.counters.Limits; import org.apache.tez.common.ReflectionUtils; import org.apache.tez.common.security.HistoryACLPolicyManager; import org.apache.tez.common.security.JobTokenSecretManager; @@ -149,6 +150,7 @@ public class TezClient { tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, isSession); this.amConfig = new AMConfiguration(tezConf, localResources, credentials); this.apiVersionInfo = new TezApiVersionInfo(); + Limits.setConfiguration(tezConf); LOG.info("Tez Client Version: " + apiVersionInfo.toString()); } http://git-wip-us.apache.org/repos/asf/tez/blob/9f6ca7ba/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java ---------------------------------------------------------------------- diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java b/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java index f5a51e4..332df8e 100644 --- a/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java +++ b/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java @@ -18,6 +18,7 @@ package org.apache.tez.common.counters; +import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience; @@ -41,7 +42,7 @@ public class Limits { return; } if (conf == null) { - conf = new Configuration(); + conf = new TezConfiguration(); } GROUP_NAME_MAX = conf.getInt(TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH, @@ -116,4 +117,11 @@ public class Limits { } } + @VisibleForTesting + @InterfaceAudience.Private + public synchronized static void reset() { + conf = null; + initialized = false; + } + } http://git-wip-us.apache.org/repos/asf/tez/blob/9f6ca7ba/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java ---------------------------------------------------------------------- diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java index dc0fbb1..92d3cd2 100644 --- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java +++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java @@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicReference; import javax.annotation.Nullable; +import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; @@ -46,6 +47,9 @@ import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.tez.common.counters.LimitExceededException; +import org.apache.tez.common.counters.Limits; +import org.apache.tez.common.counters.TezCounters; import org.apache.tez.dag.api.DAG; import org.apache.tez.dag.api.PreWarmVertex; import org.apache.tez.dag.api.ProcessorDescriptor; @@ -99,12 +103,20 @@ public class TestTezClient { } } - TezClientForTest configure() throws YarnException, IOException, ServiceException { - return configure(new HashMap(), true); + TezClientForTest configureAndCreateTezClient() throws YarnException, IOException, ServiceException { + return configureAndCreateTezClient(null); + } + + TezClientForTest configureAndCreateTezClient(TezConfiguration conf) throws YarnException, ServiceException, + IOException { + return configureAndCreateTezClient(new HashMap(), true, conf); } - TezClientForTest configure(Map lrs, boolean isSession) throws YarnException, IOException, ServiceException { - TezConfiguration conf = new TezConfiguration(); + TezClientForTest configureAndCreateTezClient(Map lrs, boolean isSession, + TezConfiguration conf) throws YarnException, IOException, ServiceException { + if (conf == null) { + conf = new TezConfiguration(); + } conf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true); conf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, isSession); TezClientForTest client = new TezClientForTest("test", conf, lrs, null); @@ -141,7 +153,7 @@ public class TestTezClient { lrs.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"), LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1)); - TezClientForTest client = configure(lrs, isSession); + TezClientForTest client = configureAndCreateTezClient(lrs, isSession, null); ArgumentCaptor captor = ArgumentCaptor.forClass(ApplicationSubmissionContext.class); when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) @@ -247,7 +259,7 @@ public class TestTezClient { @Test (timeout=5000) public void testPreWarm() throws Exception { - TezClientForTest client = configure(); + TezClientForTest client = configureAndCreateTezClient(); client.start(); when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) @@ -275,7 +287,8 @@ public class TestTezClient { } public void testMultipleSubmissionsJob(boolean isSession) throws Exception { - TezClientForTest client1 = configure(new HashMap(), isSession); + TezClientForTest client1 = configureAndCreateTezClient(new HashMap(), + isSession, null); when(client1.mockYarnClient.getApplicationReport(client1.mockAppId).getYarnApplicationState()) .thenReturn(YarnApplicationState.RUNNING); client1.start(); @@ -295,7 +308,7 @@ public class TestTezClient { // the dag resource will be added to the vertex once client1.submitDAG(dag); - TezClientForTest client2 = configure(); + TezClientForTest client2 = configureAndCreateTezClient(); when(client2.mockYarnClient.getApplicationReport(client2.mockAppId).getYarnApplicationState()) .thenReturn(YarnApplicationState.RUNNING); client2.start(); @@ -310,7 +323,7 @@ public class TestTezClient { @Test(timeout = 5000) public void testWaitTillReady_Interrupt() throws Exception { - final TezClientForTest client = configure(); + final TezClientForTest client = configureAndCreateTezClient(); client.start(); when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) @@ -330,13 +343,13 @@ public class TestTezClient { thread.join(250); thread.interrupt(); thread.join(); - Assert.assertThat(exceptionReference.get(),CoreMatchers. instanceOf(InterruptedException.class)); + Assert.assertThat(exceptionReference.get(), CoreMatchers.instanceOf(InterruptedException.class)); client.stop(); } @Test(timeout = 5000) public void testWaitTillReadyAppFailed() throws Exception { - final TezClientForTest client = configure(); + final TezClientForTest client = configureAndCreateTezClient(); client.start(); String msg = "Application Test Failed"; when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) @@ -345,7 +358,7 @@ public class TestTezClient { msg); try { client.waitTillReady(); - Assert.fail(); + fail(); } catch (SessionNotRunning e) { Assert.assertTrue(e.getMessage().contains(msg)); } @@ -354,13 +367,13 @@ public class TestTezClient { @Test(timeout = 5000) public void testWaitTillReadyAppFailedNoDiagnostics() throws Exception { - final TezClientForTest client = configure(); + final TezClientForTest client = configureAndCreateTezClient(); client.start(); when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState()) .thenReturn(YarnApplicationState.NEW).thenReturn(YarnApplicationState.FAILED); try { client.waitTillReady(); - Assert.fail(); + fail(); } catch (SessionNotRunning e) { Assert.assertTrue(e.getMessage().contains(TezClient.NO_CLUSTER_DIAGNOSTICS_MSG)); } @@ -369,7 +382,7 @@ public class TestTezClient { @Test(timeout = 5000) public void testSubmitDAGAppFailed() throws Exception { - final TezClientForTest client = configure(); + final TezClientForTest client = configureAndCreateTezClient(); client.start(); client.callRealGetSessionAMProxy = true; @@ -385,11 +398,35 @@ public class TestTezClient { try { client.submitDAG(dag); - Assert.fail(); + fail(); } catch (SessionNotRunning e) { Assert.assertTrue(e.getMessage().contains(msg)); } client.stop(); } + @Test(timeout = 5000) + public void testTezClientCounterLimits() throws YarnException, IOException, ServiceException { + Limits.reset(); + int defaultCounterLimit = TezConfiguration.TEZ_COUNTERS_MAX_DEFAULT; + + int newCounterLimit = defaultCounterLimit + 500; + + TezConfiguration conf = new TezConfiguration(); + conf.setInt(TezConfiguration.TEZ_COUNTERS_MAX, newCounterLimit); + + configureAndCreateTezClient(conf); + + TezCounters counters = new TezCounters(); + for (int i = 0 ; i < newCounterLimit ; i++) { + counters.findCounter("GroupName", "TestCounter" + i).setValue(i); + } + + try { + counters.findCounter("GroupName", "TestCounterFail").setValue(1); + fail("Expecting a LimitExceedException - too many counters"); + } catch (LimitExceededException e) { + } + } + }