tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject tez git commit: TEZ-2629. TEZ-2629. LimitExceededException in Tez client when DAG has exceeds the default max counters. (sseth)
Date Thu, 20 Aug 2015 18:43:57 GMT
Repository: tez
Updated Branches:
  refs/heads/master 24ca1de0e -> 5f15a0476


TEZ-2629. TEZ-2629. LimitExceededException in Tez client when DAG has
exceeds the default max counters. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/5f15a047
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/5f15a047
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/5f15a047

Branch: refs/heads/master
Commit: 5f15a04769f54b60cf9a4368287091c431fc3267
Parents: 24ca1de
Author: Siddharth Seth <sseth@apache.org>
Authored: Thu Aug 20 11:43:52 2015 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Thu Aug 20 11:43:52 2015 -0700

----------------------------------------------------------------------
 CHANGES.txt                                     |  4 ++
 .../java/org/apache/tez/client/TezClient.java   |  2 +
 .../org/apache/tez/common/counters/Limits.java  | 10 ++-
 .../org/apache/tez/client/TestTezClient.java    | 69 +++++++++++++++-----
 4 files changed, 68 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/5f15a047/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 7324421..2e356b8 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -10,6 +10,7 @@ INCOMPATIBLE CHANGES
   TEZ-2468. Change the minimum Java version to Java 7.
 
 ALL CHANGES:
+  TEZ-2629. LimitExceededException in Tez client when DAG has exceeds the default max counters
   TEZ-2730. tez-api missing dependency on org.codehaus.jettison for json.
   TEZ-2719. Consider reducing logs in unordered fetcher with shared-fetch option
   TEZ-2646. Add scheduling casual dependency for attempts
@@ -73,6 +74,7 @@ Release 0.7.1: Unreleased
 INCOMPATIBLE CHANGES
 
 ALL CHANGES:
+  TEZ-2629. LimitExceededException in Tez client when DAG has exceeds the default max counters
   TEZ-2540. Create both tez-dist minimal and minimal.tar.gz formats as part of build
   TEZ-2630. TezChild receives IP address instead of FQDN.
   TEZ-2211. Tez UI: Allow users to configure timezone
@@ -301,6 +303,7 @@ Release 0.6.2: Unreleased
 INCOMPATIBLE CHANGES
 
 ALL CHANGES:
+  TEZ-2629. LimitExceededException in Tez client when DAG has exceeds the default max counters
   TEZ-2311. AM can hang if kill received while recovering from previous attempt.
   TEZ-2623. Fix module dependencies related to hadoop-auth.
   TEZ-2560. fix tex-ui build for maven 3.3+
@@ -501,6 +504,7 @@ INCOMPATIBLE CHANGES
   TEZ-2552. CRC errors can cause job to run for very long time in large jobs.
 
 ALL CHANGES:
+  TEZ-2629. LimitExceededException in Tez client when DAG has exceeds the default max counters
   TEZ-2630. TezChild receives IP address instead of FQDN.
   TEZ-2635. Limit number of attempts being downloaded in unordered fetch.
   TEZ-2636. MRInput and MultiMRInput should work for cases when there are 0 physical inputs.

http://git-wip-us.apache.org/repos/asf/tez/blob/5f15a047/tez-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
index 2590879..f9d8973 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -26,6 +26,7 @@ import java.util.Map;
 import javax.annotation.Nullable;
 
 import org.apache.tez.common.RPCUtil;
+import org.apache.tez.common.counters.Limits;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -150,6 +151,7 @@ public class TezClient {
     tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, isSession);
     this.amConfig = new AMConfiguration(tezConf, localResources, credentials);
     this.apiVersionInfo = new TezApiVersionInfo();
+    Limits.setConfiguration(tezConf);
 
     LOG.info("Tez Client Version: " + apiVersionInfo.toString());
   }

http://git-wip-us.apache.org/repos/asf/tez/blob/5f15a047/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java b/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java
index f5a51e4..332df8e 100644
--- a/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java
+++ b/tez-api/src/main/java/org/apache/tez/common/counters/Limits.java
@@ -18,6 +18,7 @@
 
 package org.apache.tez.common.counters;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -41,7 +42,7 @@ public class Limits {
       return;
     }
     if (conf == null) {
-      conf = new Configuration();
+      conf = new TezConfiguration();
     }
     GROUP_NAME_MAX =
         conf.getInt(TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH,
@@ -116,4 +117,11 @@ public class Limits {
     }
   }
 
+  @VisibleForTesting
+  @InterfaceAudience.Private
+  public synchronized static void reset() {
+    conf = null;
+    initialized = false;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/5f15a047/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
index dc0fbb1..92d3cd2 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
@@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicReference;
 
 import javax.annotation.Nullable;
 
+import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
 import static org.mockito.Mockito.mock;
@@ -46,6 +47,9 @@ import org.apache.hadoop.yarn.api.records.URL;
 import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.tez.common.counters.LimitExceededException;
+import org.apache.tez.common.counters.Limits;
+import org.apache.tez.common.counters.TezCounters;
 import org.apache.tez.dag.api.DAG;
 import org.apache.tez.dag.api.PreWarmVertex;
 import org.apache.tez.dag.api.ProcessorDescriptor;
@@ -99,12 +103,20 @@ public class TestTezClient {
     }
   }
   
-  TezClientForTest configure() throws YarnException, IOException, ServiceException {
-    return configure(new HashMap<String, LocalResource>(), true);
+  TezClientForTest configureAndCreateTezClient() throws YarnException, IOException, ServiceException
{
+    return configureAndCreateTezClient(null);
+  }
+
+  TezClientForTest configureAndCreateTezClient(TezConfiguration conf) throws YarnException,
ServiceException,
+      IOException {
+    return configureAndCreateTezClient(new HashMap<String, LocalResource>(), true,
conf);
   }
   
-  TezClientForTest configure(Map<String, LocalResource> lrs, boolean isSession) throws
YarnException, IOException, ServiceException {
-    TezConfiguration conf = new TezConfiguration();
+  TezClientForTest configureAndCreateTezClient(Map<String, LocalResource> lrs, boolean
isSession,
+                                               TezConfiguration conf) throws YarnException,
IOException, ServiceException {
+    if (conf == null) {
+      conf = new TezConfiguration();
+    }
     conf.setBoolean(TezConfiguration.TEZ_IGNORE_LIB_URIS, true);
     conf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, isSession);
     TezClientForTest client = new TezClientForTest("test", conf, lrs, null);
@@ -141,7 +153,7 @@ public class TestTezClient {
     lrs.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
         LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
     
-    TezClientForTest client = configure(lrs, isSession);
+    TezClientForTest client = configureAndCreateTezClient(lrs, isSession, null);
     
     ArgumentCaptor<ApplicationSubmissionContext> captor = ArgumentCaptor.forClass(ApplicationSubmissionContext.class);
     when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
@@ -247,7 +259,7 @@ public class TestTezClient {
   
   @Test (timeout=5000)
   public void testPreWarm() throws Exception {
-    TezClientForTest client = configure();
+    TezClientForTest client = configureAndCreateTezClient();
     client.start();
 
     when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
@@ -275,7 +287,8 @@ public class TestTezClient {
   }
   
   public void testMultipleSubmissionsJob(boolean isSession) throws Exception {
-    TezClientForTest client1 = configure(new HashMap<String, LocalResource>(), isSession);
+    TezClientForTest client1 = configureAndCreateTezClient(new HashMap<String, LocalResource>(),
+        isSession, null);
     when(client1.mockYarnClient.getApplicationReport(client1.mockAppId).getYarnApplicationState())
     .thenReturn(YarnApplicationState.RUNNING);
     client1.start();
@@ -295,7 +308,7 @@ public class TestTezClient {
     // the dag resource will be added to the vertex once
     client1.submitDAG(dag);
     
-    TezClientForTest client2 = configure();
+    TezClientForTest client2 = configureAndCreateTezClient();
     when(client2.mockYarnClient.getApplicationReport(client2.mockAppId).getYarnApplicationState())
     .thenReturn(YarnApplicationState.RUNNING);
     client2.start();
@@ -310,7 +323,7 @@ public class TestTezClient {
   
   @Test(timeout = 5000)
   public void testWaitTillReady_Interrupt() throws Exception {
-    final TezClientForTest client = configure();
+    final TezClientForTest client = configureAndCreateTezClient();
     client.start();
 
     when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
@@ -330,13 +343,13 @@ public class TestTezClient {
     thread.join(250);
     thread.interrupt();
     thread.join();
-    Assert.assertThat(exceptionReference.get(),CoreMatchers. instanceOf(InterruptedException.class));
+    Assert.assertThat(exceptionReference.get(), CoreMatchers.instanceOf(InterruptedException.class));
     client.stop();
   }
   
   @Test(timeout = 5000)
   public void testWaitTillReadyAppFailed() throws Exception {
-    final TezClientForTest client = configure();
+    final TezClientForTest client = configureAndCreateTezClient();
     client.start();
     String msg = "Application Test Failed";
     when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
@@ -345,7 +358,7 @@ public class TestTezClient {
         msg);
     try {
       client.waitTillReady();
-      Assert.fail();
+      fail();
     } catch (SessionNotRunning e) {
       Assert.assertTrue(e.getMessage().contains(msg));
     }
@@ -354,13 +367,13 @@ public class TestTezClient {
   
   @Test(timeout = 5000)
   public void testWaitTillReadyAppFailedNoDiagnostics() throws Exception {
-    final TezClientForTest client = configure();
+    final TezClientForTest client = configureAndCreateTezClient();
     client.start();
     when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
         .thenReturn(YarnApplicationState.NEW).thenReturn(YarnApplicationState.FAILED);
     try {
       client.waitTillReady();
-      Assert.fail();
+      fail();
     } catch (SessionNotRunning e) {
       Assert.assertTrue(e.getMessage().contains(TezClient.NO_CLUSTER_DIAGNOSTICS_MSG));
     }
@@ -369,7 +382,7 @@ public class TestTezClient {
   
   @Test(timeout = 5000)
   public void testSubmitDAGAppFailed() throws Exception {
-    final TezClientForTest client = configure();
+    final TezClientForTest client = configureAndCreateTezClient();
     client.start();
     
     client.callRealGetSessionAMProxy = true;
@@ -385,11 +398,35 @@ public class TestTezClient {
     
     try {
       client.submitDAG(dag);
-      Assert.fail();
+      fail();
     } catch (SessionNotRunning e) {
       Assert.assertTrue(e.getMessage().contains(msg));
     }
     client.stop();
   }
 
+  @Test(timeout = 5000)
+  public void testTezClientCounterLimits() throws YarnException, IOException, ServiceException
{
+    Limits.reset();
+    int defaultCounterLimit = TezConfiguration.TEZ_COUNTERS_MAX_DEFAULT;
+
+    int newCounterLimit = defaultCounterLimit + 500;
+
+    TezConfiguration conf = new TezConfiguration();
+    conf.setInt(TezConfiguration.TEZ_COUNTERS_MAX, newCounterLimit);
+
+    configureAndCreateTezClient(conf);
+
+    TezCounters counters = new TezCounters();
+    for (int i = 0 ; i < newCounterLimit ; i++) {
+      counters.findCounter("GroupName", "TestCounter" + i).setValue(i);
+    }
+
+    try {
+      counters.findCounter("GroupName", "TestCounterFail").setValue(1);
+      fail("Expecting a LimitExceedException - too many counters");
+    } catch (LimitExceededException e) {
+    }
+  }
+
 }


Mime
View raw message