From mapreduce-commits-return-1313-apmail-hadoop-mapreduce-commits-archive=hadoop.apache.org@hadoop.apache.org Tue Mar 08 05:55:44 2011 Return-Path: Delivered-To: apmail-hadoop-mapreduce-commits-archive@minotaur.apache.org Received: (qmail 81511 invoked from network); 8 Mar 2011 05:55:43 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 8 Mar 2011 05:55:43 -0000 Received: (qmail 54662 invoked by uid 500); 8 Mar 2011 05:55:43 -0000 Delivered-To: apmail-hadoop-mapreduce-commits-archive@hadoop.apache.org Received: (qmail 54627 invoked by uid 500); 8 Mar 2011 05:55:43 -0000 Mailing-List: contact mapreduce-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: mapreduce-dev@hadoop.apache.org Delivered-To: mailing list mapreduce-commits@hadoop.apache.org Received: (qmail 54618 invoked by uid 99); 8 Mar 2011 05:55:43 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 Mar 2011 05:55:43 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 Mar 2011 05:55:42 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 0F8E42388A40; Tue, 8 Mar 2011 05:55:22 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1079203 - in /hadoop/mapreduce/branches/yahoo-merge/src: java/org/apache/hadoop/mapreduce/counters/ test/mapred/org/apache/hadoop/mapreduce/ Date: Tue, 08 Mar 2011 05:55:21 -0000 To: mapreduce-commits@hadoop.apache.org From: omalley@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110308055522.0F8E42388A40@eris.apache.org> Author: omalley Date: Tue Mar 8 05:55:21 2011 New Revision: 1079203 URL: http://svn.apache.org/viewvc?rev=1079203&view=rev Log: commit fe76e91e01b991d1b2401e9a08a697c9dc5f94ac Author: Luke Lu Date: Tue Dec 7 03:19:56 2010 -0800 Fix counters limits Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/Limits.java hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java?rev=1079203&r1=1079202&r2=1079203&view=diff ============================================================================== --- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java (original) +++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java Tue Mar 8 05:55:21 2011 @@ -78,6 +78,8 @@ public abstract class AbstractCounters gf) { groupFactory = gf; @@ -97,7 +99,7 @@ public abstract class AbstractCounters 0) { - groupFactory.limits().checkGroups(groups.size() + 1); - G group = groupFactory.newGenericGroup(Text.readString(in), null, - groupFactory.limits()); + limits.checkGroups(groups.size() + 1); + G group = groupFactory.newGenericGroup(Text.readString(in), null, limits); group.readFields(in); groups.put(group.getName(), group); } @@ -318,8 +319,9 @@ public abstract class AbstractCounters i2s = Lists.newArrayList(); private static final int VERSION = 1; - private final Limits limits = new Limits(); - private final Map> fmap = Maps.newHashMap(); { // Add builtin counter class here and the version when changed. @@ -88,19 +86,22 @@ public abstract class CounterGroupFactor /** * Create a new counter group * @param name of the group + * @param limits the counters limits policy object * @return a new counter group */ - public G newGroup(String name) { - return newGroup(name, ResourceBundles.getCounterGroupName(name, name)); + public G newGroup(String name, Limits limits) { + return newGroup(name, ResourceBundles.getCounterGroupName(name, name), + limits); } /** * Create a new counter group * @param name of the group * @param displayName of the group + * @param limits the counters limits policy object * @return a new counter group */ - public G newGroup(String name, String displayName) { + public G newGroup(String name, String displayName, Limits limits) { FrameworkGroupFactory gf = fmap.get(name); if (gf != null) return gf.newGroup(name); if (name.equals(FileSystemCounter.class.getName())) { @@ -184,9 +185,4 @@ public abstract class CounterGroupFactor * @return a new file system counter group */ protected abstract G newFileSystemGroup(); - - @InterfaceAudience.Private - public Limits limits() { - return limits; - } } Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/Limits.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/Limits.java?rev=1079203&r1=1079202&r2=1079203&view=diff ============================================================================== --- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/Limits.java (original) +++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/Limits.java Tue Mar 8 05:55:21 2011 @@ -26,14 +26,14 @@ import static org.apache.hadoop.mapreduc public class Limits { static final Configuration conf = new Configuration(); - static final int GROUP_NAME_MAX = conf.getInt(COUNTER_GROUP_NAME_MAX_KEY, - COUNTER_GROUP_NAME_MAX_DEFAULT); - static final int COUNTER_NAME_MAX = conf.getInt(COUNTER_NAME_MAX_KEY, - COUNTER_NAME_MAX_DEFAULT); - static final int GROUPS_MAX = conf.getInt(COUNTER_GROUPS_MAX_KEY, - COUNTER_GROUPS_MAX_DEFAULT); - static final int COUNTERS_MAX = conf.getInt(COUNTERS_MAX_KEY, - COUNTERS_MAX_DEFAULT); + public static final int GROUP_NAME_MAX = + conf.getInt(COUNTER_GROUP_NAME_MAX_KEY, COUNTER_GROUP_NAME_MAX_DEFAULT); + public static final int COUNTER_NAME_MAX = + conf.getInt(COUNTER_NAME_MAX_KEY, COUNTER_NAME_MAX_DEFAULT); + public static final int GROUPS_MAX = + conf.getInt(COUNTER_GROUPS_MAX_KEY, COUNTER_GROUPS_MAX_DEFAULT); + public static final int COUNTERS_MAX = + conf.getInt(COUNTERS_MAX_KEY, COUNTERS_MAX_DEFAULT); private int totalCounters; private LimitExceededException firstViolation; Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java?rev=1079203&r1=1079202&r2=1079203&view=diff ============================================================================== --- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java (original) +++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java Tue Mar 8 05:55:21 2011 @@ -22,11 +22,18 @@ import java.util.Random; import org.junit.Test; import static org.junit.Assert.*; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.mapreduce.counters.LimitExceededException; +import org.apache.hadoop.mapreduce.counters.Limits; + /** * TestCounters checks the sanity and recoverability of {@code Counters} */ public class TestCounters { + static final Log LOG = LogFactory.getLog(TestCounters.class); + /** * Verify counter value works */ @@ -56,4 +63,69 @@ public class TestCounters { } } + @Test public void testLimits() { + for (int i = 0; i < 3; ++i) { + // make sure limits apply to separate containers + testMaxCounters(new Counters()); + testMaxGroups(new Counters()); + } + } + + static final Enum FRAMEWORK_COUNTER = TaskCounter.CPU_MILLISECONDS; + static final long FRAMEWORK_COUNTER_VALUE = 8; + static final String FS_SCHEME = "HDFS"; + static final FileSystemCounter FS_COUNTER = FileSystemCounter.BYTES_READ; + static final long FS_COUNTER_VALUE = 10; + + private void testMaxCounters(final Counters counters) { + LOG.info("counters max="+ Limits.COUNTERS_MAX); + for (int i = 0; i < Limits.COUNTERS_MAX; ++i) { + counters.findCounter("test", "test"+ i); + } + setExpected(counters); + shouldThrow(LimitExceededException.class, new Runnable() { + public void run() { + counters.findCounter("test", "bad"); + } + }); + checkExpected(counters); + } + + private void testMaxGroups(final Counters counters) { + LOG.info("counter groups max="+ Limits.GROUPS_MAX); + for (int i = 0; i < Limits.GROUPS_MAX; ++i) { + // assuming COUNTERS_MAX > GROUPS_MAX + counters.findCounter("test"+ i, "test"); + } + setExpected(counters); + shouldThrow(LimitExceededException.class, new Runnable() { + public void run() { + counters.findCounter("bad", "test"); + } + }); + checkExpected(counters); + } + + private void setExpected(Counters counters) { + counters.findCounter(FRAMEWORK_COUNTER).setValue(FRAMEWORK_COUNTER_VALUE); + counters.findCounter(FS_SCHEME, FS_COUNTER).setValue(FS_COUNTER_VALUE); + } + + private void checkExpected(Counters counters) { + assertEquals(FRAMEWORK_COUNTER_VALUE, + counters.findCounter(FRAMEWORK_COUNTER).getValue()); + assertEquals(FS_COUNTER_VALUE, + counters.findCounter(FS_SCHEME, FS_COUNTER).getValue()); + } + + private void shouldThrow(Class ecls, Runnable runnable) { + try { + runnable.run(); + } catch (Exception e) { + assertSame(ecls, e.getClass()); + LOG.info("got expected: "+ e); + return; + } + assertTrue("Should've thrown "+ ecls.getSimpleName(), false); + } }