hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1079203 - in /hadoop/mapreduce/branches/yahoo-merge/src: java/org/apache/hadoop/mapreduce/counters/ test/mapred/org/apache/hadoop/mapreduce/
Date Tue, 08 Mar 2011 05:55:21 GMT
Author: omalley
Date: Tue Mar  8 05:55:21 2011
New Revision: 1079203

URL: http://svn.apache.org/viewvc?rev=1079203&view=rev
Log:
commit fe76e91e01b991d1b2401e9a08a697c9dc5f94ac
Author: Luke Lu <llu@yahoo-inc.com>
Date:   Tue Dec 7 03:19:56 2010 -0800

     Fix counters limits

Modified:
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java
    hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/Limits.java
    hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java?rev=1079203&r1=1079202&r2=1079203&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
Tue Mar  8 05:55:21 2011
@@ -78,6 +78,8 @@ public abstract class AbstractCounters<C
                   JobCounter.class.getName());
   }
 
+  private final Limits limits = new Limits();
+
   @InterfaceAudience.Private
   public AbstractCounters(CounterGroupFactory<C, G> gf) {
     groupFactory = gf;
@@ -97,7 +99,7 @@ public abstract class AbstractCounters<C
     this.groupFactory = groupFactory;
     for(G1 group: counters) {
       String name = group.getName();
-      G newGroup = groupFactory.newGroup(name, group.getDisplayName());
+      G newGroup = groupFactory.newGroup(name, group.getDisplayName(), limits);
       (isFrameworkGroup(name) ? fgroups : groups).put(name, newGroup);
       for(Counter counter: group) {
         newGroup.addCounter(counter.getName(), counter.getDisplayName(),
@@ -129,7 +131,7 @@ public abstract class AbstractCounters<C
    */
   @InterfaceAudience.Private
   public G addGroup(String name, String displayName) {
-    return addGroup(groupFactory.newGroup(name, displayName));
+    return addGroup(groupFactory.newGroup(name, displayName, limits));
   }
 
   /**
@@ -194,11 +196,11 @@ public abstract class AbstractCounters<C
     boolean isFGroup = isFrameworkGroup(groupName);
     G group = isFGroup ? fgroups.get(groupName) : groups.get(groupName);
     if (group == null) {
-      group = groupFactory.newGroup(filterGroupName(groupName));
+      group = groupFactory.newGroup(filterGroupName(groupName), limits);
       if (isFGroup) {
         fgroups.put(groupName, group);
       } else {
-        groupFactory.limits().checkGroups(groups.size() + 1);
+        limits.checkGroups(groups.size() + 1);
         groups.put(groupName, group);
       }
     }
@@ -208,7 +210,7 @@ public abstract class AbstractCounters<C
   private String filterGroupName(String oldName) {
     String newName = legacyMap.get(oldName);
     if (newName == null) {
-      return groupFactory.limits().filterGroupName(oldName);
+      return limits.filterGroupName(oldName);
     }
     LOG.warn("Group "+ oldName +" is deprecated. Use "+ newName +" instead");
     return newName;
@@ -284,9 +286,8 @@ public abstract class AbstractCounters<C
     }
     int numGroups = WritableUtils.readVInt(in);
     while (numGroups-- > 0) {
-      groupFactory.limits().checkGroups(groups.size() + 1);
-      G group = groupFactory.newGenericGroup(Text.readString(in), null,
-                                             groupFactory.limits());
+      limits.checkGroups(groups.size() + 1);
+      G group = groupFactory.newGenericGroup(Text.readString(in), null, limits);
       group.readFields(in);
       groups.put(group.getName(), group);
     }
@@ -318,8 +319,9 @@ public abstract class AbstractCounters<C
     for(G right : other) {
       G left = groups.get(right.getName());
       if (left == null) {
-        groupFactory.limits().checkGroups(groups.size() + 1);
-        left = groupFactory.newGroup(right.getName(), right.getDisplayName());
+        limits.checkGroups(groups.size() + 1);
+        left = groupFactory.newGroup(right.getName(), right.getDisplayName(),
+                                     limits);
         groups.put(right.getName(), left);
       }
       left.incrAllCounters(right);
@@ -363,6 +365,6 @@ public abstract class AbstractCounters<C
 
   @InterfaceAudience.Private
   public Limits limits() {
-    return groupFactory.limits();
+    return limits;
   }
 }

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java?rev=1079203&r1=1079202&r2=1079203&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/CounterGroupFactory.java
Tue Mar  8 05:55:21 2011
@@ -51,8 +51,6 @@ public abstract class CounterGroupFactor
   private static final List<String> i2s = Lists.newArrayList();
   private static final int VERSION = 1;
 
-  private final Limits limits = new Limits();
-
   private final Map<String, FrameworkGroupFactory<G>> fmap = Maps.newHashMap();
   {
     // Add builtin counter class here and the version when changed.
@@ -88,19 +86,22 @@ public abstract class CounterGroupFactor
   /**
    * Create a new counter group
    * @param name of the group
+   * @param limits the counters limits policy object
    * @return a new counter group
    */
-  public G newGroup(String name) {
-    return newGroup(name, ResourceBundles.getCounterGroupName(name, name));
+  public G newGroup(String name, Limits limits) {
+    return newGroup(name, ResourceBundles.getCounterGroupName(name, name),
+                    limits);
   }
 
   /**
    * Create a new counter group
    * @param name of the group
    * @param displayName of the group
+   * @param limits the counters limits policy object
    * @return a new counter group
    */
-  public G newGroup(String name, String displayName) {
+  public G newGroup(String name, String displayName, Limits limits) {
     FrameworkGroupFactory<G> gf = fmap.get(name);
     if (gf != null) return gf.newGroup(name);
     if (name.equals(FileSystemCounter.class.getName())) {
@@ -184,9 +185,4 @@ public abstract class CounterGroupFactor
    * @return a new file system counter group
    */
   protected abstract G newFileSystemGroup();
-
-  @InterfaceAudience.Private
-  public Limits limits() {
-    return limits;
-  }
 }

Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/Limits.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/Limits.java?rev=1079203&r1=1079202&r2=1079203&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/Limits.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/counters/Limits.java
Tue Mar  8 05:55:21 2011
@@ -26,14 +26,14 @@ import static org.apache.hadoop.mapreduc
 public class Limits {
 
   static final Configuration conf = new Configuration();
-  static final int GROUP_NAME_MAX = conf.getInt(COUNTER_GROUP_NAME_MAX_KEY,
-                                                COUNTER_GROUP_NAME_MAX_DEFAULT);
-  static final int COUNTER_NAME_MAX = conf.getInt(COUNTER_NAME_MAX_KEY,
-                                                  COUNTER_NAME_MAX_DEFAULT);
-  static final int GROUPS_MAX = conf.getInt(COUNTER_GROUPS_MAX_KEY,
-                                            COUNTER_GROUPS_MAX_DEFAULT);
-  static final int COUNTERS_MAX = conf.getInt(COUNTERS_MAX_KEY,
-                                              COUNTERS_MAX_DEFAULT);
+  public static final int GROUP_NAME_MAX =
+      conf.getInt(COUNTER_GROUP_NAME_MAX_KEY, COUNTER_GROUP_NAME_MAX_DEFAULT);
+  public static final int COUNTER_NAME_MAX =
+      conf.getInt(COUNTER_NAME_MAX_KEY, COUNTER_NAME_MAX_DEFAULT);
+  public static final int GROUPS_MAX =
+      conf.getInt(COUNTER_GROUPS_MAX_KEY, COUNTER_GROUPS_MAX_DEFAULT);
+  public static final int COUNTERS_MAX =
+      conf.getInt(COUNTERS_MAX_KEY, COUNTERS_MAX_DEFAULT);
 
   private int totalCounters;
   private LimitExceededException firstViolation;

Modified: hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java?rev=1079203&r1=1079202&r2=1079203&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java
(original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/test/mapred/org/apache/hadoop/mapreduce/TestCounters.java
Tue Mar  8 05:55:21 2011
@@ -22,11 +22,18 @@ import java.util.Random;
 import org.junit.Test;
 import static org.junit.Assert.*;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapreduce.counters.LimitExceededException;
+import org.apache.hadoop.mapreduce.counters.Limits;
+
 /**
  * TestCounters checks the sanity and recoverability of {@code Counters}
  */
 public class TestCounters {
 
+  static final Log LOG = LogFactory.getLog(TestCounters.class);
+
   /**
    * Verify counter value works
    */
@@ -56,4 +63,69 @@ public class TestCounters {
     }
   }
 
+  @Test public void testLimits() {
+    for (int i = 0; i < 3; ++i) {
+      // make sure limits apply to separate containers
+      testMaxCounters(new Counters());
+      testMaxGroups(new Counters());
+    }
+  }
+
+  static final Enum<?> FRAMEWORK_COUNTER = TaskCounter.CPU_MILLISECONDS;
+  static final long FRAMEWORK_COUNTER_VALUE = 8;
+  static final String FS_SCHEME = "HDFS";
+  static final FileSystemCounter FS_COUNTER = FileSystemCounter.BYTES_READ;
+  static final long FS_COUNTER_VALUE = 10;
+
+  private void testMaxCounters(final Counters counters) {
+    LOG.info("counters max="+ Limits.COUNTERS_MAX);
+    for (int i = 0; i < Limits.COUNTERS_MAX; ++i) {
+      counters.findCounter("test", "test"+ i);
+    }
+    setExpected(counters);
+    shouldThrow(LimitExceededException.class, new Runnable() {
+      public void run() {
+        counters.findCounter("test", "bad");
+      }
+    });
+    checkExpected(counters);
+  }
+
+  private void testMaxGroups(final Counters counters) {
+    LOG.info("counter groups max="+ Limits.GROUPS_MAX);
+    for (int i = 0; i < Limits.GROUPS_MAX; ++i) {
+      // assuming COUNTERS_MAX > GROUPS_MAX
+      counters.findCounter("test"+ i, "test");
+    }
+    setExpected(counters);
+    shouldThrow(LimitExceededException.class, new Runnable() {
+      public void run() {
+        counters.findCounter("bad", "test");
+      }
+    });
+    checkExpected(counters);
+  }
+
+  private void setExpected(Counters counters) {
+    counters.findCounter(FRAMEWORK_COUNTER).setValue(FRAMEWORK_COUNTER_VALUE);
+    counters.findCounter(FS_SCHEME, FS_COUNTER).setValue(FS_COUNTER_VALUE);
+  }
+
+  private void checkExpected(Counters counters) {
+    assertEquals(FRAMEWORK_COUNTER_VALUE,
+                 counters.findCounter(FRAMEWORK_COUNTER).getValue());
+    assertEquals(FS_COUNTER_VALUE,
+                 counters.findCounter(FS_SCHEME, FS_COUNTER).getValue());
+  }
+
+  private void shouldThrow(Class<? extends Exception> ecls, Runnable runnable) {
+    try {
+      runnable.run();
+    } catch (Exception e) {
+      assertSame(ecls, e.getClass());
+      LOG.info("got expected: "+ e);
+      return;
+    }
+    assertTrue("Should've thrown "+ ecls.getSimpleName(), false);
+  }
 }



Mime
View raw message