hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r1300798 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/mapred-default.xml src/mapred/org/apache/hadoop/mapred/Counters.java src/test/org/apache/hadoop/mapred/TestUserDefinedCounters.java
Date Thu, 15 Mar 2012 00:32:16 GMT
Author: tomwhite
Date: Thu Mar 15 00:32:15 2012
New Revision: 1300798

URL: http://svn.apache.org/viewvc?rev=1300798&view=rev
Log:
MAPREDUCE-2835. Make per-job counter limits configurable.

Modified:
    hadoop/common/branches/branch-1/CHANGES.txt
    hadoop/common/branches/branch-1/src/mapred/mapred-default.xml
    hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/Counters.java
    hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestUserDefinedCounters.java

Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1300798&r1=1300797&r2=1300798&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Thu Mar 15 00:32:15 2012
@@ -157,6 +157,8 @@ Release 1.1.0 - unreleased
     HDFS-3075. Backport HADOOP-4885: Try to restore failed name-node storage
     directories at checkpoint time.  (Brandon Li via szetszwo)
 
+    MAPREDUCE-2835. Make per-job counter limits configurable. (tomwhite)
+
 Release 1.0.2 - unreleased
 
   NEW FEATURES

Modified: hadoop/common/branches/branch-1/src/mapred/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/mapred-default.xml?rev=1300798&r1=1300797&r2=1300798&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/mapred-default.xml (original)
+++ hadoop/common/branches/branch-1/src/mapred/mapred-default.xml Thu Mar 15 00:32:15 2012
@@ -1233,13 +1233,36 @@
   </description>
 </property>
 
+<!--  end of node health script variables -->
+
 <property>
-  <name>mapreduce.job.counters.limit</name>
+  <name>mapreduce.job.counters.max</name>
   <value>120</value>
   <description>Limit on the number of counters allowed per job.
   </description>
 </property>
 
-<!--  end of node health script variables -->
+<property>
+  <name>mapreduce.job.counters.groups.max</name>
+  <value>50</value>
+  <description>Limit on the number of counter groups allowed per job.
+  </description>
+</property>
+
+<property>
+  <name>mapreduce.job.counters.counter.name.max</name>
+  <value>64</value>
+  <description>Limit on the length of counter names in jobs. Names
+  exceeding this limit will be truncated.
+  </description>
+</property>
+
+<property>
+  <name>mapreduce.job.counters.group.name.max</name>
+  <value>128</value>
+  <description>Limit on the length of counter group names in jobs. Names
+  exceeding this limit will be truncated.
+  </description>
+</property>
 
 </configuration>

Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/Counters.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/Counters.java?rev=1300798&r1=1300797&r2=1300798&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/Counters.java (original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/Counters.java Thu
Mar 15 00:32:15 2012
@@ -59,18 +59,22 @@ public class Counters implements Writabl
   private static char[] charsToEscape =  {GROUP_OPEN, GROUP_CLOSE, 
                                           COUNTER_OPEN, COUNTER_CLOSE, 
                                           UNIT_OPEN, UNIT_CLOSE};
+  private static final JobConf conf = new JobConf();
   /** limit on the size of the name of the group **/
-  private static final int GROUP_NAME_LIMIT = 128;
+  private static final int GROUP_NAME_LIMIT = 
+    conf.getInt("mapreduce.job.counters.group.name.max", 128);
   /** limit on the size of the counter name **/
-  private static final int COUNTER_NAME_LIMIT = 64;
+  private static final int COUNTER_NAME_LIMIT = 
+    conf.getInt("mapreduce.job.counters.counter.name.max", 64);
   
-  private static final JobConf conf = new JobConf();
   /** limit on counters **/
   public static int MAX_COUNTER_LIMIT = 
-    conf.getInt("mapreduce.job.counters.limit", 120);
+    conf.getInt("mapreduce.job.counters.limit", // deprecated in 0.23
+        conf.getInt("mapreduce.job.counters.max", 120));
 
   /** the max groups allowed **/
-  static final int MAX_GROUP_LIMIT = 50;
+  public static final int MAX_GROUP_LIMIT = 
+    conf.getInt("mapreduce.job.counters.groups.max", 50);
   
   /** the number of current counters**/
   private int numCounters = 0;

Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestUserDefinedCounters.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestUserDefinedCounters.java?rev=1300798&r1=1300797&r2=1300798&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestUserDefinedCounters.java
(original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestUserDefinedCounters.java
Thu Mar 15 00:32:15 2012
@@ -130,7 +130,7 @@ public class TestUserDefinedCounters ext
     assertEquals(4,
         runningJob.getCounters().getGroup("StringCounter")
         .getCounter("MapRecords"));
-    assertTrue(counters.getGroupNames().size() <= 51);
+    assertTrue(counters.getGroupNames().size() <= Counters.MAX_GROUP_LIMIT);
     int i = 0;
     while (counters.size() < Counters.MAX_COUNTER_LIMIT) {
       counters.incrCounter("IncrCounter", "limit " + i, 2);



Mime
View raw message