hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1438843 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/mapred/StatisticsCollector.java src/test/org/apache/hadoop/mapred/TestStatisticsCollector.java
Date Sat, 26 Jan 2013 04:29:54 GMT
Author: tucu
Date: Sat Jan 26 04:29:54 2013
New Revision: 1438843

URL: http://svn.apache.org/viewvc?rev=1438843&view=rev
Log:
MAPREDUCE-4963. StatisticsCollector improperly keeps track of Last Day and Last Hour statistics
for new TaskTrackers. (rkanter via tucu)

Modified:
    hadoop/common/branches/branch-1/CHANGES.txt
    hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/StatisticsCollector.java
    hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestStatisticsCollector.java

Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1438843&r1=1438842&r2=1438843&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Sat Jan 26 04:29:54 2013
@@ -451,6 +451,9 @@ Release 1.2.0 - unreleased
     MAPREDUCE-2264. Job status exceeds 100% in some cases. 
     (devaraj.k & sandyr via tucu)
 
+    MAPREDUCE-4963. StatisticsCollector improperly keeps track of "Last Day" 
+    and "Last Hour" statistics for new TaskTrackers. (rkanter via tucu)
+
 Release 1.1.2 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/StatisticsCollector.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/StatisticsCollector.java?rev=1438843&r1=1438842&r2=1438843&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/StatisticsCollector.java
(original)
+++ hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/StatisticsCollector.java
Sat Jan 26 04:29:54 2013
@@ -206,6 +206,7 @@ class StatisticsCollector {
       private final LinkedList<Integer> buckets = new LinkedList<Integer>();
       private int value;
       private int currentValue;
+      private int updates;
 
       public synchronized int getValue() {
         return value;
@@ -261,9 +262,6 @@ class StatisticsCollector {
     final int collectBuckets;
     final int updatesPerBucket;
     
-    private int updates;
-    private int buckets;
-
     TimeWindowStatUpdater(TimeWindow w, int updatePeriod) {
       if (updatePeriod > w.updateGranularity) {
         throw new RuntimeException(
@@ -274,18 +272,14 @@ class StatisticsCollector {
     }
 
     synchronized void update() {
-      updates++;
-      if (updates == updatesPerBucket) {
-        for(TimeStat stat : statToCollect.values()) {
+      for (TimeStat stat : statToCollect.values()) {
+        stat.updates++;
+        if (stat.updates == updatesPerBucket) {
           stat.addBucket();
+          stat.updates = 0;
         }
-        updates = 0;
-        buckets++;
-        if (buckets > collectBuckets) {
-          for (TimeStat stat : statToCollect.values()) {
-            stat.removeBucket();
-          }
-          buckets--;
+        if (stat.buckets.size() > collectBuckets) {
+          stat.removeBucket();
         }
       }
     }

Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestStatisticsCollector.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestStatisticsCollector.java?rev=1438843&r1=1438842&r2=1438843&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestStatisticsCollector.java
(original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestStatisticsCollector.java
Sat Jan 26 04:29:54 2013
@@ -21,6 +21,7 @@ import junit.framework.TestCase;
 
 import org.apache.hadoop.mapred.StatisticsCollector.TimeWindow;
 import org.apache.hadoop.mapred.StatisticsCollector.Stat;
+import org.apache.hadoop.mapred.StatisticsCollector.Stat.TimeStat;
 
 public class TestStatisticsCollector extends TestCase{
 
@@ -80,4 +81,60 @@ public class TestStatisticsCollector ext
     assertEquals(95, stat.getValues().get(sincStart).getValue());
   }
 
+  public void testBucketing() throws Exception {
+    StatisticsCollector collector = new StatisticsCollector();
+    TimeWindow window = new TimeWindow("test", 33, 11);
+    // We'll collect 3 buckets before we start removing: 33 / 11 = 3
+    // We'll do 2 updates per bucket (5 is default period): 11 / 5 = 2
+    TimeWindow[] windows = {window};
+    Stat stat1 = collector.createStat("TaskTracker A", windows);
+
+    // TT A starts out with 0 buckets
+    assertEquals(0, stat1.getValues().get(window).getValue());
+    stat1.inc(1);
+    collector.update();
+    assertEquals(0, stat1.getValues().get(window).getValue());
+    stat1.inc(1);
+    collector.update();
+    assertEquals(2, stat1.getValues().get(window).getValue());
+    stat1.inc(1);
+    collector.update();
+    assertEquals(2, stat1.getValues().get(window).getValue());
+    stat1.inc(2);
+    collector.update();
+    assertEquals(2+3, stat1.getValues().get(window).getValue());
+    stat1.inc(0);
+    collector.update();
+    assertEquals(2+3, stat1.getValues().get(window).getValue());
+    stat1.inc(1);
+    // At the next update, we now have 3 buckets for TT 1
+    collector.update();
+    assertEquals(2+3+1, stat1.getValues().get(window).getValue());
+    stat1.inc(4);
+    collector.update();
+    assertEquals(2+3+1, stat1.getValues().get(window).getValue());
+    // At the next update, we're going to drop the earliest bucket for TT A and
+    // keep a max of 3 buckets forever
+    collector.update();
+    assertEquals(3+1+4, stat1.getValues().get(window).getValue());
+
+    // A new TaskTracker connects and gets a Stat allocated for it
+    Stat stat2 = collector.createStat("TaskTracker B", windows);
+
+    // TT B starts out with 0 buckets even though TT A already has 3
+    assertEquals(0, stat2.getValues().get(window).getValue());
+    stat2.inc(10);
+    collector.update();
+    assertEquals(3+1+4, stat1.getValues().get(window).getValue());
+    assertEquals(0, stat2.getValues().get(window).getValue());
+    stat1.inc(3);
+    stat2.inc(2);
+    // At the next update, we're going to drop the earliest bucket for TT A
+    // but we shouldn't drop the earliest bucket for TT B because it only
+    // has one bucket so far (which would result in a value of 0 instead of 12)
+    collector.update();
+    assertEquals(1+4+3, stat1.getValues().get(window).getValue());
+    assertEquals(12, stat2.getValues().get(window).getValue());
+  }
+
 }



Mime
View raw message