hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rkan...@apache.org
Subject hadoop git commit: MAPREDUCE-6199. AbstractCounters are not reset completely on deserialization (adhoot via rkanter)
Date Fri, 19 Dec 2014 23:44:41 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 8ee40a158 -> f9341c1e2


MAPREDUCE-6199. AbstractCounters are not reset completely on deserialization (adhoot via rkanter)

(cherry picked from commit 390a7c12f543b2c94a74f08d6d2a28410472043a)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9341c1e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9341c1e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9341c1e

Branch: refs/heads/branch-2
Commit: f9341c1e2cffc4e3ac37cb7da4e535ca5c6d3913
Parents: 8ee40a1
Author: Robert Kanter <rkanter@apache.org>
Authored: Fri Dec 19 15:43:48 2014 -0800
Committer: Robert Kanter <rkanter@apache.org>
Committed: Fri Dec 19 15:44:01 2014 -0800

----------------------------------------------------------------------
 hadoop-mapreduce-project/CHANGES.txt            |  3 ++
 .../mapreduce/counters/AbstractCounters.java    |  4 ++
 .../hadoop/mapreduce/counters/Limits.java       |  7 ++++
 .../apache/hadoop/mapreduce/TestCounters.java   | 39 +++++++++++++++++++-
 4 files changed, 52 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9341c1e/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 2db4cbd..6721e56 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -58,6 +58,9 @@ Release 2.7.0 - UNRELEASED
     MAPREDUCE-6045. need close the DataInputStream after open it in
     TestMapReduce.java (zxu via rkanter)
 
+    MAPREDUCE-6199. AbstractCounters are not reset completely on
+    deserialization (adhoot via rkanter)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9341c1e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
index 401bbb2..dd81ebb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
@@ -307,6 +307,10 @@ public abstract class AbstractCounters<C extends Counter,
       fgroups.put(group.getName(), group);
     }
     int numGroups = WritableUtils.readVInt(in);
+    if (!groups.isEmpty()) {
+      groups.clear();
+      limits.reset();
+    }
     while (numGroups-- > 0) {
       limits.checkGroups(groups.size() + 1);
       G group = groupFactory.newGenericGroup(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9341c1e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
index 3821694..9546c8d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/Limits.java
@@ -124,8 +124,15 @@ public class Limits {
     return firstViolation;
   }
 
+  // This allows initialization of global settings and not for an instance
   public static synchronized void reset(Configuration conf) {
     isInited = false;
     init(conf);
   }
+
+  // This allows resetting of an instance to allow reuse
+  public synchronized void reset() {
+    totalCounters = 0;
+    firstViolation = null;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9341c1e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
index 83d689c..0215568 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestCounters.java
@@ -17,8 +17,12 @@
  */
 package org.apache.hadoop.mapreduce;
 
+import java.io.IOException;
 import java.util.Random;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
 import org.junit.Test;
 import static org.junit.Assert.*;
 
@@ -70,7 +74,40 @@ public class TestCounters {
       testMaxGroups(new Counters());
     }
   }
-  
+
+  @Test public void testResetOnDeserialize() throws IOException {
+    // Allow only one counterGroup
+    Configuration conf = new Configuration();
+    conf.setInt(MRJobConfig.COUNTER_GROUPS_MAX_KEY, 1);
+    Limits.init(conf);
+
+    Counters countersWithOneGroup = new Counters();
+    countersWithOneGroup.findCounter("firstOf1Allowed", "First group");
+    boolean caughtExpectedException = false;
+    try {
+      countersWithOneGroup.findCounter("secondIsTooMany", "Second group");
+    }
+    catch (LimitExceededException _) {
+      caughtExpectedException = true;
+    }
+
+    assertTrue("Did not throw expected exception",
+        caughtExpectedException);
+
+    Counters countersWithZeroGroups = new Counters();
+    DataOutputBuffer out = new DataOutputBuffer();
+    countersWithZeroGroups.write(out);
+
+    DataInputBuffer in = new DataInputBuffer();
+    in.reset(out.getData(), out.getLength());
+
+    countersWithOneGroup.readFields(in);
+
+    // After reset one should be able to add a group
+    countersWithOneGroup.findCounter("firstGroupAfterReset", "After reset " +
+        "limit should be set back to zero");
+  }
+
   @Test
   public void testCountersIncrement() {
     Counters fCounters = new Counters();


Mime
View raw message