hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject [2/2] hadoop git commit: HADOOP-12706. TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp times out occasionally. Contributed by Sangjin Lee and Colin Patrick McCabe (cherry picked from commit cdf88952599a43b1ef5adda792bfb195c7529fad)
Date Thu, 14 Jan 2016 23:48:08 GMT
HADOOP-12706. TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp times out occasionally.
Contributed by Sangjin Lee and Colin Patrick McCabe
(cherry picked from commit cdf88952599a43b1ef5adda792bfb195c7529fad)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/81cda760
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/81cda760
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/81cda760

Branch: refs/heads/branch-2
Commit: 81cda7608b7c30e485fb2703d7429d46da85c182
Parents: 86456b0
Author: Jason Lowe <jlowe@apache.org>
Authored: Thu Jan 14 23:45:03 2016 +0000
Committer: Jason Lowe <jlowe@apache.org>
Committed: Thu Jan 14 23:47:29 2016 +0000

----------------------------------------------------------------------
 hadoop-common-project/hadoop-common/CHANGES.txt |  6 ++++++
 .../apache/hadoop/fs/FCStatisticsBaseTest.java  | 21 +++++++++++++++-----
 2 files changed, 22 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/81cda760/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index f18d85c..d5c8788 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1025,6 +1025,9 @@ Release 2.7.3 - UNRELEASED
     HADOOP-12107. long running apps may have a huge number of StatisticsData
     instances under FileSystem (Sangjin Lee via Ming Ma)
 
+    HADOOP-12706. TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp
+    times out occasionally (Sangjin Lee and Colin Patrick McCabe via jlowe)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES
@@ -1849,6 +1852,9 @@ Release 2.6.4 - UNRELEASED
     HADOOP-12107. long running apps may have a huge number of StatisticsData
     instances under FileSystem (Sangjin Lee via Ming Ma)
 
+    HADOOP-12706. TestLocalFsFCStatistics#testStatisticsThreadLocalDataCleanUp
+    times out occasionally (Sangjin Lee and Colin Patrick McCabe via jlowe)
+
 Release 2.6.3 - 2015-12-17
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/81cda760/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
index 3e33362..2e208d2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
@@ -32,6 +32,8 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.FileSystem.Statistics;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
@@ -46,6 +48,8 @@ import com.google.common.util.concurrent.Uninterruptibles;
  * </p>
  */
 public abstract class FCStatisticsBaseTest {
+  private static final Log LOG = LogFactory.getLog(FCStatisticsBaseTest.class);
+
   static protected int blockSize = 512;
   static protected int numBlocks = 1;
   
@@ -110,7 +114,7 @@ public abstract class FCStatisticsBaseTest {
     fc.delete(filePath, true);
   }
 
-  @Test(timeout=60000)
+  @Test(timeout=70000)
   public void testStatisticsThreadLocalDataCleanUp() throws Exception {
     final Statistics stats = new Statistics("test");
     // create a small thread pool to test the statistics
@@ -137,17 +141,24 @@ public abstract class FCStatisticsBaseTest {
     es.shutdownNow();
     es.awaitTermination(1, TimeUnit.MINUTES);
     es = null;
-    System.gc();
+    System.gc(); // force GC to garbage collect threads
 
-    // wait for up to 10 seconds
+    // wait for up to 60 seconds
     GenericTestUtils.waitFor(new Supplier<Boolean>() {
           @Override
           public Boolean get() {
             int size = stats.getAllThreadLocalDataSize();
             allDataSize.set(size);
-            return size == 0;
+            if (size == 0) {
+              return true;
+            }
+            LOG.warn("not all references have been cleaned up; still " +
+                allDataSize.get() + " references left");
+            LOG.warn("triggering another GC");
+            System.gc();
+            return false;
           }
-        }, 1000, 10*1000);
+        }, 500, 60*1000);
     Assert.assertEquals(0, allDataSize.get());
     Assert.assertEquals(size, stats.getReadOps());
   }


Mime
View raw message