ignite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sboi...@apache.org
Subject [24/43] ignite git commit: IGNITE-4694 Add tests to check there are no memory leaks in PageMemory (pending)
Date Fri, 10 Mar 2017 07:23:45 GMT
IGNITE-4694 Add tests to check there are no memory leaks in PageMemory (pending)


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6743d5c6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6743d5c6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6743d5c6

Branch: refs/heads/ignite-4712
Commit: 6743d5c6b7f7a8b71b9560acf06a86d6cb031523
Parents: 25c1927
Author: Igor Seliverstov <gvvinblade@gmail.com>
Authored: Thu Feb 16 13:41:51 2017 +0300
Committer: Igor Seliverstov <gvvinblade@gmail.com>
Committed: Wed Mar 1 10:44:54 2017 +0300

----------------------------------------------------------------------
 .../IgniteDbMemoryLeakAbstractTest.java         | 118 +++++++++++++++----
 .../database/IgniteDbMemoryLeakIndexedTest.java |  29 -----
 .../IgniteDbMemoryLeakLargeObjectsTest.java     |  16 +--
 .../IgniteDbMemoryLeakLargePagesTest.java       |  13 +-
 .../database/IgniteDbMemoryLeakTest.java        |  14 ++-
 .../IgniteDbMemoryLeakWithExpirationTest.java   |   7 +-
 .../database/IgniteDbMemoryLeakIndexedTest.java |  42 +++++++
 7 files changed, 165 insertions(+), 74 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index bca3af0..819405e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -30,13 +30,37 @@ import org.jetbrains.annotations.NotNull;
  * Base class for memory leaks tests.
  */
 public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTest {
-
+    /** */
+    @SuppressWarnings("WeakerAccess") protected static final int CONCURRENCY_LEVEL = 8;
     /** */
     private volatile Exception ex = null;
 
     /** */
+    private long warmUpEndTime;
+
+    /** */
+    private long endTime;
+
+    /** */
+    private long loadedPages = 0;
+
+    /** */
+    private long delta = 0;
+
+    /** */
+    private long probeCnt = 0;
+
+    /** */
     private static final ThreadLocal<Random> THREAD_LOCAL_RANDOM = new ThreadLocal<>();
 
+    @Override protected void beforeTest() throws Exception {
+        super.beforeTest();
+
+        long startTime = System.nanoTime();
+        warmUpEndTime = startTime + TimeUnit.SECONDS.toNanos(warmUp());
+        endTime = warmUpEndTime + TimeUnit.SECONDS.toNanos(duration());
+    }
+
     /** {@inheritDoc} */
     @Override protected void configure(IgniteConfiguration cfg) {
         cfg.setMetricsLogFrequency(5000);
@@ -44,10 +68,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
 
     /** {@inheritDoc} */
     @Override protected void configure(MemoryConfiguration mCfg) {
-        int concLvl = Runtime.getRuntime().availableProcessors();
-
-        mCfg.setConcurrencyLevel(concLvl);
-        mCfg.setPageCacheSize(1024 * 1024 * concLvl); //minimal possible value
+        mCfg.setConcurrencyLevel(CONCURRENCY_LEVEL);
     }
 
     /**
@@ -57,6 +78,13 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
         return 300;
     }
 
+    /**
+     * @return Warm up duration.
+     */
+    @SuppressWarnings("WeakerAccess") protected int warmUp() {
+        return 300;
+    }
+
     /** {@inheritDoc} */
     @Override protected int gridCount() {
         return 1;
@@ -69,14 +97,14 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
 
     /** {@inheritDoc} */
     @Override protected long getTestTimeout() {
-        return (duration() + 1) * 1000;
+        return (warmUp() + duration() + 1) * 1000; // One extra second to stop all threads
     }
 
     /**
      * @param ig Ignite instance.
      * @return IgniteCache.
      */
-    protected abstract IgniteCache<Object,Object> cache(IgniteEx ig);
+    protected abstract IgniteCache<Object, Object> cache(IgniteEx ig);
 
     /**
      * @return Cache key to perform an operation.
@@ -84,8 +112,8 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
     protected abstract Object key();
 
     /**
-     * @return Cache value to perform an operation.
      * @param key Cache key to perform an operation.
+     * @return Cache value to perform an operation.
      */
     protected abstract Object value(Object key);
 
@@ -99,8 +127,11 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
         switch (getRandom().nextInt(3)) {
             case 0:
                 cache.getAndPut(key, value);
+
+                break;
             case 1:
                 cache.get(key);
+
                 break;
             case 2:
                 cache.getAndRemove(key);
@@ -113,7 +144,7 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
     @NotNull protected static Random getRandom() {
         Random rnd = THREAD_LOCAL_RANDOM.get();
 
-        if(rnd == null){
+        if (rnd == null) {
             rnd = new GridRandom();
             THREAD_LOCAL_RANDOM.set(rnd);
         }
@@ -125,49 +156,96 @@ public abstract class IgniteDbMemoryLeakAbstractTest extends IgniteDbAbstractTes
      * @throws Exception If failed.
      */
     public void testMemoryLeak() throws Exception {
-        final long end = System.nanoTime() + TimeUnit.SECONDS.toNanos(duration());
 
         final IgniteEx ignite = grid(0);
         final IgniteCache<Object, Object> cache = cache(ignite);
 
         Runnable target = new Runnable() {
             @Override public void run() {
-                while (ex == null && System.nanoTime() < end) {
+                while (ex == null && System.nanoTime() < endTime) {
                     try {
                         operation(cache);
-                        check(ignite);
                     }
                     catch (Exception e) {
                         ex = e;
-
                         break;
                     }
                 }
             }
         };
 
-        Thread[] threads = new Thread[Runtime.getRuntime().availableProcessors()];
+        Thread[] threads = new Thread[CONCURRENCY_LEVEL];
+
+        info("Warming up is started.");
 
         for (int i = 0; i < threads.length; i++) {
             threads[i] = new Thread(target);
             threads[i].start();
         }
 
-        for (Thread thread : threads) {
-            thread.join();
+        Thread.sleep(TimeUnit.NANOSECONDS.toMillis(warmUpEndTime - System.nanoTime()));
+
+        info("Warming up is ended.");
+
+        while (System.nanoTime() < endTime) {
+            try {
+                check(ignite);
+            }
+            catch (Exception e) {
+                ex = e;
+
+                break;
+            }
+
+            Thread.sleep(TimeUnit.SECONDS.toMillis(5));
         }
 
-        if(ex != null){
+        for (Thread thread : threads)
+            thread.join();
+
+        if (ex != null)
             throw ex;
-        }
     }
 
     /**
-     * Callback to check the current state
+     * Callback to check the current state.
      *
-     * @param ig Ignite instance
+     * @param ig Ignite instance.
      * @throws Exception If failed.
      */
     protected void check(IgniteEx ig) throws Exception {
+        long pagesActual = ig.context().cache().context().database().pageMemory().loadedPages();
+        long pagesMax = pagesMax();
+
+        assertTrue(
+            "Maximal allowed pages number is exceeded. [allowed=" + pagesMax + "; actual=
" + pagesActual + "]",
+            pagesActual <= pagesMax);
+
+        if (loadedPages > 0) {
+            delta += pagesActual - loadedPages;
+            int allowedDelta = pagesDelta();
+
+            if(probeCnt++ > 12) { // we need some statistic first. Minimal statistic is
taken for a minute.
+                long actualDelta = delta / probeCnt;
+
+                assertTrue(
+                    "Average growth pages in the number is more than expected. [allowed="
+ allowedDelta + "; actual=" + actualDelta + "]",
+                    actualDelta <= allowedDelta);
+            }
+        }
+
+        loadedPages = pagesActual;
+    }
+
+    /**
+     * @return Maximal allowed pages number.
+     */
+    protected abstract long pagesMax();
+
+    /**
+     * @return Expected average number of pages, on which their total number can grow per
5 seconds.
+     */
+    @SuppressWarnings("WeakerAccess") protected int pagesDelta() {
+        return 5;
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
deleted file mode 100644
index acc6c2f..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.database;
-
-/**
- *
- */
-public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakTest {
-
-    /** {@inheritDoc} */
-    @Override protected boolean indexingEnabled() {
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
index 8943743..c1b1c9b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargeObjectsTest.java
@@ -36,23 +36,25 @@ public class IgniteDbMemoryLeakLargeObjectsTest extends IgniteDbMemoryLeakAbstra
 
     /** {@inheritDoc} */
     @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
-        return ig.cache("non-primitive");
+        return ig.cache("large");
     }
 
     /** {@inheritDoc} */
     @Override protected Object key() {
-        return new DbKey(getRandom().nextInt(200_000));
+        return new LargeDbKey(getRandom().nextInt(10_000), 1024);
     }
 
     /** {@inheritDoc} */
     @Override protected Object value(Object key) {
-        return new DbValue(((DbKey)key).val, "test-value-" + getRandom().nextInt(200), getRandom().nextInt(500));
+        return new LargeDbValue("test-value-1-" + getRandom().nextInt(200), "test-value-2-"
+ getRandom().nextInt(200), ARRAY);
     }
 
-    /** {@inheritDoc} */
-    @Override protected void check(IgniteEx ig) {
-        long pages = ig.context().cache().context().database().pageMemory().loadedPages();
+    @Override protected int warmUp() {
+        return 600;
+    }
 
-        assertTrue(pages < 20000);
+    /** {@inheritDoc} */
+    @Override protected long pagesMax() {
+        return 2000000;
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
index 8e4d0b4..9dd9fdc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakLargePagesTest.java
@@ -18,19 +18,16 @@
 package org.apache.ignite.internal.processors.database;
 
 import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.internal.IgniteEx;
 
 /**
  *
  */
 public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakTest {
-
     /** {@inheritDoc} */
     @Override protected void configure(MemoryConfiguration mCfg) {
-        int concLvl = Runtime.getRuntime().availableProcessors();
-        mCfg.setConcurrencyLevel(concLvl);
-        mCfg.setPageCacheSize(1024 * 1024 * concLvl * 16);
+        super.configure(mCfg);
 
+        mCfg.setPageCacheSize(67108864); // The space for 4000 large pages
     }
 
     /** {@inheritDoc} */
@@ -39,9 +36,7 @@ public class IgniteDbMemoryLeakLargePagesTest extends IgniteDbMemoryLeakTest
{
     }
 
     /** {@inheritDoc} */
-    @Override protected void check(IgniteEx ig) {
-        long pages = ig.context().cache().context().database().pageMemory().loadedPages();
-
-        assertTrue(pages < 4000);
+    @Override protected long pagesMax() {
+        return 4000;
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
index 81d831b..94e419d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakTest.java
@@ -18,6 +18,7 @@
 package org.apache.ignite.internal.processors.database;
 
 import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.MemoryConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 
 /**
@@ -25,6 +26,13 @@ import org.apache.ignite.internal.IgniteEx;
  */
 public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest {
     /** {@inheritDoc} */
+    @Override protected void configure(MemoryConfiguration mCfg) {
+        super.configure(mCfg);
+
+        mCfg.setPageCacheSize(20971520); // The space for 20000 pages
+    }
+
+    /** {@inheritDoc} */
     @Override protected IgniteCache<Object, Object> cache(IgniteEx ig) {
         return ig.cache("non-primitive");
     }
@@ -40,9 +48,7 @@ public class IgniteDbMemoryLeakTest extends IgniteDbMemoryLeakAbstractTest
{
     }
 
     /** {@inheritDoc} */
-    @Override protected void check(IgniteEx ig) {
-        long pages = ig.context().cache().context().database().pageMemory().loadedPages();
-
-        assertTrue(pages < 20000);
+    @Override protected long pagesMax() {
+        return 20_000;
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
index a31ffb4..c1e279b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakWithExpirationTest.java
@@ -39,10 +39,7 @@ public class IgniteDbMemoryLeakWithExpirationTest extends IgniteDbMemoryLeakTest
         return ig.cache("non-primitive").withExpiryPolicy(EXPIRY);
     }
 
-    /** {@inheritDoc} */
-    @Override protected void check(IgniteEx ig) {
-        long pages = ig.context().cache().context().database().pageMemory().loadedPages();
-
-        assertTrue(pages < 7000);
+    @Override protected long pagesMax() {
+        return 7000;
     }
 }

http://git-wip-us.apache.org/repos/asf/ignite/blob/6743d5c6/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
new file mode 100644
index 0000000..c917bc1
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakIndexedTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import org.apache.ignite.configuration.MemoryConfiguration;
+
+/**
+ *
+ */
+public class IgniteDbMemoryLeakIndexedTest extends IgniteDbMemoryLeakTest {
+    /** {@inheritDoc} */
+    @Override protected void configure(MemoryConfiguration mCfg) {
+        super.configure(mCfg);
+
+        mCfg.setPageCacheSize(25165824); // The space for 24000 pages
+    }
+
+    /** {@inheritDoc} */
+    @Override protected boolean indexingEnabled() {
+        return true;
+    }
+
+    /** {@inheritDoc} */
+    @Override protected long pagesMax() {
+        return 24_000;
+    }
+}


Mime
View raw message