Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 83EB8200B2B for ; Mon, 13 Jun 2016 12:16:07 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 828DF160A3C; Mon, 13 Jun 2016 10:16:07 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id A51D9160A19 for ; Mon, 13 Jun 2016 12:16:06 +0200 (CEST) Received: (qmail 61127 invoked by uid 500); 13 Jun 2016 10:16:05 -0000 Mailing-List: contact commits-help@ignite.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ignite.apache.org Delivered-To: mailing list commits@ignite.apache.org Received: (qmail 61114 invoked by uid 99); 13 Jun 2016 10:16:05 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 13 Jun 2016 10:16:05 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8F0D1E01E2; Mon, 13 Jun 2016 10:16:05 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dmagda@apache.org To: commits@ignite.apache.org Date: Mon, 13 Jun 2016 10:16:05 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/2] ignite git commit: IGNITE-2616 - NonHeap memory usage metrics don't work as expected. Reviewed and merged by Denis Magda. archived-at: Mon, 13 Jun 2016 10:16:07 -0000 Repository: ignite Updated Branches: refs/heads/master 10bbc36df -> 7d12c851d IGNITE-2616 - NonHeap memory usage metrics don't work as expected. Reviewed and merged by Denis Magda. Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2040c3dd Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2040c3dd Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2040c3dd Branch: refs/heads/master Commit: 2040c3dda9dd5b4796797ed37a0fc01890dcb861 Parents: 17020cc Author: Vladislav Pyatkov Authored: Mon Jun 13 13:13:54 2016 +0300 Committer: Denis Magda Committed: Mon Jun 13 13:13:54 2016 +0300 ---------------------------------------------------------------------- .../discovery/GridDiscoveryManager.java | 23 ++++- .../internal/ClusterNodeMetricsSelfTest.java | 101 ++++++++++++++++++- 2 files changed, 121 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/2040c3dd/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java index 0d3ff0a..bbf3ebd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java @@ -113,6 +113,7 @@ import org.apache.ignite.spi.discovery.DiscoverySpiHistorySupport; import org.apache.ignite.spi.discovery.DiscoverySpiListener; import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator; import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport; +import org.apache.ignite.spi.discovery.tcp.internal.TcpDiscoveryNode; import org.apache.ignite.thread.IgniteThread; import org.jetbrains.annotations.Nullable; import org.jsr166.ConcurrentHashMap8; @@ -878,7 +879,7 @@ public class GridDiscoveryManager extends GridManagerAdapter { nm.setHeapMemoryMaximum(metrics.getHeapMemoryMaximum()); nm.setHeapMemoryTotal(metrics.getHeapMemoryMaximum()); nm.setNonHeapMemoryInitialized(metrics.getNonHeapMemoryInitialized()); - nm.setNonHeapMemoryUsed(metrics.getNonHeapMemoryUsed()); + nonHeapMemoryUsed(nm); nm.setNonHeapMemoryCommitted(metrics.getNonHeapMemoryCommitted()); nm.setNonHeapMemoryMaximum(metrics.getNonHeapMemoryMaximum()); nm.setNonHeapMemoryTotal(metrics.getNonHeapMemoryMaximum()); @@ -906,6 +907,26 @@ public class GridDiscoveryManager extends GridManagerAdapter { return nm; } + /** + * @param nm Initializing metrics snapshot. + */ + private void nonHeapMemoryUsed(ClusterMetricsSnapshot nm) { + long nonHeapUsed = metrics.getNonHeapMemoryUsed(); + + Map nodeCacheMetrics = cacheMetrics(); + + if (nodeCacheMetrics != null) { + for (Map.Entry entry : nodeCacheMetrics.entrySet()) { + CacheMetrics e = entry.getValue(); + + if (e != null) + nonHeapUsed += e.getOffHeapAllocatedSize(); + } + } + + nm.setNonHeapMemoryUsed(nonHeapUsed); + } + /** {@inheritDoc} */ @Override public Map cacheMetrics() { Collection> caches = ctx.cache().internalCaches(); http://git-wip-us.apache.org/repos/asf/ignite/blob/2040c3dd/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsSelfTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsSelfTest.java index d684446..77359ab 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsSelfTest.java @@ -22,7 +22,11 @@ import java.util.UUID; import java.util.concurrent.CountDownLatch; import org.apache.ignite.GridTestTask; import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheMemoryMode; +import org.apache.ignite.cache.eviction.fifo.FifoEvictionPolicy; import org.apache.ignite.cluster.ClusterMetrics; +import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.events.Event; import org.apache.ignite.internal.processors.task.GridInternal; @@ -52,6 +56,18 @@ public class ClusterNodeMetricsSelfTest extends GridCommonAbstractTest { /** Number of messages. */ private static final int MSG_CNT = 3; + /** Size of value in bytes. */ + private static final int VAL_SIZE = 512 * 1024; + + /** Amount of cache entries. */ + private static final int MAX_VALS_AMOUNT = 400; + + /** With OFFHEAP_VALUES policy. */ + private final String OFF_HEAP_VALUE_NAME = "offHeapValuesCfg"; + + /** With ONHEAP_TIERED policy. */ + private final String ON_HEAP_TIERED_NAME = "onHeapTieredCfg"; + /** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { startGrid(); @@ -75,7 +91,89 @@ public class ClusterNodeMetricsSelfTest extends GridCommonAbstractTest { cfg.setCacheConfiguration(); cfg.setMetricsUpdateFrequency(0); - return cfg; + CacheConfiguration offHeapValuesCfg = defaultCacheConfiguration(); + offHeapValuesCfg.setName(OFF_HEAP_VALUE_NAME); + offHeapValuesCfg.setStatisticsEnabled(true); + offHeapValuesCfg.setMemoryMode(CacheMemoryMode.OFFHEAP_VALUES); + offHeapValuesCfg.setOffHeapMaxMemory(MAX_VALS_AMOUNT * VAL_SIZE); + + CacheConfiguration onHeapTieredCfg = defaultCacheConfiguration(); + onHeapTieredCfg.setName(ON_HEAP_TIERED_NAME); + onHeapTieredCfg.setStatisticsEnabled(true); + onHeapTieredCfg.setMemoryMode(CacheMemoryMode.ONHEAP_TIERED); + onHeapTieredCfg.setOffHeapMaxMemory(MAX_VALS_AMOUNT * VAL_SIZE); + + FifoEvictionPolicy plc = new FifoEvictionPolicy(); + plc.setMaxMemorySize(MAX_VALS_AMOUNT * VAL_SIZE); + plc.setMaxSize(0); + + onHeapTieredCfg.setEvictionPolicy(plc); + + return cfg.setCacheConfiguration(offHeapValuesCfg, onHeapTieredCfg); + } + + /** + * @throws Exception If failed. + */ + public void testAllocatedMemory() throws Exception { + Ignite ignite = grid(); + + final IgniteCache onHeapCache = ignite.getOrCreateCache(ON_HEAP_TIERED_NAME); + final IgniteCache offHeapCache = ignite.getOrCreateCache(OFF_HEAP_VALUE_NAME); + + long prevTieredOffHeapSize = onHeapCache.metrics().getOffHeapAllocatedSize(); + long prevValuesOffHeapSize = offHeapCache.metrics().getOffHeapAllocatedSize(); + + assertEquals(0, prevTieredOffHeapSize); + assertEquals(0, prevValuesOffHeapSize); + + long prevClusterNonHeapMemoryUsed = ignite.cluster().metrics().getNonHeapMemoryUsed(); + + fillCache(onHeapCache); + + assertTrue(onHeapCache.metrics().getOffHeapAllocatedSize() > (MAX_VALS_AMOUNT - 5) + * VAL_SIZE + prevTieredOffHeapSize); + assertEquals(0, offHeapCache.metrics().getOffHeapAllocatedSize()); + + assertTrue(prevClusterNonHeapMemoryUsed < ignite.cluster().metrics().getNonHeapMemoryUsed()); + + prevClusterNonHeapMemoryUsed = ignite.cluster().metrics().getNonHeapMemoryUsed(); + prevTieredOffHeapSize = onHeapCache.metrics().getOffHeapAllocatedSize(); + + fillCache(offHeapCache); + + assertTrue(offHeapCache.metrics().getOffHeapAllocatedSize() > (MAX_VALS_AMOUNT - 5) * VAL_SIZE); + assertEquals(prevTieredOffHeapSize, onHeapCache.metrics().getOffHeapAllocatedSize()); + assertTrue((MAX_VALS_AMOUNT - 5) * VAL_SIZE + prevClusterNonHeapMemoryUsed < + ignite.cluster().metrics().getNonHeapMemoryUsed()); + } + + /** + * Fill cache with values. + * @param cache Ignite cache. + * @throws Exception If failed. + */ + private void fillCache(final IgniteCache cache) throws Exception{ + final byte[] val = new byte[VAL_SIZE]; + + for (int i = 0; i < MAX_VALS_AMOUNT * 4; i++) + cache.put(i, val); + + // Let metrics update twice. + final CountDownLatch latch = new CountDownLatch(2); + + grid().events().localListen(new IgnitePredicate() { + @Override public boolean apply(Event evt) { + assert evt.type() == EVT_NODE_METRICS_UPDATED; + + latch.countDown(); + + return true; + } + }, EVT_NODE_METRICS_UPDATED); + + // Wait for metrics update. + latch.await(); } /** @@ -267,7 +365,6 @@ public class ClusterNodeMetricsSelfTest extends GridCommonAbstractTest { assert metrics0.getHeapMemoryUsed() > 0; assert metrics0.getHeapMemoryTotal() > 0; - assert metrics0.getNonHeapMemoryMaximum() > 0; } /**