From commits-return-29017-archive-asf-public=cust-asf.ponee.io@geode.apache.org Wed Oct 31 16:06:27 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 900AE18065D for ; Wed, 31 Oct 2018 16:06:25 +0100 (CET) Received: (qmail 30133 invoked by uid 500); 31 Oct 2018 15:06:24 -0000 Mailing-List: contact commits-help@geode.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@geode.apache.org Delivered-To: mailing list commits@geode.apache.org Received: (qmail 30124 invoked by uid 99); 31 Oct 2018 15:06:24 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 31 Oct 2018 15:06:24 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id D42BB81E50; Wed, 31 Oct 2018 15:06:23 +0000 (UTC) Date: Wed, 31 Oct 2018 15:06:23 +0000 To: "commits@geode.apache.org" Subject: [geode] branch develop updated: GEODE-5943: Refactor EvictionDUnitTest and OffHeapEvictionDUnitTest (#2734) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <154099838226.14434.7884831679802362606@gitbox.apache.org> From: jinmeiliao@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: geode X-Git-Refname: refs/heads/develop X-Git-Reftype: branch X-Git-Oldrev: 5adf016a99fe037c6a7cc53828168632c1def482 X-Git-Newrev: 6646cada89ce0f300bc7478f1283bf000dc07a77 X-Git-Rev: 6646cada89ce0f300bc7478f1283bf000dc07a77 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. jinmeiliao pushed a commit to branch develop in repository https://gitbox.apache.org/repos/asf/geode.git The following commit(s) were added to refs/heads/develop by this push: new 6646cad GEODE-5943: Refactor EvictionDUnitTest and OffHeapEvictionDUnitTest (#2734) 6646cad is described below commit 6646cada89ce0f300bc7478f1283bf000dc07a77 Author: jinmeiliao AuthorDate: Wed Oct 31 08:06:11 2018 -0700 GEODE-5943: Refactor EvictionDUnitTest and OffHeapEvictionDUnitTest (#2734) * GEODE-5943: Refactor EvictionDUnitTest and OffHeapEvictionDUnitTest * break the test into dunit and integration tests * use the rules to better see what's the setup and what the test is doing --- .../internal/cache/eviction/EvictionDUnitTest.java | 428 +++++++++++------ .../internal/cache/eviction/EvictionTestBase.java | 534 --------------------- .../cache/eviction/OffHeapEvictionDUnitTest.java | 148 ------ .../cache/eviction/EvictionIntegrationTest.java | 204 ++++++++ .../apache/geode/test/junit/rules/VMProvider.java | 10 +- 5 files changed, 489 insertions(+), 835 deletions(-) diff --git a/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/eviction/EvictionDUnitTest.java b/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/eviction/EvictionDUnitTest.java old mode 100755 new mode 100644 index 5ca8608..04e7f07 --- a/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/eviction/EvictionDUnitTest.java +++ b/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/eviction/EvictionDUnitTest.java @@ -14,196 +14,330 @@ */ package org.apache.geode.internal.cache.eviction; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import org.junit.Assert; +import static java.lang.Math.abs; +import static org.apache.geode.distributed.ConfigurationProperties.OFF_HEAP_MEMORY_SIZE; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.util.Arrays; +import java.util.Collection; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; + +import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; -import org.apache.geode.cache.EvictionAlgorithm; +import org.apache.geode.cache.Cache; +import org.apache.geode.cache.DataPolicy; +import org.apache.geode.cache.EvictionAction; import org.apache.geode.cache.EvictionAttributes; -import org.apache.geode.internal.cache.BucketRegion; -import org.apache.geode.internal.cache.DistributedRegion; +import org.apache.geode.cache.Region; +import org.apache.geode.cache.RegionShortcut; +import org.apache.geode.internal.OSProcess; +import org.apache.geode.internal.cache.GemFireCacheImpl; +import org.apache.geode.internal.cache.InternalCache; +import org.apache.geode.internal.cache.LocalRegion; import org.apache.geode.internal.cache.PartitionedRegion; -import org.apache.geode.test.dunit.LogWriterUtils; +import org.apache.geode.internal.cache.control.HeapMemoryMonitor; +import org.apache.geode.internal.cache.control.InternalResourceManager; +import org.apache.geode.internal.cache.control.InternalResourceManager.ResourceType; +import org.apache.geode.internal.cache.control.MemoryEvent; +import org.apache.geode.internal.cache.control.MemoryThresholds; +import org.apache.geode.test.awaitility.GeodeAwaitility; +import org.apache.geode.test.dunit.IgnoredException; +import org.apache.geode.test.dunit.SerializableRunnableIF; +import org.apache.geode.test.dunit.rules.ClusterStartupRule; +import org.apache.geode.test.dunit.rules.MemberVM; import org.apache.geode.test.junit.categories.EvictionTest; +import org.apache.geode.test.junit.rules.ServerStarterRule; +import org.apache.geode.test.junit.rules.VMProvider; +import org.apache.geode.test.junit.runners.CategoryWithParameterizedRunnerFactory; +@RunWith(Parameterized.class) +@Parameterized.UseParametersRunnerFactory(CategoryWithParameterizedRunnerFactory.class) @Category({EvictionTest.class}) -public class EvictionDUnitTest extends EvictionTestBase { +public class EvictionDUnitTest { - @Test - public void testDummyInlineNCentralizedEviction() { - prepareScenario1(EvictionAlgorithm.LRU_HEAP, 0); - putData("PR1", 50, 1); + @Parameterized.Parameters(name = "offHeap={0}") + public static Collection booleans() { + return Arrays.asList(true, false); + } - final int expectedEviction1 = getExpectedEvictionRatioOnVm(dataStore1); - final int expectedEviction2 = getExpectedEvictionRatioOnVm(dataStore2); + @Parameterized.Parameter + public static boolean offHeap; - raiseFakeNotification(dataStore1, "PR1", expectedEviction1); - raiseFakeNotification(dataStore2, "PR1", expectedEviction2); - validateNoOfEvictions("PR1", expectedEviction1 + expectedEviction2); + @Rule + public ClusterStartupRule cluster = new ClusterStartupRule(); - putData("PR1", 4, 1); - validateNoOfEvictions("PR1", 4 + expectedEviction1 + expectedEviction2); - } + private MemberVM server0, server1; - @Test - public void testThreadPoolSize() { - prepareScenario1(EvictionAlgorithm.LRU_HEAP, 0); - putData("PR1", 50, 1); - raiseFakeNotification(dataStore1, "PR1", getExpectedEvictionRatioOnVm(dataStore1)); - verifyThreadPoolTaskCount(HeapEvictor.MAX_EVICTOR_THREADS); + @Before + public void before() { + int locatorPort = ClusterStartupRule.getDUnitLocatorPort(); + Properties properties = new Properties(); + if (offHeap) { + properties.setProperty(OFF_HEAP_MEMORY_SIZE, "200m"); + } + server0 = cluster.startServerVM(0, s -> s.withNoCacheServer() + .withProperties(properties).withConnectionToLocator(locatorPort)); + server1 = cluster.startServerVM(1, s -> s.withNoCacheServer() + .withProperties(properties).withConnectionToLocator(locatorPort)); + + VMProvider.invokeInEveryMember(() -> { + HeapMemoryMonitor.setTestDisableMemoryUpdates(true); + System.setProperty("gemfire.memoryEventTolerance", "0"); + InternalCache cache = ClusterStartupRule.getCache(); + if (offHeap) { + cache.getResourceManager().setEvictionOffHeapPercentage(85); + cache.getInternalResourceManager().getOffHeapMonitor().stopMonitoring(true); + } else { + cache.getResourceManager().setEvictionHeapPercentage(85); + } + }, server0, server1); } @Test - public void testCentralizedEvictionnForDistributedRegionWithDummyEvent() { - prepareScenario1(EvictionAlgorithm.LRU_HEAP, 0); - createDistributedRegion(); - putDataInDistributedRegion(50, 1); - raiseFakeNotification(dataStore1, "DR1", getExpectedEvictionRatioOnVm(dataStore1)); - } + public void testDummyInlineNCentralizedEviction() { + VMProvider.invokeInEveryMember(() -> { + ServerStarterRule server = (ServerStarterRule) ClusterStartupRule.memberStarter; + server.createPartitionRegion("PR1", + f -> f.setOffHeap(offHeap).setEvictionAttributes( + EvictionAttributes.createLRUHeapAttributes(null, EvictionAction.LOCAL_DESTROY)), + f -> f.setTotalNumBuckets(4).setRedundantCopies(0)); + + }, server0, server1); + + server0.invoke(() -> { + Region region = ClusterStartupRule.getCache().getRegion("PR1"); + for (int counter = 1; counter <= 50; counter++) { + region.put(counter, new byte[1024 * 1024]); + } + }); + + int server0ExpectedEviction = server0.invoke(EvictionDUnitTest::getExpectedEviction); + int server1ExpectedEviction = server1.invoke(EvictionDUnitTest::getExpectedEviction); + + server0.invoke(() -> { + GemFireCacheImpl cache = (GemFireCacheImpl) ClusterStartupRule.getCache(); + PartitionedRegion region = (PartitionedRegion) cache.getRegion("PR1"); + GeodeAwaitility.await() + .until(() -> (abs(region.getTotalEvictions() - server0ExpectedEviction) <= 1)); + }); + + server1.invoke(() -> { + GemFireCacheImpl cache = (GemFireCacheImpl) ClusterStartupRule.getCache(); + PartitionedRegion region = (PartitionedRegion) cache.getRegion("PR1"); + GeodeAwaitility.await() + .until(() -> (abs(region.getTotalEvictions() - server1ExpectedEviction) <= 1)); + }); + + Long server0EvictionCount = server0.invoke(() -> getActualEviction("PR1")); + Long server1EvictionCount = server1.invoke(() -> getActualEviction("PR1")); + + assertThat(server0EvictionCount + server1EvictionCount) + .isEqualTo(server0ExpectedEviction + server1ExpectedEviction); + + // do 4 puts again in PR1 + server0.invoke(() -> { + Region region = ClusterStartupRule.getCache().getRegion("PR1"); + for (int counter = 1; counter <= 4; counter++) { + region.put(counter, new byte[1024 * 1024]); + } + }); - /** - * Test Case Description: 2 VM's. 2 PR's. 4 buckets each PR. PR1 has action -Local destroy and PR2 - * has action - Overflow To Disk. - * - * Test Case verifies:If naturally Eviction up and eviction Down events are raised. Centralized - * and Inline eviction are happening. All this verification is done thorugh logs. It also verifies - * that during eviction, if one node goes down and then comes up again causing GII to take place, - * the system doesnot throw an OOME. - */ - @Test - public void testEvictionWithNodeDown() { - prepareScenario2(EvictionAlgorithm.LRU_HEAP, "PR3", "PR4"); - putDataInDataStore3("PR3", 100, 1); - fakeNotification(); - print("PR3"); - killVm(); - bringVMBackToLife(); - assertEquals(100, getPRSize("PR3")); - assertEquals(0, getPRSize("PR4")); + server0EvictionCount = server0.invoke(() -> getActualEviction("PR1")); + server1EvictionCount = server1.invoke(() -> getActualEviction("PR1")); + + assertThat(server0EvictionCount + server1EvictionCount) + .isEqualTo(4 + server0ExpectedEviction + server1ExpectedEviction); } @Test - public void testEntryLruEvictions() { - int extraEntries = 1; - createCache(); - maxEntries = 3; - createPartitionedRegion(true, EvictionAlgorithm.LRU_ENTRY, "PR1", 4, 1, 1000, maxEntries); + public void testThreadPoolSize() { + VMProvider.invokeInEveryMember(() -> { + ServerStarterRule server = (ServerStarterRule) ClusterStartupRule.memberStarter; + server.createPartitionRegion("PR1", + f -> f.setOffHeap(offHeap).setEvictionAttributes( + EvictionAttributes.createLRUHeapAttributes(null, EvictionAction.LOCAL_DESTROY)), + f -> f.setTotalNumBuckets(4).setRedundantCopies(0)); + + }, server0, server1); + + server0.invoke(() -> { + GemFireCacheImpl cache = (GemFireCacheImpl) ClusterStartupRule.getCache(); + PartitionedRegion region = (PartitionedRegion) cache.getRegion("PR1"); + for (int counter = 1; counter <= 50; counter++) { + region.put(counter, new byte[1024 * 1024]); + } - final PartitionedRegion pr = (PartitionedRegion) cache.getRegion("PR1"); - LogWriterUtils.getLogWriter().info("PR- " + pr.getEvictionAttributes().getMaximum()); + int expectedEviction = getExpectedEviction(); + GeodeAwaitility.await() + .until(() -> (abs(region.getTotalEvictions() - expectedEviction) <= 1)); - for (int counter = 1; counter <= maxEntries + extraEntries; counter++) { - pr.put(new Integer(counter), new byte[1 * 1024 * 1024]); - } + ExecutorService evictorThreadPool = getEvictor(cache).getEvictorThreadPool(); + if (evictorThreadPool != null) { + long taskCount = ((ThreadPoolExecutor) evictorThreadPool).getTaskCount(); + assertThat(taskCount).isLessThanOrEqualTo(HeapEvictor.MAX_EVICTOR_THREADS); + } - Assert.assertEquals(extraEntries, pr.getTotalEvictions()); + }); } @Test - public void testEntryLru() { - createCache(); - maxEntries = 12; - createPartitionedRegion(true, EvictionAlgorithm.LRU_ENTRY, "PR1", 4, 1, 1000, maxEntries); - - final PartitionedRegion pr = (PartitionedRegion) cache.getRegion("PR1"); - LogWriterUtils.getLogWriter().info("PR- " + pr.getEvictionAttributes().getMaximum()); - for (int i = 0; i < 3; i++) { - // assume mod-based hashing for bucket creation - pr.put(new Integer(i), "value0"); - pr.put(new Integer(i + pr.getPartitionAttributes().getTotalNumBuckets()), "value1"); - pr.put(new Integer(i + (pr.getPartitionAttributes().getTotalNumBuckets()) * 2), "value2"); - } - pr.put(new Integer(3), "value0"); + public void testCheckEntryLruEvictionsIn2DataStore() { + int maxEntries = 20; + VMProvider.invokeInEveryMember(() -> { + ServerStarterRule server = (ServerStarterRule) ClusterStartupRule.memberStarter; + server.createPartitionRegion("PR1", + f -> f.setOffHeap(offHeap) + .setEvictionAttributes( + EvictionAttributes.createLRUEntryAttributes(maxEntries, + EvictionAction.LOCAL_DESTROY)), + f -> f.setTotalNumBuckets(4).setRedundantCopies(0)); + + }, server0, server1); + + // put 60 entries in server0's region + server0.invoke(() -> { + Region region = ClusterStartupRule.getCache().getRegion("PR1"); + for (int counter = 1; counter <= 60; counter++) { + region.put(counter, new byte[1024 * 1024]); + } + }); - for (int i = 0; i < 2; i++) { - pr.put(new Integer(i + pr.getPartitionAttributes().getTotalNumBuckets()) * 3, "value1"); - } - assertEquals(0, pr.getTotalEvictions()); + Long server0EvictionCount = server0.invoke(() -> getActualEviction("PR1")); + Long server1EvictionCount = server1.invoke(() -> getActualEviction("PR1")); + + assertThat(server0EvictionCount + server1EvictionCount).isEqualTo(20); } @Test - public void testCheckEntryLruEvictionsIn1DataStore() { - int extraEntries = 10; - createCache(); - maxEntries = 20; - createPartitionedRegion(true, EvictionAlgorithm.LRU_ENTRY, "PR1", 5, 1, 1000, maxEntries); + public void testCentralizedEvictionForDistributedRegionWithDummyEvent() { + server0.invoke(() -> { + ServerStarterRule server = (ServerStarterRule) ClusterStartupRule.memberStarter; + LocalRegion dr1 = + (LocalRegion) server.createRegion(RegionShortcut.LOCAL, "DR1", + f -> f.setOffHeap(offHeap).setDataPolicy(DataPolicy.NORMAL).setEvictionAttributes( + EvictionAttributes.createLRUHeapAttributes(null, EvictionAction.LOCAL_DESTROY))); + for (int counter = 1; counter <= 50; counter++) { + dr1.put(counter, new byte[1024 * 1024]); + } - final PartitionedRegion pr = (PartitionedRegion) cache.getRegion("PR1"); - LogWriterUtils.getLogWriter().info("PR- " + pr.getEvictionAttributes().getMaximum()); + int expectedEviction = getExpectedEviction(); - for (int counter = 1; counter <= maxEntries + extraEntries; counter++) { - pr.put(new Integer(counter), new byte[1 * 1024 * 1024]); - } + GeodeAwaitility.await().until(() -> (abs(dr1.getTotalEvictions() - expectedEviction) <= 1)); - assertEquals(extraEntries, pr.getTotalEvictions()); - - for (final Iterator i = pr.getDataStore().getAllLocalBuckets().iterator(); i.hasNext();) { - final Map.Entry entry = (Map.Entry) i.next(); - final BucketRegion bucketRegion = (BucketRegion) entry.getValue(); - if (bucketRegion == null) { - continue; - } - LogWriterUtils.getLogWriter().info("FINAL bucket= " + bucketRegion.getFullPath() + "size= " - + bucketRegion.size() + " count= " + bucketRegion.entryCount()); - assertEquals(4, bucketRegion.size()); - } + assertThat(dr1.getTotalEvictions()).isEqualTo(expectedEviction); + }); } - @Test - public void testCheckEntryLruEvictionsIn2DataStore() { - maxEntries = 20; - prepareScenario1(EvictionAlgorithm.LRU_ENTRY, maxEntries); - putData("PR1", 60, 1); - validateNoOfEvictions("PR1", 20); + private static long getActualEviction(String region) { + final PartitionedRegion pr = + (PartitionedRegion) ClusterStartupRule.getCache().getRegion(region); + return pr.getTotalEvictions(); } - @Test - public void testMemLruForPRAndDR() { - createCache(); - createPartitionedRegion(true, EvictionAlgorithm.LRU_MEMORY, "PR1", 4, 1, 1000, 40); - createDistRegionWithMemEvictionAttr(); - PartitionedRegion pr = (PartitionedRegion) cache.getRegion("PR1"); - DistributedRegion dr = (DistributedRegion) cache.getRegion("DR1"); - - assertEquals(pr.getLocalMaxMemory(), pr.getEvictionAttributes().getMaximum()); - assertEquals(EvictionAttributes.DEFAULT_MEMORY_MAXIMUM, - dr.getEvictionAttributes().getMaximum()); - - for (int i = 0; i < 41; i++) { - pr.put(new Integer(i), new byte[1 * 1024 * 1024]); - } - - assertTrue(1 <= pr.getTotalEvictions()); - assertTrue(pr.getTotalEvictions() <= 2); - - for (int i = 0; i < 11; i++) { - dr.put(new Integer(i), new byte[1 * 1024 * 1024]); + private static int getExpectedEviction() { + GemFireCacheImpl cache = (GemFireCacheImpl) ClusterStartupRule.getCache(); + HeapEvictor evictor = getEvictor(cache); + evictor.setTestAbortAfterLoopCount(1); + + if (offHeap) { + cache.getInternalResourceManager().getOffHeapMonitor() + .updateStateAndSendEvent(188743680); + } else { + HeapMemoryMonitor hmm = cache.getInternalResourceManager().getHeapMonitor(); + hmm.setTestMaxMemoryBytes(100); + hmm.updateStateAndSendEvent(90); } - assertTrue(1 <= dr.getTotalEvictions()); - assertTrue(dr.getTotalEvictions() <= 2); + return (int) Math + .ceil((evictor.getTotalBytesToEvict() / 2) / (double) ((1024 * 1024) + 100)) * 2; } + /** + * + * Test Case verifies:If naturally Eviction up and eviction Down events are raised. Centralized + * and Inline eviction are happening. All this verification is done through logs. It also verifies + * that during eviction, if one node goes down and then comes up again causing GII to take place, + * the system does not throw an OOME. + */ @Test - public void testEachTaskSize() { - createCache(); - createPartitionedRegion(true, EvictionAlgorithm.LRU_HEAP, "PR1", 6, 1, 1000, 40); - createPartitionedRegion(true, EvictionAlgorithm.LRU_HEAP, "PR2", 10, 1, 1000, 40); - createPartitionedRegion(true, EvictionAlgorithm.LRU_HEAP, "PR3", 15, 1, 1000, 40); - createDistRegion(); - - List taskSetSizes = getTestTaskSetSizes(); - if (taskSetSizes != null) { - for (Integer size : taskSetSizes) { - assertEquals(8, size.intValue()); + public void testEvictionWithNodeDown() { + IgnoredException.addIgnoredException("java.io.IOException"); + SerializableRunnableIF setupVM = () -> { + GemFireCacheImpl cache = (GemFireCacheImpl) ClusterStartupRule.getCache(); + final File[] diskDirs = new File[1]; + diskDirs[0] = + new File("Partitioned_Region_Eviction/" + "LogFile" + "_" + OSProcess.getId()); + diskDirs[0].mkdirs(); + + ServerStarterRule server = (ServerStarterRule) ClusterStartupRule.memberStarter; + server.createPartitionRegion("PR1", + f -> f.setOffHeap(offHeap).setEvictionAttributes( + EvictionAttributes.createLRUHeapAttributes(null, EvictionAction.OVERFLOW_TO_DISK)) + .setDiskSynchronous(true) + .setDiskStoreName(cache.createDiskStoreFactory().setDiskDirs(diskDirs) + .create("EvictionTest").getName()), + f -> f.setTotalNumBuckets(2).setRedundantCopies(1)); + + server.createPartitionRegion("PR2", + f -> f.setOffHeap(offHeap).setEvictionAttributes( + EvictionAttributes.createLRUHeapAttributes(null, EvictionAction.OVERFLOW_TO_DISK)) + .setDiskSynchronous(true) + .setDiskStoreName(cache.createDiskStoreFactory().setDiskDirs(diskDirs) + .create("EvictionTest").getName()), + f -> f.setTotalNumBuckets(2).setRedundantCopies(1)); + }; + + VMProvider.invokeInEveryMember(setupVM, server0, server1); + + server0.invoke(() -> { + Region region = ClusterStartupRule.getCache().getRegion("PR1"); + for (int counter = 1; counter <= 100; counter++) { + region.put(counter, new byte[1024 * 1024]); } + }); + + // raise fake event + VMProvider.invokeInEveryMember(() -> { + GemFireCacheImpl cache = (GemFireCacheImpl) ClusterStartupRule.getCache(); + HeapEvictor evictor = getEvictor(cache); + HeapMemoryMonitor hmm = + ((InternalResourceManager) cache.getResourceManager()).getHeapMonitor(); + MemoryEvent event = + new MemoryEvent(offHeap ? ResourceType.OFFHEAP_MEMORY : ResourceType.HEAP_MEMORY, + MemoryThresholds.MemoryState.NORMAL, MemoryThresholds.MemoryState.EVICTION, + cache.getDistributedSystem().getDistributedMember(), 90, true, hmm.getThresholds()); + evictor.onEvent(event); + }, server0, server1); + + // stop server1 + server1.stop(false); + + // restart server1 and create the regions again and verify the data are preserved + server1 = cluster.startServerVM(1, s -> s.withNoCacheServer() + .withConnectionToLocator(ClusterStartupRule.getDUnitLocatorPort())); + server1.invoke(setupVM); + server1.invoke(() -> { + Cache cache = ClusterStartupRule.getCache(); + assertThat(cache.getRegion("PR1").size()).isEqualTo(100); + assertThat(cache.getRegion("PR2").size()).isEqualTo(0); + }); + + + } + + private static HeapEvictor getEvictor(GemFireCacheImpl cache) { + if (offHeap) { + return cache.getOffHeapEvictor(); + } else { + return cache.getHeapEvictor(); } } } diff --git a/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/eviction/EvictionTestBase.java b/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/eviction/EvictionTestBase.java deleted file mode 100755 index 73235c4..0000000 --- a/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/eviction/EvictionTestBase.java +++ /dev/null @@ -1,534 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.geode.internal.cache.eviction; - -import static java.lang.Math.abs; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.io.File; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ThreadPoolExecutor; - -import org.apache.geode.cache.AttributesFactory; -import org.apache.geode.cache.Cache; -import org.apache.geode.cache.CacheException; -import org.apache.geode.cache.CacheFactory; -import org.apache.geode.cache.DataPolicy; -import org.apache.geode.cache.EvictionAction; -import org.apache.geode.cache.EvictionAlgorithm; -import org.apache.geode.cache.EvictionAttributes; -import org.apache.geode.cache.PartitionAttributesFactory; -import org.apache.geode.cache.Region; -import org.apache.geode.cache.util.ObjectSizer; -import org.apache.geode.cache30.CacheSerializableRunnable; -import org.apache.geode.distributed.DistributedSystem; -import org.apache.geode.distributed.internal.DistributionConfig; -import org.apache.geode.internal.OSProcess; -import org.apache.geode.internal.cache.BucketRegion; -import org.apache.geode.internal.cache.DistributedRegion; -import org.apache.geode.internal.cache.GemFireCacheImpl; -import org.apache.geode.internal.cache.LocalRegion; -import org.apache.geode.internal.cache.PartitionedRegion; -import org.apache.geode.internal.cache.control.HeapMemoryMonitor; -import org.apache.geode.internal.cache.control.InternalResourceManager; -import org.apache.geode.internal.cache.control.InternalResourceManager.ResourceType; -import org.apache.geode.internal.cache.control.MemoryEvent; -import org.apache.geode.internal.cache.control.MemoryThresholds.MemoryState; -import org.apache.geode.test.awaitility.GeodeAwaitility; -import org.apache.geode.test.dunit.Assert; -import org.apache.geode.test.dunit.Host; -import org.apache.geode.test.dunit.LogWriterUtils; -import org.apache.geode.test.dunit.SerializableCallable; -import org.apache.geode.test.dunit.SerializableRunnable; -import org.apache.geode.test.dunit.VM; -import org.apache.geode.test.dunit.WaitCriterion; -import org.apache.geode.test.dunit.cache.internal.JUnit4CacheTestCase; - - -public class EvictionTestBase extends JUnit4CacheTestCase { - - protected static Cache cache = null; - - protected static VM dataStore1 = null; - - protected static VM dataStore2 = null; - - protected static VM dataStore3 = null; - - protected static VM dataStore4 = null; - - protected static Region region = null; - - static int maxEntries = 20; - - static int maxSizeInMb = 20; - - static int totalNoOfBuckets = 4; - - @Override - public final void postSetUp() throws Exception { - Host host = Host.getHost(0); - dataStore1 = host.getVM(0); - dataStore2 = host.getVM(1); - dataStore3 = host.getVM(2); - dataStore4 = host.getVM(3); - } - - public void prepareScenario1(EvictionAlgorithm evictionAlgorithm, int maxEntries) { - createCacheInAllVms(); - createPartitionedRegionInAllVMS(true, evictionAlgorithm, "PR1", totalNoOfBuckets, 1, 10000, - maxEntries); - createPartitionedRegionInAllVMS(true, evictionAlgorithm, "PR2", totalNoOfBuckets, 2, 10000, - maxEntries); - } - - public void raiseFakeNotification(VM vm, final String prName, final int noOfExpectedEvictions) { - vm.invoke(new CacheSerializableRunnable("fakeNotification") { - @Override - public void run2() throws CacheException { - final LocalRegion region = (LocalRegion) cache.getRegion(prName); - getEvictor().setTestAbortAfterLoopCount(1); - - InternalResourceManager irm = ((GemFireCacheImpl) cache).getInternalResourceManager(); - HeapMemoryMonitor hmm = irm.getHeapMonitor(); - hmm.setTestMaxMemoryBytes(100); - hmm.updateStateAndSendEvent(90); - - WaitCriterion wc = new WaitCriterion() { - public boolean done() { - final long currentEvictions = region.getTotalEvictions(); - if (abs(currentEvictions - noOfExpectedEvictions) <= 1) { // Margin of error is 1 - return true; - } else if (currentEvictions > noOfExpectedEvictions) { - fail(description()); - } - return false; - } - - public String description() { - return "expected " + noOfExpectedEvictions + " evictions, but got " - + region.getTotalEvictions(); - } - }; - GeodeAwaitility.await().untilAsserted(wc); - } - }); - } - - public void prepareScenario2(final EvictionAlgorithm evictionAlgorithm, - final String partitionRegion1, final String partitionRegion2) { - dataStore3.invoke(new SerializableRunnable() { - @Override - public void run() { - createCache(); - } - }); - - dataStore4.invoke(new SerializableRunnable() { - @Override - public void run() { - createCache(); - } - }); - - dataStore3.invoke(() -> EvictionTestBase.setTolerance()); - dataStore4.invoke(() -> EvictionTestBase.setTolerance()); - - dataStore3.invoke(new SerializableRunnable() { - @Override - public void run() { - createPartitionedRegion(true, evictionAlgorithm, partitionRegion1, 2, 2, 10000, 0); - } - }); - - dataStore4.invoke(new SerializableRunnable() { - @Override - public void run() { - createPartitionedRegion(true, evictionAlgorithm, partitionRegion1, 2, 2, 10000, 0); - } - }); - - dataStore3.invoke(new SerializableRunnable() { - @Override - public void run() { - createPartitionedRegion(true, evictionAlgorithm, partitionRegion2, 2, 2, 10000, 0); - } - }); - - dataStore4.invoke(new SerializableRunnable() { - @Override - public void run() { - createPartitionedRegion(true, evictionAlgorithm, partitionRegion2, 2, 2, 10000, 0); - } - }); - } - - public void fakeNotification() { - dataStore3.invoke(new SerializableRunnable() { - @Override - public void run() { - sendFakeNotification(); - } - }); - - dataStore4.invoke(new SerializableRunnable() { - @Override - public void run() { - sendFakeNotification(); - } - }); - } - - public void sendFakeNotification() { - HeapMemoryMonitor hmm = ((InternalResourceManager) cache.getResourceManager()).getHeapMonitor(); - MemoryEvent event = new MemoryEvent(getResourceType(), MemoryState.NORMAL, MemoryState.EVICTION, - cache.getDistributedSystem().getDistributedMember(), 90, true, hmm.getThresholds()); - getEvictor().onEvent(event); - } - - public static void setTolerance() { - System.setProperty(DistributionConfig.GEMFIRE_PREFIX + "memoryEventTolerance", - Integer.toString(0)); - } - - public void createDistributedRegion() { - dataStore1.invoke(new SerializableRunnable() { - @Override - public void run() { - createDistRegion(); - } - }); - } - - public void createDistRegion() { - final AttributesFactory factory = new AttributesFactory(); - factory.setOffHeap(getOffHeapEnabled()); - factory.setDataPolicy(DataPolicy.NORMAL); - factory.setEvictionAttributes( - EvictionAttributes.createLRUHeapAttributes(null, EvictionAction.LOCAL_DESTROY)); - DistributedRegion distRegion = (DistributedRegion) cache.createRegion("DR1", factory.create()); - assertNotNull(distRegion); - - } - - public static void createDistRegionWithMemEvictionAttr() { - final AttributesFactory factory = new AttributesFactory(); - factory.setDataPolicy(DataPolicy.NORMAL); - factory.setEvictionAttributes(EvictionAttributes.createLRUMemoryAttributes(ObjectSizer.DEFAULT, - EvictionAction.LOCAL_DESTROY)); - DistributedRegion distRegion = (DistributedRegion) cache.createRegion("DR1", factory.create()); - assertNotNull(distRegion); - - } - - - public void createCacheInAllVms() { - dataStore1.invoke(new SerializableRunnable() { - @Override - public void run() { - createCache(); - } - }); - - dataStore2.invoke(new SerializableRunnable() { - @Override - public void run() { - createCache(); - } - }); - } - - public static void createCacheInVm() { - new EvictionTestBase().createCache(); - } - - public void createCache() { - try { - HeapMemoryMonitor.setTestDisableMemoryUpdates(true); - System.setProperty(DistributionConfig.GEMFIRE_PREFIX + "memoryEventTolerance", "0"); - - Properties props = new Properties(); - DistributedSystem ds = getSystem(props); - assertNotNull(ds); - ds.disconnect(); - ds = getSystem(props); - cache = CacheFactory.create(ds); - assertNotNull(cache); - LogWriterUtils.getLogWriter().info("cache= " + cache); - LogWriterUtils.getLogWriter().info("cache closed= " + cache.isClosed()); - cache.getResourceManager().setEvictionHeapPercentage(85); - LogWriterUtils.getLogWriter() - .info("eviction= " + cache.getResourceManager().getEvictionHeapPercentage()); - LogWriterUtils.getLogWriter() - .info("critical= " + cache.getResourceManager().getCriticalHeapPercentage()); - } catch (Exception e) { - Assert.fail("Failed while creating the cache", e); - } - } - - public List getTestTaskSetSizes() { - return getEvictor().testOnlyGetSizeOfTasks(); - } - - protected void createPartitionedRegionInAllVMS(final boolean setEvictionOn, - final EvictionAlgorithm evictionAlgorithm, final String regionName, - final int totalNoOfBuckets, final int evictionAction, final int evictorInterval, - final int maxEntries) { - - dataStore1.invoke(new SerializableRunnable() { - @Override - public void run() { - createPartitionedRegion(setEvictionOn, evictionAlgorithm, regionName, totalNoOfBuckets, - evictionAction, evictorInterval, maxEntries); - } - }); - - dataStore2.invoke(new SerializableRunnable() { - @Override - public void run() { - createPartitionedRegion(setEvictionOn, evictionAlgorithm, regionName, totalNoOfBuckets, - evictionAction, evictorInterval, maxEntries); - } - }); - } - - public void createPartitionedRegion(boolean setEvictionOn, EvictionAlgorithm evictionAlgorithm, - String regionName, int totalNoOfBuckets, int evictionAction, int evictorInterval, - int maxEnteries) { - - final AttributesFactory factory = new AttributesFactory(); - - factory.setOffHeap(getOffHeapEnabled()); - - PartitionAttributesFactory partitionAttributesFactory = new PartitionAttributesFactory() - .setRedundantCopies(totalNoOfBuckets == 4 ? 0 : 1).setTotalNumBuckets(totalNoOfBuckets); - if (evictionAlgorithm.isLRUMemory()) - partitionAttributesFactory.setLocalMaxMemory(maxEnteries); - - factory.setPartitionAttributes(partitionAttributesFactory.create()); - if (setEvictionOn) { - if (evictionAlgorithm.isLRUHeap()) { - factory.setEvictionAttributes(EvictionAttributes.createLRUHeapAttributes(null, - evictionAction == 1 ? EvictionAction.LOCAL_DESTROY : EvictionAction.OVERFLOW_TO_DISK)); - } else if (evictionAlgorithm.isLRUMemory()) { - - factory.setEvictionAttributes(EvictionAttributes.createLRUMemoryAttributes( - ObjectSizer.DEFAULT, - evictionAction == 1 ? EvictionAction.LOCAL_DESTROY : EvictionAction.OVERFLOW_TO_DISK)); - } else { - factory.setEvictionAttributes(EvictionAttributes.createLRUEntryAttributes(maxEnteries, - evictionAction == 1 ? EvictionAction.LOCAL_DESTROY : EvictionAction.OVERFLOW_TO_DISK)); - } - if (evictionAction == 2) { - final File[] diskDirs = new File[1]; - diskDirs[0] = - new File("Partitioned_Region_Eviction/" + "LogFile" + "_" + OSProcess.getId()); - diskDirs[0].mkdirs(); - factory.setDiskSynchronous(true); - factory.setDiskStoreName(cache.createDiskStoreFactory().setDiskDirs(diskDirs) - .create("EvictionTestBase").getName()); - } - } - - region = cache.createRegion(regionName, factory.create()); - assertNotNull(region); - LogWriterUtils.getLogWriter().info("Partitioned Region created Successfully :" + region); - } - - public static void putData(final String regionName, final int noOfElememts, - final int sizeOfElement) { - dataStore1.invoke(new CacheSerializableRunnable("putData") { - @Override - public void run2() throws CacheException { - final Region pr = cache.getRegion(regionName); - for (int counter = 1; counter <= noOfElememts; counter++) { - pr.put(new Integer(counter), new byte[sizeOfElement * 1024 * 1024]); - } - } - }); - } - - public static void putDataInDistributedRegion(final int noOfElememts, final int sizeOfElement) { - dataStore1.invoke(new CacheSerializableRunnable("putData") { - @Override - public void run2() throws CacheException { - final Region pr = cache.getRegion("DR1"); - for (int counter = 1; counter <= noOfElememts; counter++) { - pr.put(new Integer(counter), new byte[sizeOfElement * 1024 * 1024]); - LogWriterUtils.getLogWriter().info("Amar put data element no->" + counter); - } - } - }); - } - - public void validateNoOfEvictions(final String regionName, final int noOfEvictions) { - final SerializableCallable validate = new SerializableCallable("Validate evictions") { - public Object call() throws Exception { - final PartitionedRegion pr = (PartitionedRegion) cache.getRegion(regionName); - - for (final Iterator i = pr.getDataStore().getAllLocalBuckets().iterator(); i.hasNext();) { - final Map.Entry entry = (Map.Entry) i.next(); - final BucketRegion bucketRegion = (BucketRegion) entry.getValue(); - if (bucketRegion == null) { - continue; - } - LogWriterUtils.getLogWriter() - .info("FINAL bucket= " + bucketRegion.getFullPath() + "size= " + bucketRegion.size()); - } - - return pr.getTotalEvictions(); - } - }; - long evictionsInVM1 = (Long) dataStore1.invoke(validate); - long evictionsInVM2 = (Long) dataStore2.invoke(validate); - LogWriterUtils.getLogWriter() - .info("EEE evicitons = " + noOfEvictions + " " + (evictionsInVM1 + evictionsInVM2)); - assertEquals(noOfEvictions, (evictionsInVM1 + evictionsInVM2)); - } - - public void verifyThreadPoolTaskCount(final int taskCountToBeVerified) { - final SerializableCallable getThreadPoolTaskCount = - new SerializableCallable("Validate evictions") { - public Object call() throws Exception { - return getEvictor().getEvictorThreadPool() != null - ? ((ThreadPoolExecutor) getEvictor().getEvictorThreadPool()).getTaskCount() : 0; - } - }; - Long taskCountOfVM = (Long) dataStore1.invoke(getThreadPoolTaskCount); - assertTrue(taskCountOfVM > 0 && taskCountOfVM <= taskCountToBeVerified); - } - - public static void putDataInDataStore3(final String regionName, final int noOfElememts, - final int sizeOfElement) { - dataStore3.invoke(new CacheSerializableRunnable("putData") { - @Override - public void run2() throws CacheException { - final Region pr = cache.getRegion(regionName); - for (int counter = 1; counter <= noOfElememts; counter++) { - pr.put(new Integer(counter), new byte[sizeOfElement * 1024 * 1024]); - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new CacheSerializableRunnableException(e.getLocalizedMessage(), e); - } - } - } - }); - } - - public static void print(final String regionName) { - dataStore3.invoke(new CacheSerializableRunnable("putData") { - @Override - public void run2() throws CacheException { - final PartitionedRegion pr = (PartitionedRegion) cache.getRegion(regionName); - Set bucket = pr.getDataStore().getAllLocalBucketRegions(); - Iterator itr = bucket.iterator(); - while (itr.hasNext()) { - BucketRegion br = (BucketRegion) itr.next(); - LogWriterUtils.getLogWriter().info("Print " + br.size()); - } - } - }); - } - - public void validateNoOfEvictionsInDataStore3N4(final String regionName, - final int noOfEvictions) { - final SerializableCallable validate = new SerializableCallable("Validate evictions") { - public Object call() throws Exception { - final PartitionedRegion pr = (PartitionedRegion) cache.getRegion(regionName); - return pr.getTotalEvictions(); - } - }; - long evictionsInVM1 = (Long) dataStore3.invoke(validate); - long evictionsInVM2 = (Long) dataStore4.invoke(validate); - assertEquals(noOfEvictions, evictionsInVM1 + evictionsInVM2); - } - - public void killVm() { - dataStore4.invoke(() -> EvictionTestBase.close()); - - } - - public static void close() { - if (cache != null && !cache.isClosed()) { - cache.close(); - cache.getDistributedSystem().disconnect(); - } - } - - public void bringVMBackToLife() { - dataStore4.invoke(new SerializableRunnable() { - @Override - public void run() { - createCache(); - } - }); - - dataStore4.invoke(new SerializableRunnable() { - @Override - public void run() { - createPartitionedRegion(true, EvictionAlgorithm.LRU_HEAP, "PR3", 2, 2, 10000, 0); - } - }); - - dataStore4.invoke(new SerializableRunnable() { - @Override - public void run() { - createPartitionedRegion(true, EvictionAlgorithm.LRU_HEAP, "PR4", 2, 2, 10000, 0); - } - }); - } - - public int getPRSize(String string) { - Integer prSize = (Integer) dataStore4.invoke(() -> EvictionTestBase.returnPRSize(string)); - return prSize; - - } - - public static int returnPRSize(String string) { - final PartitionedRegion pr = (PartitionedRegion) cache.getRegion(string); - return pr.size(); - } - - public boolean getOffHeapEnabled() { - return false; - } - - public HeapEvictor getEvictor() { - return ((GemFireCacheImpl) cache).getHeapEvictor(); - } - - @SuppressWarnings("serial") - public int getExpectedEvictionRatioOnVm(final VM vm) { - return (Integer) vm.invoke(new SerializableCallable() { - @Override - public Object call() throws Exception { - return (int) Math - .ceil((getEvictor().getTotalBytesToEvict() / 2) / (double) ((1024 * 1024) + 100)) * 2; - } - }); - } - - public ResourceType getResourceType() { - return ResourceType.HEAP_MEMORY; - } -} diff --git a/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/eviction/OffHeapEvictionDUnitTest.java b/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/eviction/OffHeapEvictionDUnitTest.java deleted file mode 100644 index e849a60..0000000 --- a/geode-core/src/distributedTest/java/org/apache/geode/internal/cache/eviction/OffHeapEvictionDUnitTest.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License - * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express - * or implied. See the License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.geode.internal.cache.eviction; - -import static java.lang.Math.abs; -import static org.apache.geode.distributed.ConfigurationProperties.OFF_HEAP_MEMORY_SIZE; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; - -import java.util.Properties; - -import org.junit.experimental.categories.Category; - -import org.apache.geode.cache.CacheException; -import org.apache.geode.cache.CacheFactory; -import org.apache.geode.cache30.CacheSerializableRunnable; -import org.apache.geode.distributed.DistributedSystem; -import org.apache.geode.internal.cache.GemFireCacheImpl; -import org.apache.geode.internal.cache.LocalRegion; -import org.apache.geode.internal.cache.OffHeapTestUtil; -import org.apache.geode.internal.cache.control.InternalResourceManager.ResourceType; -import org.apache.geode.test.awaitility.GeodeAwaitility; -import org.apache.geode.test.dunit.Assert; -import org.apache.geode.test.dunit.Invoke; -import org.apache.geode.test.dunit.LogWriterUtils; -import org.apache.geode.test.dunit.SerializableRunnable; -import org.apache.geode.test.dunit.VM; -import org.apache.geode.test.dunit.WaitCriterion; -import org.apache.geode.test.junit.categories.EvictionTest; - -/** - * Performs eviction dunit tests for off-heap memory. - */ -@Category({EvictionTest.class}) -public class OffHeapEvictionDUnitTest extends EvictionDUnitTest { - - @Override - public final void preTearDownAssertions() throws Exception { - SerializableRunnable checkOrphans = new SerializableRunnable() { - - @Override - public void run() { - if (hasCache()) { - OffHeapTestUtil.checkOrphans(getCache()); - } - } - }; - Invoke.invokeInEveryVM(checkOrphans); - checkOrphans.run(); - } - - @Override - public Properties getDistributedSystemProperties() { - Properties properties = super.getDistributedSystemProperties(); - properties.setProperty(OFF_HEAP_MEMORY_SIZE, "200m"); - - return properties; - } - - @Override - public void createCache() { - try { - Properties props = new Properties(); - DistributedSystem ds = getSystem(props); - assertNotNull(ds); - ds.disconnect(); - ds = getSystem(getDistributedSystemProperties()); - cache = CacheFactory.create(ds); - assertNotNull(cache); - LogWriterUtils.getLogWriter().info("cache= " + cache); - LogWriterUtils.getLogWriter().info("cache closed= " + cache.isClosed()); - cache.getResourceManager().setEvictionOffHeapPercentage(85); - ((GemFireCacheImpl) cache).getInternalResourceManager().getOffHeapMonitor() - .stopMonitoring(true); - LogWriterUtils.getLogWriter() - .info("eviction= " + cache.getResourceManager().getEvictionOffHeapPercentage()); - LogWriterUtils.getLogWriter() - .info("critical= " + cache.getResourceManager().getCriticalOffHeapPercentage()); - } catch (Exception e) { - Assert.fail("Failed while creating the cache", e); - } - } - - @Override - public void raiseFakeNotification(VM vm, final String prName, final int noOfExpectedEvictions) { - vm.invoke(new CacheSerializableRunnable("fakeNotification") { - @Override - public void run2() throws CacheException { - final LocalRegion region = (LocalRegion) cache.getRegion(prName); - getEvictor().setTestAbortAfterLoopCount(1); - - ((GemFireCacheImpl) cache).getInternalResourceManager().getOffHeapMonitor() - .updateStateAndSendEvent(188743680); - - WaitCriterion wc = new WaitCriterion() { - public boolean done() { - // we have a primary - final long currentEvictions = region.getTotalEvictions(); - if (abs(currentEvictions - noOfExpectedEvictions) <= 1) { // Margin of error is 1 - return true; - } else if (currentEvictions > noOfExpectedEvictions) { - fail(description()); - } - return false; - } - - public String description() { - return "expected " + noOfExpectedEvictions + " evictions, but got " - + region.getTotalEvictions(); - } - }; - GeodeAwaitility.await().untilAsserted(wc); - } - }); - } - - @Override - public boolean getOffHeapEnabled() { - return true; - } - - @Override - public HeapEvictor getEvictor() { - return ((GemFireCacheImpl) cache).getOffHeapEvictor(); - } - - @Override - public ResourceType getResourceType() { - return ResourceType.OFFHEAP_MEMORY; - } - - @Override - public void testDummyInlineNCentralizedEviction() { - super.testDummyInlineNCentralizedEviction(); - } -} diff --git a/geode-core/src/integrationTest/java/org/apache/geode/internal/cache/eviction/EvictionIntegrationTest.java b/geode-core/src/integrationTest/java/org/apache/geode/internal/cache/eviction/EvictionIntegrationTest.java new file mode 100644 index 0000000..7b1d910 --- /dev/null +++ b/geode-core/src/integrationTest/java/org/apache/geode/internal/cache/eviction/EvictionIntegrationTest.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License + * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express + * or implied. See the License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.geode.internal.cache.eviction; + +import static org.apache.geode.distributed.ConfigurationProperties.OFF_HEAP_MEMORY_SIZE; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.geode.cache.DataPolicy; +import org.apache.geode.cache.EvictionAction; +import org.apache.geode.cache.EvictionAttributes; +import org.apache.geode.cache.RegionShortcut; +import org.apache.geode.cache.util.ObjectSizer; +import org.apache.geode.internal.cache.BucketRegion; +import org.apache.geode.internal.cache.GemFireCacheImpl; +import org.apache.geode.internal.cache.LocalRegion; +import org.apache.geode.internal.cache.PartitionedRegion; +import org.apache.geode.test.junit.categories.EvictionTest; +import org.apache.geode.test.junit.rules.ServerStarterRule; +import org.apache.geode.test.junit.runners.CategoryWithParameterizedRunnerFactory; + + +@RunWith(Parameterized.class) +@Parameterized.UseParametersRunnerFactory(CategoryWithParameterizedRunnerFactory.class) +@Category({EvictionTest.class}) +public class EvictionIntegrationTest { + + @Parameterized.Parameters(name = "offHeap={0}") + public static Collection booleans() { + return Arrays.asList(true, false); + } + + @Parameterized.Parameter + public boolean offHeap; + + @Rule + public ServerStarterRule server = new ServerStarterRule().withNoCacheServer(); + + @Before + public void setUp() { + if (offHeap) { + server.withProperty(OFF_HEAP_MEMORY_SIZE, "200m"); + } + server.startServer(); + } + + @Test + public void testEntryLruEvictions() { + int maxEntry = 3; + PartitionedRegion pr1 = (PartitionedRegion) server.createPartitionRegion("PR1", + f -> f.setOffHeap(offHeap) + .setEvictionAttributes( + EvictionAttributes.createLRUEntryAttributes(maxEntry, + EvictionAction.LOCAL_DESTROY)), + f -> f.setRedundantCopies(0).setTotalNumBuckets(4)); + + // put in one extra entry after maxEntry + for (int counter = 1; counter <= maxEntry + 1; counter++) { + pr1.put(counter, new byte[1024 * 1024]); + } + + assertThat(pr1.getTotalEvictions()).isEqualTo(1); + } + + @Test + public void testEntryLru() { + int maxEntry = 12; + PartitionedRegion pr1 = (PartitionedRegion) server.createPartitionRegion("PR1", + f -> f.setOffHeap(offHeap) + .setEvictionAttributes( + EvictionAttributes.createLRUEntryAttributes(maxEntry, + EvictionAction.LOCAL_DESTROY)), + f -> f.setRedundantCopies(0).setTotalNumBuckets(4)); + + for (int i = 0; i < 3; i++) { + // assume mod-based hashing for bucket creation + pr1.put(i, "value0"); + pr1.put(i + pr1.getPartitionAttributes().getTotalNumBuckets(), "value1"); + pr1.put(i + (pr1.getPartitionAttributes().getTotalNumBuckets() * 2), "value2"); + } + pr1.put(3, "value0"); + + for (int i = 0; i < 2; i++) { + pr1.put(i + pr1.getPartitionAttributes().getTotalNumBuckets() * 3, "value1"); + } + assertThat(pr1.getTotalEvictions()).isEqualTo(0); + } + + @Test + public void testCheckEntryLruEvictionsIn1DataStore() { + int extraEntries = 10; + int maxEntries = 20; + PartitionedRegion pr1 = (PartitionedRegion) server.createPartitionRegion("PR1", + f -> f.setOffHeap(offHeap) + .setEvictionAttributes( + EvictionAttributes.createLRUEntryAttributes(maxEntries, + EvictionAction.LOCAL_DESTROY)), + f -> f.setRedundantCopies(1).setTotalNumBuckets(5)); + + for (int counter = 1; counter <= maxEntries + extraEntries; counter++) { + pr1.put(counter, new byte[1024 * 1024]); + } + + assertThat(pr1.getTotalEvictions()).isEqualTo(extraEntries); + + for (Map.Entry integerBucketRegionEntry : pr1.getDataStore() + .getAllLocalBuckets()) { + final BucketRegion bucketRegion = + (BucketRegion) ((Map.Entry) integerBucketRegionEntry).getValue(); + if (bucketRegion == null) { + continue; + } + assertThat(bucketRegion.size()).isEqualTo(4); + } + } + + @Test + public void testMemLruForPRAndDR() { + int maxEntries = 40; + PartitionedRegion pr1 = (PartitionedRegion) server.createPartitionRegion("PR1", + f -> f.setOffHeap(offHeap) + .setEvictionAttributes(EvictionAttributes.createLRUMemoryAttributes( + ObjectSizer.DEFAULT, + EvictionAction.LOCAL_DESTROY)), + f -> f.setRedundantCopies(0).setTotalNumBuckets(4).setLocalMaxMemory(maxEntries)); + + LocalRegion dr1 = + (LocalRegion) server.createRegion(RegionShortcut.LOCAL, "DR1", + f -> f.setOffHeap(offHeap).setDataPolicy(DataPolicy.NORMAL).setEvictionAttributes( + EvictionAttributes.createLRUMemoryAttributes(ObjectSizer.DEFAULT, + EvictionAction.LOCAL_DESTROY))); + + assertThat(pr1.getLocalMaxMemory()).isEqualTo(pr1.getEvictionAttributes().getMaximum()); + assertThat(dr1.getEvictionAttributes().getMaximum()) + .isEqualTo(EvictionAttributes.DEFAULT_MEMORY_MAXIMUM); + + for (int i = 0; i < 41; i++) { + pr1.put(i, new byte[1024 * 1024]); + } + + assertThat(pr1.getTotalEvictions()).isGreaterThanOrEqualTo(1).isLessThanOrEqualTo(2); + for (int i = 0; i < 11; i++) { + dr1.put(i, new byte[1024 * 1024]); + } + + assertThat(dr1.getTotalEvictions()).isGreaterThanOrEqualTo(1).isLessThanOrEqualTo(2); + } + + @Test + public void testEachTaskSize() { + server.createPartitionRegion("PR1", + f -> f.setOffHeap(offHeap) + .setEvictionAttributes(EvictionAttributes.createLRUHeapAttributes( + null, EvictionAction.LOCAL_DESTROY)), + f -> f.setRedundantCopies(1).setTotalNumBuckets(6)); + + server.createPartitionRegion("PR2", + f -> f.setOffHeap(offHeap) + .setEvictionAttributes(EvictionAttributes.createLRUHeapAttributes( + null, EvictionAction.LOCAL_DESTROY)), + f -> f.setRedundantCopies(1).setTotalNumBuckets(10)); + + server.createPartitionRegion("PR3", + f -> f.setOffHeap(offHeap) + .setEvictionAttributes(EvictionAttributes.createLRUHeapAttributes( + null, EvictionAction.LOCAL_DESTROY)), + f -> f.setRedundantCopies(1).setTotalNumBuckets(15)); + + server.createRegion(RegionShortcut.LOCAL, "DR1", + f -> f.setOffHeap(offHeap).setDataPolicy(DataPolicy.NORMAL).setEvictionAttributes( + EvictionAttributes.createLRUHeapAttributes(null, EvictionAction.LOCAL_DESTROY))); + + HeapEvictor evictor = ((GemFireCacheImpl) server.getCache()).getHeapEvictor(); + List taskSetSizes = evictor.testOnlyGetSizeOfTasks(); + + for (Integer size : taskSetSizes) { + assertThat(size.intValue()).isEqualTo(8); + } + } + +} diff --git a/geode-dunit/src/main/java/org/apache/geode/test/junit/rules/VMProvider.java b/geode-dunit/src/main/java/org/apache/geode/test/junit/rules/VMProvider.java index 154cb6c..4176a44 100644 --- a/geode-dunit/src/main/java/org/apache/geode/test/junit/rules/VMProvider.java +++ b/geode-dunit/src/main/java/org/apache/geode/test/junit/rules/VMProvider.java @@ -57,15 +57,13 @@ public abstract class VMProvider { } public boolean isLocator() { - return getVM().invoke(() -> { - return ClusterStartupRule.getLocator() != null; - }); + return getVM().invoke(() -> ClusterStartupRule.getLocator() != null); } + // a server can be started without a cache server, so as long as this member has no locator, + // it's deemed as a server public boolean isServer() { - return getVM().invoke(() -> { - return ClusterStartupRule.getServer() != null; - }); + return getVM().invoke(() -> ClusterStartupRule.getLocator() == null); } public void invoke(final SerializableRunnableIF runnable) {