geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dschnei...@apache.org
Subject [1/7] geode git commit: initial cut of fix. not sure what the state of this revision is
Date Tue, 11 Apr 2017 23:58:00 GMT
Repository: geode
Updated Branches:
  refs/heads/feature/GEODE-2097 [created] 9ade788cf


initial cut of fix. not sure what the state of this revision is


Project: http://git-wip-us.apache.org/repos/asf/geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/87452d32
Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/87452d32
Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/87452d32

Branch: refs/heads/feature/GEODE-2097
Commit: 87452d327dc8fd60b1bea0668c5dd3292d7775b4
Parents: 5731300
Author: Darrel Schneider <dschneider@pivotal.io>
Authored: Wed Dec 21 11:40:30 2016 -0800
Committer: Darrel Schneider <dschneider@pivotal.io>
Committed: Wed Dec 21 11:40:30 2016 -0800

----------------------------------------------------------------------
 .../internal/cache/AbstractLRURegionMap.java    |   5 +-
 .../geode/internal/cache/AbstractRegionMap.java |   4 +-
 .../geode/internal/cache/LocalRegion.java       |   2 +-
 .../internal/cache/PlaceHolderDiskRegion.java   |   2 +-
 .../geode/internal/cache/ProxyRegionMap.java    |   4 +-
 .../geode/internal/cache/lru/EnableLRU.java     |   3 +
 .../cache/lru/HeapLRUCapacityController.java    |  25 +++--
 .../cache/lru/LRUCapacityController.java        |   6 +
 .../internal/cache/lru/LRUMapCallbacks.java     |   4 +-
 .../cache/lru/MemLRUCapacityController.java     |   6 +
 .../internal/cache/lru/LRUClockJUnitTest.java   |   6 +
 .../OffHeapLRURecoveryRegressionTest.java       | 111 +++++++++++++++++++
 12 files changed, 159 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/geode/blob/87452d32/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractLRURegionMap.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractLRURegionMap.java
b/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractLRURegionMap.java
index 328ff35..05cf44d 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractLRURegionMap.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractLRURegionMap.java
@@ -36,6 +36,7 @@ import org.apache.geode.internal.cache.lru.LRUStatistics;
 import org.apache.geode.internal.cache.lru.MemLRUCapacityController;
 import org.apache.geode.internal.cache.lru.NewLIFOClockHand;
 import org.apache.geode.internal.cache.lru.NewLRUClockHand;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
 import org.apache.geode.internal.cache.versions.RegionVersionVector;
 import org.apache.geode.internal.cache.versions.VersionSource;
 import org.apache.geode.internal.i18n.LocalizedStrings;
@@ -856,8 +857,8 @@ public abstract class AbstractLRURegionMap extends AbstractRegionMap {
   }
 
   @Override
-  public final boolean lruLimitExceeded() {
-    return _getCCHelper().mustEvict(_getLruList().stats(), null, 0);
+  public final boolean lruLimitExceeded(DiskRegionView drv) {
+    return _getCCHelper().lruLimitExceeded(_getLruList().stats(), drv);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/geode/blob/87452d32/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractRegionMap.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractRegionMap.java
b/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractRegionMap.java
index 96936eef..fd0c4fe 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractRegionMap.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractRegionMap.java
@@ -31,6 +31,7 @@ import org.apache.geode.internal.cache.FilterRoutingInfo.FilterInfo;
 import org.apache.geode.internal.cache.ha.HAContainerWrapper;
 import org.apache.geode.internal.cache.ha.HARegionQueue;
 import org.apache.geode.internal.cache.lru.LRUEntry;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
 import org.apache.geode.internal.cache.region.entry.RegionEntryFactoryBuilder;
 import org.apache.geode.internal.cache.tier.sockets.CacheClientNotifier;
 import org.apache.geode.internal.cache.tier.sockets.ClientProxyMembershipID;
@@ -478,7 +479,8 @@ public abstract class AbstractRegionMap implements RegionMap {
     e.decRefCount(null, lr);
   }
 
-  public boolean lruLimitExceeded() {
+  @Override
+  public boolean lruLimitExceeded(DiskRegionView drv) {
     return false;
   }
 

http://git-wip-us.apache.org/repos/asf/geode/blob/87452d32/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java b/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
index fd4b6c7..55bf62e 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
@@ -11095,7 +11095,7 @@ public class LocalRegion extends AbstractRegion implements LoaderHelperFactory,
   }
 
   public boolean lruLimitExceeded() {
-    return this.entries.lruLimitExceeded();
+    return this.entries.lruLimitExceeded(getDiskRegionView());
   }
 
   public DiskEntry getDiskEntry(Object key) {

http://git-wip-us.apache.org/repos/asf/geode/blob/87452d32/geode-core/src/main/java/org/apache/geode/internal/cache/PlaceHolderDiskRegion.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/PlaceHolderDiskRegion.java
b/geode-core/src/main/java/org/apache/geode/internal/cache/PlaceHolderDiskRegion.java
index db01162..9946f26 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/PlaceHolderDiskRegion.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/PlaceHolderDiskRegion.java
@@ -127,7 +127,7 @@ public class PlaceHolderDiskRegion extends AbstractDiskRegion implements
DiskRec
   }
 
   public boolean lruLimitExceeded() {
-    return getRecoveredEntryMap().lruLimitExceeded();
+    return getRecoveredEntryMap().lruLimitExceeded(this);
   }
 
   public DiskStoreID getDiskStoreID() {

http://git-wip-us.apache.org/repos/asf/geode/blob/87452d32/geode-core/src/main/java/org/apache/geode/internal/cache/ProxyRegionMap.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/ProxyRegionMap.java
b/geode-core/src/main/java/org/apache/geode/internal/cache/ProxyRegionMap.java
index 92c7b6f..8ed07f8 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/ProxyRegionMap.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/ProxyRegionMap.java
@@ -35,6 +35,7 @@ import org.apache.geode.internal.InternalStatisticsDisabledException;
 import org.apache.geode.internal.cache.AbstractRegionMap.ARMLockTestHook;
 import org.apache.geode.internal.cache.lru.LRUEntry;
 import org.apache.geode.internal.cache.lru.NewLRUClockHand;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
 import org.apache.geode.internal.cache.tier.sockets.ClientProxyMembershipID;
 import org.apache.geode.internal.cache.versions.RegionVersionVector;
 import org.apache.geode.internal.cache.versions.VersionHolder;
@@ -366,7 +367,8 @@ final class ProxyRegionMap implements RegionMap {
     // nothing needed
   }
 
-  public final boolean lruLimitExceeded() {
+  @Override
+  public final boolean lruLimitExceeded(DiskRegionView drv) {
     return false;
   }
 

http://git-wip-us.apache.org/repos/asf/geode/blob/87452d32/geode-core/src/main/java/org/apache/geode/internal/cache/lru/EnableLRU.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/EnableLRU.java b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/EnableLRU.java
index 6aaf8cc..5beae60 100755
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/EnableLRU.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/EnableLRU.java
@@ -20,6 +20,7 @@ import org.apache.geode.StatisticsType;
 import org.apache.geode.cache.EvictionAction;
 import org.apache.geode.cache.EvictionAlgorithm;
 import org.apache.geode.cache.Region;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
 
 /**
  * Marker interface to eviction controller that determines if LRU list maintainance is required.
@@ -108,5 +109,7 @@ public interface EnableLRU {
    */
   public void afterEviction();
 
+  public boolean lruLimitExceeded(LRUStatistics lruStatistics, DiskRegionView drv);
+
 }
 

http://git-wip-us.apache.org/repos/asf/geode/blob/87452d32/geode-core/src/main/java/org/apache/geode/internal/cache/lru/HeapLRUCapacityController.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/HeapLRUCapacityController.java
b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/HeapLRUCapacityController.java
index 5e86ce8..f6c8848 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/HeapLRUCapacityController.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/HeapLRUCapacityController.java
@@ -26,6 +26,7 @@ import org.apache.geode.distributed.internal.DistributionConfig;
 import org.apache.geode.internal.statistics.StatisticsTypeFactoryImpl;
 import org.apache.geode.internal.cache.*;
 import org.apache.geode.internal.cache.control.InternalResourceManager;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
 import org.apache.geode.internal.i18n.LocalizedStrings;
 
 import java.util.Properties;
@@ -267,20 +268,10 @@ public class HeapLRUCapacityController extends LRUAlgorithm {
        * greater than the overflow threshold, then we evict the LRU entry.
        */
       public boolean mustEvict(LRUStatistics stats, Region region, int delta) {
-        final GemFireCacheImpl cache;
-        if (region != null) {
-          cache = (GemFireCacheImpl) region.getRegionService();
-        } else {
-          cache = GemFireCacheImpl.getInstance();
-        }
+        final GemFireCacheImpl cache = (GemFireCacheImpl) region.getRegionService();
         InternalResourceManager resourceManager = cache.getResourceManager();
-
-        if (region == null) {
-          return resourceManager.getHeapMonitor().getState().isEviction();
-        }
-
         final boolean monitorStateIsEviction;
-        if (!((AbstractRegion) region).getOffHeap()) {
+        if (!region.getAttributes().getOffHeap()) {
           monitorStateIsEviction = resourceManager.getHeapMonitor().getState().isEviction();
         } else {
           monitorStateIsEviction = resourceManager.getOffHeapMonitor().getState().isEviction();
@@ -292,6 +283,16 @@ public class HeapLRUCapacityController extends LRUAlgorithm {
 
         return monitorStateIsEviction && ((LocalRegion) region).getRegionMap().sizeInVM()
> 0;
       }
+
+      @Override
+      public boolean lruLimitExceeded(LRUStatistics lruStatistics, DiskRegionView drv) {
+        InternalResourceManager resourceManager = drv.getDiskStore().getCache().getResourceManager();
+        if (!drv.getOffHeap()) {
+          return resourceManager.getHeapMonitor().getState().isEviction();
+        } else {
+          return resourceManager.getOffHeapMonitor().getState().isEviction();
+        }
+      }
     };
   }
 

http://git-wip-us.apache.org/repos/asf/geode/blob/87452d32/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUCapacityController.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUCapacityController.java
b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUCapacityController.java
index 3596a07..293be05 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUCapacityController.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUCapacityController.java
@@ -18,6 +18,7 @@ import org.apache.geode.*;
 import org.apache.geode.cache.*;
 import org.apache.geode.internal.statistics.StatisticsTypeFactoryImpl;
 import org.apache.geode.internal.cache.*;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
 import org.apache.geode.internal.i18n.LocalizedStrings;
 
 import java.util.*;
@@ -287,6 +288,11 @@ public final class LRUCapacityController extends LRUAlgorithm implements
Declara
       public boolean mustEvict(LRUStatistics stats, Region region, int delta) {
         return stats.getCounter() + delta > stats.getLimit();
       }
+
+      @Override
+      public boolean lruLimitExceeded(LRUStatistics lruStatistics, DiskRegionView drv) {
+        return stats.getCounter() > stats.getLimit();
+      }
     };
   }
 

http://git-wip-us.apache.org/repos/asf/geode/blob/87452d32/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUMapCallbacks.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUMapCallbacks.java
b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUMapCallbacks.java
index 27a4ec0..ad9db55 100755
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUMapCallbacks.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUMapCallbacks.java
@@ -17,6 +17,7 @@ package org.apache.geode.internal.cache.lru;
 // import org.apache.geode.cache.Region;
 import org.apache.geode.internal.cache.RegionEntry;
 // import org.apache.geode.internal.cache.LocalRegion;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
 
 /**
  * The lru action on the map for evicting items must be called while the current thread is
free of
@@ -54,8 +55,9 @@ public interface LRUMapCallbacks {
   /**
    * Return true if the lru has exceeded its limit and needs to evict. Note that this method
is
    * currently used to prevent disk recovery from faulting in values once the limit is exceeded.
+   * @param drv TODO
    */
-  public boolean lruLimitExceeded();
+  public boolean lruLimitExceeded(DiskRegionView drv);
 
   public void lruCloseStats();
 

http://git-wip-us.apache.org/repos/asf/geode/blob/87452d32/geode-core/src/main/java/org/apache/geode/internal/cache/lru/MemLRUCapacityController.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/MemLRUCapacityController.java
b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/MemLRUCapacityController.java
index 2c2e8ec..f637dc0 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/MemLRUCapacityController.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/MemLRUCapacityController.java
@@ -31,6 +31,7 @@ import org.apache.geode.internal.statistics.StatisticsTypeFactoryImpl;
 import org.apache.geode.internal.cache.AbstractLRURegionMap.CDValueWrapper;
 import org.apache.geode.internal.cache.CachedDeserializableFactory;
 import org.apache.geode.internal.cache.Token;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
 import org.apache.geode.internal.i18n.LocalizedStrings;
 
 
@@ -421,6 +422,11 @@ public final class MemLRUCapacityController extends LRUAlgorithm implements
Decl
       public boolean mustEvict(LRUStatistics stats, Region region, int delta) {
         return stats.getCounter() + delta > stats.getLimit();
       }
+
+      @Override
+      public boolean lruLimitExceeded(LRUStatistics lruStatistics, DiskRegionView drv) {
+        return stats.getCounter() > stats.getLimit();
+      }
     };
   }
 

http://git-wip-us.apache.org/repos/asf/geode/blob/87452d32/geode-core/src/test/java/org/apache/geode/internal/cache/lru/LRUClockJUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/internal/cache/lru/LRUClockJUnitTest.java
b/geode-core/src/test/java/org/apache/geode/internal/cache/lru/LRUClockJUnitTest.java
index 8095d5a..39759f4 100755
--- a/geode-core/src/test/java/org/apache/geode/internal/cache/lru/LRUClockJUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/internal/cache/lru/LRUClockJUnitTest.java
@@ -40,6 +40,7 @@ import org.apache.geode.distributed.DistributedSystem;
 import org.apache.geode.internal.statistics.StatisticsTypeFactoryImpl;
 import org.apache.geode.internal.cache.InternalRegionArguments;
 import org.apache.geode.internal.cache.PlaceHolderDiskRegion;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
 import org.apache.geode.test.junit.categories.IntegrationTest;
 
 /**
@@ -483,6 +484,11 @@ public class LRUClockJUnitTest {
       stats.setLimit(limit());
       return stats;
     }
+
+    @Override
+    public boolean lruLimitExceeded(LRUStatistics lruStatistics, DiskRegionView drv) {
+      throw new UnsupportedOperationException("Not implemented");
+    }
   }
 
   /** overridden in SharedLRUClockTest to test SharedLRUClockHand */

http://git-wip-us.apache.org/repos/asf/geode/blob/87452d32/geode-core/src/test/java/org/apache/geode/internal/offheap/OffHeapLRURecoveryRegressionTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/internal/offheap/OffHeapLRURecoveryRegressionTest.java
b/geode-core/src/test/java/org/apache/geode/internal/offheap/OffHeapLRURecoveryRegressionTest.java
new file mode 100644
index 0000000..f02b7da
--- /dev/null
+++ b/geode-core/src/test/java/org/apache/geode/internal/offheap/OffHeapLRURecoveryRegressionTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.geode.internal.offheap;
+
+import static org.apache.geode.distributed.ConfigurationProperties.LOCATORS;
+import static org.apache.geode.distributed.ConfigurationProperties.MCAST_PORT;
+import static org.junit.Assert.assertEquals;
+
+import java.util.Properties;
+
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.geode.cache.CacheFactory;
+import org.apache.geode.cache.DiskStore;
+import org.apache.geode.cache.DiskStoreFactory;
+import org.apache.geode.cache.Region;
+import org.apache.geode.cache.RegionFactory;
+import org.apache.geode.cache.RegionShortcut;
+import org.apache.geode.distributed.ConfigurationProperties;
+import org.apache.geode.internal.cache.GemFireCacheImpl;
+import org.apache.geode.test.junit.categories.IntegrationTest;
+
+/**
+ * Test to reproduce GEODE-2097.
+ */
+@Category(IntegrationTest.class)
+public class OffHeapLRURecoveryRegressionTest {
+
+  static final String DS_NAME = "OffHeapLRURecoveryRegressionTestDS";
+  /**
+   * Test populates an offheap heaplru persistent region that contains
+   * more data than can fit in offheap memory.
+   * It then recovers the region to demonstrate that recovering
+   * this data will not try to put everything into offheap but
+   * instead leave some of it on disk.
+   */
+  @Test
+  public void recoveringTooMuchDataDoesNotRunOutOfOffHeapMemory() {
+    final int ENTRY_COUNT = 40;
+    long expectedUsedMemory;
+    GemFireCacheImpl gfc = createCache();
+    try {
+      Region<Object, Object> r = createRegion(gfc);
+      byte[] v = new byte[1024*1024];
+      for (int i=0; i < ENTRY_COUNT; i++) {
+        r.put(i, v);
+      }
+      expectedUsedMemory = MemoryAllocatorImpl.getAllocator().getUsedMemory();
+    } finally {
+      gfc.close();
+    }
+    System.setProperty("gemfire.disk.recoverValuesSync", "true");
+    try {
+      gfc = createCache();
+      try {
+        Region<Object, Object> r = createRegion(gfc);
+        try {
+          assertEquals(ENTRY_COUNT, r.size());
+          assertEquals(expectedUsedMemory, MemoryAllocatorImpl.getAllocator().getUsedMemory());
+        } finally {
+          r.destroyRegion();
+          DiskStore ds = gfc.findDiskStore(DS_NAME);
+          ds.destroy();
+        }
+      } finally {
+        gfc.close();
+      }
+    } finally {
+      System.clearProperty("gemfire.disk.recoverValuesSync");
+    }
+  }
+  
+  private GemFireCacheImpl createCache() {
+    Properties props = new Properties();
+    props.setProperty(LOCATORS, "");
+    props.setProperty(MCAST_PORT, "0");
+    props.setProperty(ConfigurationProperties.OFF_HEAP_MEMORY_SIZE, "20m");
+    GemFireCacheImpl result = (GemFireCacheImpl) new CacheFactory(props).create();
+    result.getResourceManager().setEvictionOffHeapPercentage(50.0f);
+    return result;
+  }
+  private Region<Object, Object> createRegion(GemFireCacheImpl gfc) {
+    DiskStoreFactory dsf = gfc.createDiskStoreFactory();
+    dsf.create(DS_NAME);
+    RegionFactory<Object, Object> rf = gfc.createRegionFactory(RegionShortcut.LOCAL_PERSISTENT_OVERFLOW);
+    rf.setOffHeap(true);
+    rf.setDiskStoreName(DS_NAME);
+    Region<Object, Object> r = rf.create("OffHeapLRURecoveryRegressionTestRegion");
+    return r;
+  }
+
+  private void closeCache(GemFireCacheImpl gfc) {
+    gfc.close();
+    MemoryAllocatorImpl.freeOffHeapMemory();
+  }
+}


Mime
View raw message