geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From zho...@apache.org
Subject [09/50] [abbrv] geode git commit: GEODE-3198: Allow bucket creation when over local-max-memory
Date Fri, 21 Jul 2017 06:41:21 GMT
GEODE-3198: Allow bucket creation when over local-max-memory

The bucket will not be created on a member that is over local max
memory, unless all members have failed to create the bucket.
In that case the implementation sets a "force" flag that will
now cause the bucket to be created even on a member that is over
local max memory.

This closes #636


Project: http://git-wip-us.apache.org/repos/asf/geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/43d98e03
Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/43d98e03
Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/43d98e03

Branch: refs/heads/feature/GEM-1483
Commit: 43d98e03f0cb1bdefcc7d18ba4a12272056475e7
Parents: 9f6bfaf
Author: Nick Reich <nreich@pivotal.io>
Authored: Fri Jul 14 11:09:31 2017 -0700
Committer: Darrel Schneider <dschneider@pivotal.io>
Committed: Fri Jul 14 11:54:19 2017 -0700

----------------------------------------------------------------------
 .../cache/PartitionedRegionDataStore.java       |   2 +-
 .../PartitionedRegionDataStoreJUnitTest.java    |  26 +++
 ...artitionedRegionLocalMaxMemoryDUnitTest.java | 229 +++++++------------
 3 files changed, 105 insertions(+), 152 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/geode/blob/43d98e03/geode-core/src/main/java/org/apache/geode/internal/cache/PartitionedRegionDataStore.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/PartitionedRegionDataStore.java
b/geode-core/src/main/java/org/apache/geode/internal/cache/PartitionedRegionDataStore.java
index 2589626..0318c75 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/PartitionedRegionDataStore.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/PartitionedRegionDataStore.java
@@ -934,7 +934,7 @@ public class PartitionedRegionDataStore implements HasCachePerfStats {
       }
       return false;
     }
-    if (!canAccommodateMoreBytesSafely(size)) {
+    if (!forceCreation && !canAccommodateMoreBytesSafely(size)) {
       if (logger.isDebugEnabled()) {
         logger.debug(
             "Partitioned Region {} has exceeded local maximum memory configuration {} Mb,
current size is {} Mb",

http://git-wip-us.apache.org/repos/asf/geode/blob/43d98e03/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionDataStoreJUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionDataStoreJUnitTest.java
b/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionDataStoreJUnitTest.java
index 66792f2..cd0917e 100644
--- a/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionDataStoreJUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionDataStoreJUnitTest.java
@@ -224,4 +224,30 @@ public class PartitionedRegionDataStoreJUnitTest {
       regionAck.put(new Integer(key), "foo");
     }
   }
+
+  @Test
+  public void doesNotCreateBucketIfOverMemoryLimit() {
+    final int numMBytes = 5;
+    final PartitionedRegion regionAck = (PartitionedRegion) new RegionFactory()
+        .setPartitionAttributes(new PartitionAttributesFactory().setRedundantCopies(0)
+            .setLocalMaxMemory(numMBytes).create())
+        .create(this.regionName);
+
+    boolean createdBucket =
+        regionAck.getDataStore().handleManageBucketRequest(1, Integer.MAX_VALUE, null, false);
+    assertFalse(createdBucket);
+  }
+
+  @Test
+  public void createsBucketWhenForcedIfOverMemoryLimit() {
+    final int numMBytes = 5;
+    final PartitionedRegion regionAck = (PartitionedRegion) new RegionFactory()
+        .setPartitionAttributes(new PartitionAttributesFactory().setRedundantCopies(0)
+            .setLocalMaxMemory(numMBytes).create())
+        .create(this.regionName);
+
+    boolean createdBucket =
+        regionAck.getDataStore().handleManageBucketRequest(1, Integer.MAX_VALUE, null, true);
+    assertTrue(createdBucket);
+  }
 }

http://git-wip-us.apache.org/repos/asf/geode/blob/43d98e03/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionLocalMaxMemoryDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionLocalMaxMemoryDUnitTest.java
b/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionLocalMaxMemoryDUnitTest.java
index 89daed0..92699e5 100755
--- a/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionLocalMaxMemoryDUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/internal/cache/PartitionedRegionLocalMaxMemoryDUnitTest.java
@@ -21,16 +21,16 @@ import java.io.DataOutput;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Iterator;
 import java.util.List;
 
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
 import org.apache.geode.DataSerializable;
 import org.apache.geode.DataSerializer;
 import org.apache.geode.cache.Cache;
-import org.apache.geode.cache.PartitionedRegionStorageException;
 import org.apache.geode.cache.Region;
 import org.apache.geode.cache.util.ObjectSizer;
 import org.apache.geode.cache30.CacheSerializableRunnable;
@@ -42,202 +42,128 @@ import org.apache.geode.test.dunit.VM;
 import org.apache.geode.test.junit.categories.DistributedTest;
 
 /**
- * This class is to test localMaxMemory property of partition region while creation of bucket.
+ * This class is to test LOCAL_MAX_MEMORY property of partition region while creation of
bucket.
  */
 @Category(DistributedTest.class)
 public class PartitionedRegionLocalMaxMemoryDUnitTest extends PartitionedRegionDUnitTestCase
{
 
-  /** Prefix is used in name of Partition Region */
-  private static String prPrefix = null;
+  private static final int LOCAL_MAX_MEMORY = 1;
+  private static int MAX_REGIONS = 1;
+  private static final int REDUNDANCY = 1;
 
-  /** Maximum number of regions * */
-  static int MAX_REGIONS = 1;
+  private String regionNamePrefix = null;
+  private List<VM> vms = new ArrayList<>(2);
 
-  /** local maxmemory used for the creation of the partition region */
-  int localMaxMemory = 1;
+  @Before
+  public void setup() {
+    Host host = Host.getHost(0);
+    vms.add(host.getVM(0));
+    vms.add(host.getVM(1));
+  }
 
-  /** to store references of 4 vms */
-  VM vm[] = new VM[4];
+  @After
+  public void cleanup() {
+    destroyRegion(vms.get(0));
+  }
 
   /**
    * This test performs following operations <br>
-   * 1.Create Partition region with localMaxMemory = 1MB on all the VMs </br>
+   * 1.Create Partition region with LOCAL_MAX_MEMORY = 1MB on all the VMs </br>
    * <br>
    * 2.Put objects in partition region so that only one bucket gets created and size of that
bucket
-   * exceeds localMaxMemory <br>
+   * exceeds LOCAL_MAX_MEMORY <br>
    * 3.Put object such that new bucket gets formed</br>
    * <br>
-   * 4.Test should throw PartitionedRegionStorageException when it tries to create new bucket</br>
+   * 4.Test should create a new bucket</br>
    */
   @Test
   public void testLocalMaxMemoryInPartitionedRegion() {
-    Host host = Host.getHost(0);
-    /** creating 4 VMs */
-    this.vm[0] = host.getVM(0);
-    this.vm[1] = host.getVM(1);
-    this.vm[2] = null;
-    this.vm[3] = null;
-    /** Prefix will be used for naming the partititon Region */
-    prPrefix = "testLocalMaxMemoryInPartitionedRegion";
-    /** these indices represents range of partition regions present in each VM */
-    int startIndexForRegion = 0;
-    int endIndexForRegion = MAX_REGIONS;
-    int startIndexForNode = 0;
-    int endIndexForNode = 2;
-    // creating partition region on 4 nodes with
-    // localMaxMemory=1MB redundancy = 3
-    List vmList = addNodeToList(startIndexForNode, endIndexForNode);
-    localMaxMemory = 1;
-    final int redundancy = 1;
-    System.setProperty(PartitionedRegion.RETRY_TIMEOUT_PROPERTY, "20000");
-    createPartitionRegion(vmList, startIndexForRegion, endIndexForRegion, localMaxMemory,
-        redundancy, false);
-    System.setProperty(PartitionedRegion.RETRY_TIMEOUT_PROPERTY,
-        Integer.toString(PartitionedRegionHelper.DEFAULT_TOTAL_WAIT_RETRY_ITERATION));
-    putFromOneVm(vm[0], true);
-    putFromOneVm(vm[0], false);
-    destroyRegion(vm[0]);
+    regionNamePrefix = "maxMemoryTest";
+    createPartitionRegionOnAllVMs(false);
+    VM vm = vms.get(0);
+    putFromOneVm(vm, 10, true);
+    putFromOneVm(vm, 21, false);
   }
 
   /**
-   * This test makes sure that we don't enforce the localMaxMemory setting when eviction
is enabled.
+   * This test makes sure that we don't enforce the LOCAL_MAX_MEMORY setting when eviction
is
+   * enabled.
    */
   @Test
   public void testLocalMaxMemoryInPartitionedRegionWithEviction() {
-    Host host = Host.getHost(0);
-    /** creating 4 VMs */
-    this.vm[0] = host.getVM(0);
-    this.vm[1] = host.getVM(1);
-    this.vm[2] = null;
-    this.vm[3] = null;
-    /** Prefix will be used for naming the partititon Region */
-    prPrefix = "testLocalMaxMemoryInPartitionedRegion";
-    /** these indices represents range of partition regions present in each VM */
-    int startIndexForRegion = 0;
-    int endIndexForRegion = MAX_REGIONS;
-    int startIndexForNode = 0;
-    int endIndexForNode = 2;
-    // creating partition region on 4 nodes with
-    // localMaxMemory=1MB redundancy = 3
-    List vmList = addNodeToList(startIndexForNode, endIndexForNode);
-    localMaxMemory = 1;
-    final int redundancy = 1;
-    System.setProperty(PartitionedRegion.RETRY_TIMEOUT_PROPERTY, "20000");
-    createPartitionRegion(vmList, startIndexForRegion, endIndexForRegion, localMaxMemory,
-        redundancy, true);
-    System.setProperty(PartitionedRegion.RETRY_TIMEOUT_PROPERTY,
-        Integer.toString(PartitionedRegionHelper.DEFAULT_TOTAL_WAIT_RETRY_ITERATION));
-    putFromOneVm(vm[0], true);
-    putFromOneVm(vm[0], true);
-    destroyRegion(vm[0]);
+    regionNamePrefix = "maxMemoryWithEvictionTest";
+    createPartitionRegionOnAllVMs(true);
+    VM vm = vms.get(0);
+    putFromOneVm(vm, 10, true);
+    putFromOneVm(vm, 10, true);
   }
 
-  /**
-   * function is used perform put() operation from one VM
-   * 
-   * @param vm
-   * @param objectFlg
-   */
-  private void putFromOneVm(VM vm, boolean objectFlg) {
-    vm.invoke(putObjectInPartitionRegion(objectFlg));
+  private void putFromOneVm(VM vm, int objectId, boolean fillMemory) {
+    if (fillMemory) {
+      vm.invoke(fillRegion(objectId));
+    } else {
+      vm.invoke(putObjectInPartitionRegion(objectId));
+    }
   }
 
-  /**
-   * This function is used to put objects of different hashcode depending upon value of objectFlag
-   * 
-   * @param objectFlg
-   * @return
-   */
-  private CacheSerializableRunnable putObjectInPartitionRegion(final boolean objectFlg) {
+  private CacheSerializableRunnable putObjectInPartitionRegion(int objectId) {
+    CacheSerializableRunnable putObject = new CacheSerializableRunnable("putObject") {
+      public void run2() {
+        PartitionedRegion pr = getRegion();
+        TestObject1 kv = new TestObject1("testObject1" + 0, objectId);
+        pr.put(kv, kv);
+        LogWriterUtils.getLogWriter().info(
+            "putObjectInPartitionRegion() - Put operation with different identifier done
successfully");
+      }
+    };
+    return putObject;
+  }
 
+  private CacheSerializableRunnable fillRegion(int objectId) {
     CacheSerializableRunnable putObject = new CacheSerializableRunnable("putObject") {
       public void run2() {
         Cache cache = getCache();
-        PartitionedRegion pr = (PartitionedRegion) cache
-            .getRegion(Region.SEPARATOR + "testLocalMaxMemoryInPartitionedRegion0");
-        assertNotNull("Name of region : " + pr.getName(), pr);
-        int i = 0;
+        PartitionedRegion pr = getRegion();
 
-        if (objectFlg == true) {
-          long size = 0;
-          while ((size =
-              pr.getDataStore().currentAllocatedMemory()) < PartitionedRegionHelper.BYTES_PER_MB)
{
-            cache.getLogger().info("size: " + size);
-            Object obj = new TestObject1("testObject1" + i, 10);
-            pr.put(obj, obj);
-            i++;
-          }
-          assertEquals(1, pr.getDataStore().localBucket2RegionMap.size());
-          LogWriterUtils.getLogWriter()
-              .info("putObjectInPartitionRegion() - Put operation done successfully");
-        } else {
-          final String expectedExceptions = PartitionedRegionStorageException.class.getName();
-          getCache().getLogger()
-              .info("<ExpectedException action=add>" + expectedExceptions + "</ExpectedException>");
-          try {
-            TestObject1 kv = new TestObject1("testObject1" + i, 21);
-            pr.put(kv, kv);
-            fail("Bucket gets created even if no memory is available");
-          } catch (PartitionedRegionStorageException e) {
-            LogWriterUtils.getLogWriter().info(
-                "putObjectInPartitionRegion()- got correct PartitionedRegionStorageException
while creating bucket when no memory is available");
-          }
-          getCache().getLogger().info(
-              "<ExpectedException action=remove>" + expectedExceptions + "</ExpectedException>");
-        }
+        fillAllMemoryWithPuts(cache, pr, objectId);
+        assertEquals(1, pr.getDataStore().localBucket2RegionMap.size());
+        LogWriterUtils.getLogWriter()
+            .info("putObjectInPartitionRegion() - Put operation done successfully");
       }
     };
     return putObject;
   }
 
-  /**
-   * This function createas multiple partition regions on nodes specified in the vmList
-   * 
-   * @param evict
-   */
-  private void createPartitionRegion(List vmList, int startIndexForRegion, int endIndexForRegion,
-      int localMaxMemory, int redundancy, boolean evict) {
-    Iterator nodeIterator = vmList.iterator();
-    while (nodeIterator.hasNext()) {
-      VM vm = (VM) nodeIterator.next();
-      vm.invoke(createMultiplePartitionRegion(prPrefix, startIndexForRegion, endIndexForRegion,
-          redundancy, localMaxMemory, evict));
-    }
+  private PartitionedRegion getRegion() {
+    Cache cache = getCache();
+    return (PartitionedRegion) cache.getRegion(Region.SEPARATOR + regionNamePrefix + "0");
   }
 
-  /**
-   * This function adds nodes to node list
-   * 
-   * @param startIndexForNode
-   * @param endIndexForNode
-   * @return
-   */
-  private List addNodeToList(int startIndexForNode, int endIndexForNode) {
-    List localvmList = new ArrayList();
-    for (int i = startIndexForNode; i < endIndexForNode; i++) {
-      localvmList.add(vm[i]);
+  private void fillAllMemoryWithPuts(Cache cache, PartitionedRegion pr, int objectId) {
+    int i = 0;
+    long allocatedMemory;
+    while ((allocatedMemory =
+        pr.getDataStore().currentAllocatedMemory()) < PartitionedRegionHelper.BYTES_PER_MB)
{
+      cache.getLogger().info("size: " + allocatedMemory);
+      Object obj = new TestObject1("testObject1" + i, objectId);
+      pr.put(obj, obj);
+      i++;
     }
-    return localvmList;
+    assertEquals(1, pr.getDataStore().localBucket2RegionMap.size());
   }
 
-  /**
-   * this function creates vms in given host
-   * 
-   * @param host
-   */
-  private void createVMs(Host host) {
-    for (int i = 0; i < 4; i++) {
-      vm[i] = host.getVM(i);
+  private void createPartitionRegionOnAllVMs(boolean evict) {
+    for (VM vm : vms) {
+      vm.invoke(createMultiplePartitionRegion(regionNamePrefix, 0, MAX_REGIONS, REDUNDANCY,
+          LOCAL_MAX_MEMORY, evict));
     }
   }
 
   private void destroyRegion(VM vm) {
     SerializableRunnable destroyObj = new CacheSerializableRunnable("destroyObj") {
       public void run2() {
-        Cache cache = getCache();
-        PartitionedRegion pr = (PartitionedRegion) cache
-            .getRegion(Region.SEPARATOR + "testLocalMaxMemoryInPartitionedRegion0");
-        assertNotNull("Name of region : " + pr.getName(), pr);
+        PartitionedRegion pr = getRegion();
+        assertNotNull(pr);
         pr.destroyRegion();
       }
     };
@@ -245,7 +171,8 @@ public class PartitionedRegionLocalMaxMemoryDUnitTest extends PartitionedRegionD
   }
 
   /**
-   * Object used for the put() operation as key and object
+   * Object used for the put() operation as key and object. The objectIdentifier is used
to provide
+   * a predetermined hashcode for the object.
    */
   static public class TestObject1 implements DataSerializable, Sizeable {
     String name;


Mime
View raw message