geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From u..@apache.org
Subject [2/9] incubator-geode git commit: GEODE-1128: Fix colocation configuration problems
Date Mon, 22 Aug 2016 17:41:59 GMT
GEODE-1128: Fix colocation configuration problems

Test parent regions on child region creates and throw IllegalStateException
if parent region does not exist

Log warnings when persistent colocated child regions don't exist

Create Unit and DUnit tests for new exceptions and warnings

This closes #234


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/da29fb36
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/da29fb36
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/da29fb36

Branch: refs/heads/feature/GEODE-420
Commit: da29fb366726f7ca28b3273bf99592bc7f5681a0
Parents: 306fda0
Author: Ken Howe <khowe@pivotal.io>
Authored: Thu Aug 18 15:00:22 2016 -0700
Committer: Darrel Schneider <dschneider@pivotal.io>
Committed: Thu Aug 18 16:29:15 2016 -0700

----------------------------------------------------------------------
 .../internal/cache/ColocationHelper.java        |  38 +-
 .../internal/cache/ColocationLogger.java        | 180 ++++
 .../internal/cache/PartitionRegionConfig.java   |   2 +-
 .../internal/cache/PartitionedRegion.java       |  38 +
 .../gemfire/internal/i18n/LocalizedStrings.java |   3 +
 .../internal/cache/ColocationHelperTest.java    | 148 +++
 ...tentColocatedPartitionedRegionDUnitTest.java | 973 ++++++++++++++++++-
 .../PersistentPartitionedRegionTestBase.java    |  17 +-
 8 files changed, 1382 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/da29fb36/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/ColocationHelper.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/ColocationHelper.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/ColocationHelper.java
index 012a77f..aec8ef8 100755
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/ColocationHelper.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/ColocationHelper.java
@@ -29,7 +29,10 @@ import com.gemstone.gemfire.internal.cache.execute.InternalRegionFunctionContext
 import com.gemstone.gemfire.internal.cache.partitioned.PRLocallyDestroyedException;
 import com.gemstone.gemfire.internal.cache.persistence.PRPersistentConfig;
 import com.gemstone.gemfire.internal.cache.wan.parallel.ParallelGatewaySenderQueue;
+import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
 import com.gemstone.gemfire.internal.logging.LogService;
+import com.gemstone.gemfire.internal.logging.log4j.LocalizedMessage;
+
 import org.apache.logging.log4j.Logger;
 
 import java.io.Serializable;
@@ -57,9 +60,10 @@ public class ColocationHelper {
   /**
    * An utility method to retrieve colocated region of a given partitioned
    * region
-   * 
+   *
    * @param partitionedRegion
    * @return colocated PartitionedRegion
+   * @throws IllegalStateException for missing colocated region
    * @since GemFire 5.8Beta
    */
   public static PartitionedRegion getColocatedRegion(
@@ -75,11 +79,21 @@ public class ColocationHelper {
         .getCache());
     PartitionRegionConfig prConf = (PartitionRegionConfig)prRoot
         .get(getRegionIdentifier(colocatedWith));
+    if (prConf == null) {
+      throw new IllegalStateException(LocalizedStrings.ColocationHelper_REGION_SPECIFIED_IN_COLOCATEDWITH_DOES_NOT_EXIST.toLocalizedString(
+          new Object[] {colocatedWith, partitionedRegion.getFullPath()}));
+    }
     int prID = prConf.getPRId();
     PartitionedRegion colocatedPR = null;
     try {
       colocatedPR = PartitionedRegion.getPRFromId(prID);
-      colocatedPR.waitOnBucketMetadataInitialization();
+      if (colocatedPR != null) {
+        colocatedPR.waitOnBucketMetadataInitialization();
+      }
+      else {
+        throw new IllegalStateException(LocalizedStrings.ColocationHelper_REGION_SPECIFIED_IN_COLOCATEDWITH_DOES_NOT_EXIST.toLocalizedString(
+            new Object[] {colocatedWith, partitionedRegion.getFullPath()}));
+      }
     }
     catch (PRLocallyDestroyedException e) {
       if (logger.isDebugEnabled()) {
@@ -186,8 +200,7 @@ public class ColocationHelper {
    * member, but have not yet been created.
    */
   private static boolean hasOfflineColocatedChildRegions(PartitionedRegion region) {
-    
-    boolean hasOfflineChildren;
+    boolean hasOfflineChildren = false;
     int oldLevel = LocalRegion.setThreadInitLevelRequirement(LocalRegion.ANY_INIT);
     try {
       GemFireCacheImpl cache = region.getCache();
@@ -208,24 +221,26 @@ public class ColocationHelper {
               //If the child region is offline, return true
               //unless it is a parallel queue that the user has removed.
               if(!ignoreUnrecoveredQueue(region, childName)) {
-                return true;
+                region.addMissingColocatedRegionLogger(childName);
+                hasOfflineChildren = true;
               }
             } else {
               //Otherwise, look for offline children of that region.
               if(hasOfflineColocatedChildRegions(childRegion)) {
-                return true;
+                hasOfflineChildren = true;
+                // Add the offline children of this child to the region's missingChildren list
+                region.addMissingColocatedRegionLogger(childRegion);
               }
             }
           }
         }
       }
-    
     } finally {
       LocalRegion.setThreadInitLevelRequirement(oldLevel);
     }
-    return false;
+    return hasOfflineChildren;
   }
-  
+
 
   private static boolean ignoreUnrecoveredQueue(PartitionedRegion region, String childName) {
     //Hack for #50120 if the childRegion is an async queue, but we
@@ -381,9 +396,10 @@ public class ColocationHelper {
     while ( itr.hasNext()) {
       try {
         String prName = (String)itr.next();
-        /*if (prName == prName) {
+        if (prName.equals(partitionedRegion.getRegionIdentifier())) {
+          // region can't be a child of itself
           continue;
-        }*/
+        }
         try {
           prConf = (PartitionRegionConfig)prRoot.get(prName);
         }

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/da29fb36/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/ColocationLogger.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/ColocationLogger.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/ColocationLogger.java
new file mode 100644
index 0000000..fab8eca
--- /dev/null
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/ColocationLogger.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gemstone.gemfire.internal.cache;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.logging.log4j.Logger;
+
+import com.gemstone.gemfire.CancelCriterion;
+import com.gemstone.gemfire.SystemFailure;
+import com.gemstone.gemfire.distributed.DistributedSystem;
+import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
+import com.gemstone.gemfire.internal.logging.LogService;
+import com.gemstone.gemfire.internal.logging.log4j.LocalizedMessage;
+
+/**
+ * Provides logging when regions are missing from a colocation hierarchy. This logger runs in
+ * it's own thread and waits for child regions to be created before logging them as missing.
+ *
+ */
+public class ColocationLogger implements Runnable {
+  private static final Logger logger = LogService.getLogger();
+
+  private final PartitionedRegion region;
+  private final List<String> missingChildren = new ArrayList<String>();
+  private final Thread loggerThread;
+  private final Object loggerLock = new Object();
+
+  /**
+   * Sleep period (milliseconds) between posting log entries.
+   */
+  private static final int DEFAULT_LOG_INTERVAL = 30000;
+  private static int LOG_INTERVAL = DEFAULT_LOG_INTERVAL;
+
+  /**
+   * @param region the region that owns this logger instance
+   */
+  public ColocationLogger(PartitionedRegion region) {
+    this.region = region;
+    loggerThread = new Thread(this,"ColocationLogger for " + region.getName());
+    loggerThread.start();
+  }
+
+  public void run()
+  {
+    CancelCriterion stopper = region
+        .getGemFireCache().getDistributedSystem().getCancelCriterion();
+    DistributedSystem.setThreadsSocketPolicy(true /* conserve sockets */);
+    SystemFailure.checkFailure();
+    if (stopper.cancelInProgress() != null) {
+      return;
+    }
+    try {
+      run2();
+    } catch (VirtualMachineError err) {
+      SystemFailure.initiateFailure(err);
+      // If this ever returns, rethrow the error.  We're poisoned
+      // now, so don't let this thread continue.
+      throw err;
+    } catch (Throwable t) {
+      // Whenever you catch Error or Throwable, you must also
+      // catch VirtualMachineError (see above).  However, there is
+      // _still_ a possibility that you are dealing with a cascading
+      // error condition, so you also need to check to see if the JVM
+      // is still usable:
+      SystemFailure.checkFailure();
+      if (logger.isDebugEnabled()) {
+        logger.debug("Unexpected exception in colocation", t);
+      }
+    }
+  }
+
+  /**
+   * Writes a log entry every SLEEP_PERIOD when there are missing colocated child regions
+   * for this region.
+   * @throws InterruptedException
+   */
+  private void run2() throws InterruptedException {
+    boolean firstLogIteration = true;
+    synchronized(loggerLock) {
+      while (true) {
+        int sleepMillis = getLogInterval();
+        // delay for first log message is half the time of the interval between subsequent log messages
+        if (firstLogIteration) {
+          firstLogIteration = false;
+          sleepMillis /= 2;
+        }
+        loggerLock.wait(sleepMillis);
+        PRHARedundancyProvider rp = region.getRedundancyProvider();
+        if (rp != null && rp.isPersistentRecoveryComplete()) {
+          //Terminate the logging thread, recoverycomplete is only true when there are no missing colocated regions
+          break;
+        }
+        List<String>  existingRegions;
+        Map coloHierarchy = ColocationHelper.getAllColocationRegions(region);
+        missingChildren.removeAll(coloHierarchy.keySet());
+        if(missingChildren.isEmpty()) {
+          break;
+        }
+        logMissingRegions(region);
+      }
+    }
+  }
+
+  public void stopLogger() {
+    synchronized (loggerLock) {
+      missingChildren.clear();
+      loggerLock.notify();
+    }
+  }
+
+  public void addMissingChildRegion(String childFullPath) {
+    synchronized (loggerLock) {
+      if (!missingChildren.contains(childFullPath)) {
+        missingChildren.add(childFullPath);
+      }
+    }
+  }
+
+  public void addMissingChildRegions(PartitionedRegion childRegion) {
+    List<String> missingDescendants = childRegion.getMissingColocatedChildren();
+    for (String name:missingDescendants) {
+      addMissingChildRegion(name);
+    }
+  }
+
+  public List<String> getMissingChildRegions() {
+    return new ArrayList<String>(missingChildren);
+  }
+
+  /**
+   * Write the a logger warning for a PR that has colocated child regions that are missing.
+   * @param region the parent region that has missing child regions
+   */
+  private void logMissingRegions(PartitionedRegion region) {
+    String namesOfMissing = "";
+    if (!missingChildren.isEmpty()) {
+      namesOfMissing = String.join("\n\t", missingChildren);
+    }
+    String multipleChildren;
+    String singular = "";
+    String plural = "s";
+    multipleChildren = missingChildren.size() > 1 ? plural : singular;
+    namesOfMissing = String.join("\n\t", multipleChildren, namesOfMissing);
+    logger.warn(LocalizedMessage.create(LocalizedStrings.ColocationLogger_PERSISTENT_DATA_RECOVERY_OF_REGION_PREVENTED_BY_OFFLINE_COLOCATED_CHILDREN,
+        new Object[]{region.getFullPath(), namesOfMissing}));
+  }
+
+  public static int getLogInterval() {
+    return LOG_INTERVAL;
+  }
+
+  /*
+   * Test hook to allow unit test tests to run faster by tweak the interval between log messages
+   */
+  public synchronized static int testhookSetLogInterval(int sleepMillis) {
+    int currentSleep = LOG_INTERVAL;
+    LOG_INTERVAL = sleepMillis;
+    return currentSleep;
+  }
+  public synchronized static void testhookResetLogInterval() {
+    LOG_INTERVAL = DEFAULT_LOG_INTERVAL;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/da29fb36/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionRegionConfig.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionRegionConfig.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionRegionConfig.java
index 6d7c1ca..926c47a 100644
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionRegionConfig.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionRegionConfig.java
@@ -42,7 +42,7 @@ import com.gemstone.gemfire.internal.util.VersionedArrayList;
  * Maintains configuration information for a PartitionedRegion. Instances are
  * stored in the allPartitionedRegion.
  */
-public final class PartitionRegionConfig extends ExternalizableDSFID implements Versionable
+public class PartitionRegionConfig extends ExternalizableDSFID implements Versionable
 {
 
   /** PRId. */

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/da29fb36/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java
index a87ed9c..485b94d 100755
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/cache/PartitionedRegion.java
@@ -303,6 +303,8 @@ public class PartitionedRegion extends LocalRegion implements
   
   private final PartitionedRegion colocatedWithRegion;
 
+  private ColocationLogger missingColocatedRegionLogger;
+
   private List<BucketRegion> sortedBuckets; 
   
   private ScheduledExecutorService bucketSorter;
@@ -7462,6 +7464,41 @@ public class PartitionedRegion extends LocalRegion implements
     basicDestroyRegion(event, true);
   }
 
+  private void stopMissingColocatedRegionLogger() {
+    if (missingColocatedRegionLogger != null) {
+      missingColocatedRegionLogger.stopLogger();
+    }
+    missingColocatedRegionLogger = null;
+  }
+
+  public void addMissingColocatedRegionLogger() {
+    if (missingColocatedRegionLogger == null) {
+      missingColocatedRegionLogger = new ColocationLogger(this);
+    }
+  }
+
+  public void addMissingColocatedRegionLogger(String childName) {
+    if (missingColocatedRegionLogger == null) {
+      missingColocatedRegionLogger = new ColocationLogger(this);
+    }
+    missingColocatedRegionLogger.addMissingChildRegion(childName);
+  }
+
+  public void addMissingColocatedRegionLogger(PartitionedRegion childRegion) {
+    if (missingColocatedRegionLogger == null) {
+      missingColocatedRegionLogger = new ColocationLogger(this);
+    }
+    missingColocatedRegionLogger.addMissingChildRegions(childRegion);
+  }
+
+  public List<String> getMissingColocatedChildren() {
+    ColocationLogger regionLogger = missingColocatedRegionLogger;
+    if (regionLogger != null) {
+      return regionLogger.getMissingChildRegions();
+    }
+    return Collections.emptyList();
+  }
+
   /**Globally destroy the partitioned region by sending a message
    * to a data store to do the destroy.
    * @return true if the region was destroyed successfully
@@ -7970,6 +8007,7 @@ public class PartitionedRegion extends LocalRegion implements
     this.cache.getResourceManager(false).removeResourceListener(this);
     
     final Operation op = event.getOperation();
+    stopMissingColocatedRegionLogger();
     if (op.isClose() || Operation.REGION_LOCAL_DESTROY.equals(op)) {
       try {
         if (Operation.CACHE_CLOSE.equals(op) || 

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/da29fb36/geode-core/src/main/java/com/gemstone/gemfire/internal/i18n/LocalizedStrings.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/com/gemstone/gemfire/internal/i18n/LocalizedStrings.java b/geode-core/src/main/java/com/gemstone/gemfire/internal/i18n/LocalizedStrings.java
index 443fe78..5f9213d 100755
--- a/geode-core/src/main/java/com/gemstone/gemfire/internal/i18n/LocalizedStrings.java
+++ b/geode-core/src/main/java/com/gemstone/gemfire/internal/i18n/LocalizedStrings.java
@@ -3770,6 +3770,9 @@ public class LocalizedStrings {
 
   public static final StringId AbstractDistributionConfig_SSL_ENABLED_COMPONENTS_SET_INVALID_DEPRECATED_SSL_SET = new StringId(6639,"When using ssl-enabled-components one cannot use any other SSL properties other than cluster-ssl-* or the corresponding aliases");
 
+  public static final StringId ColocationHelper_REGION_SPECIFIED_IN_COLOCATEDWITH_DOES_NOT_EXIST = new StringId(6640, "Region specified in ''colocated-with'' ({0}) for region {1} does not exist. It should be created before setting ''colocated-with'' attribute for this region.");
+  public static final StringId ColocationLogger_PERSISTENT_DATA_RECOVERY_OF_REGION_PREVENTED_BY_OFFLINE_COLOCATED_CHILDREN = new StringId(6641, "Persistent data recovery for region {0} is prevented by offline colocated region{1}");
+
   /** Testing strings, messageId 90000-99999 **/
   
   /** These are simple messages for testing, translated with Babelfish. **/

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/da29fb36/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/ColocationHelperTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/ColocationHelperTest.java b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/ColocationHelperTest.java
new file mode 100644
index 0000000..4206f6e
--- /dev/null
+++ b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/ColocationHelperTest.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.gemstone.gemfire.internal.cache;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import org.mockito.ArgumentCaptor;
+
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.LogEvent;
+
+import org.apache.logging.log4j.core.Logger;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.gemstone.gemfire.cache.PartitionAttributes;
+import com.gemstone.gemfire.cache.Region;
+import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem;
+import com.gemstone.gemfire.test.fake.Fakes;
+import com.gemstone.gemfire.test.junit.categories.UnitTest;
+
+@Category(UnitTest.class)
+public class ColocationHelperTest {
+  private GemFireCacheImpl cache;
+  private GemFireCacheImpl oldCacheInstance;
+  private InternalDistributedSystem system;
+  private PartitionedRegion pr;
+  private DistributedRegion prRoot;
+  private PartitionAttributes pa;
+  private PartitionRegionConfig prc;
+  private Logger logger;
+  private Appender mockAppender;
+  private ArgumentCaptor<LogEvent> loggingEventCaptor;
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @Before
+  public void setUp() throws Exception {
+    cache = Fakes.cache();
+    system = (InternalDistributedSystem) cache.getDistributedSystem();
+    pr = mock(PartitionedRegion.class);
+    prRoot = mock(DistributedRegion.class);
+    pa = mock(PartitionAttributes.class);
+    prc = mock(PartitionRegionConfig.class);
+    cache = Fakes.cache();
+    oldCacheInstance = GemFireCacheImpl.setInstanceForTests(cache);
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @After
+  public void tearDown() throws Exception {
+    GemFireCacheImpl.setInstanceForTests(oldCacheInstance);
+  }
+
+  /**
+   * Test method for {@link com.gemstone.gemfire.internal.cache.ColocationHelper#getColocatedRegion(com.gemstone.gemfire.internal.cache.PartitionedRegion)}.
+   */
+  @Test
+  public void testGetColocatedRegionThrowsIllegalStateExceptionForMissingParentRegion() {
+    when(pr.getCache()).thenReturn(cache);
+    when(cache.getRegion(PartitionedRegionHelper.PR_ROOT_REGION_NAME, true)).thenReturn(mock(DistributedRegion.class));
+    when(pr.getPartitionAttributes()).thenReturn(pa);
+    when(pr.getFullPath()).thenReturn("/region1");
+    when(pa.getColocatedWith()).thenReturn("region2");
+
+    PartitionedRegion colocatedPR;
+    boolean caughtIllegalStateException = false;
+    try {
+      colocatedPR = ColocationHelper.getColocatedRegion(pr);
+    } catch (Exception e) {
+      assertEquals("Expected IllegalStateException for missing colocated parent region", IllegalStateException.class, e.getClass());
+      assertTrue("Expected IllegalStateException to be thrown for missing colocated region",
+          e.getMessage().matches("Region specified in 'colocated-with' .* does not exist.*"));
+      caughtIllegalStateException = true;
+    }
+    assertTrue(caughtIllegalStateException);
+  }
+
+  /**
+   * Test method for {@link com.gemstone.gemfire.internal.cache.ColocationHelper#getColocatedRegion(com.gemstone.gemfire.internal.cache.PartitionedRegion)}.
+   */
+  @Test
+  public void testGetColocatedRegionLogsWarningForMissingRegionWhenPRConfigHasRegion() {
+    when(pr.getCache()).thenReturn(cache);
+    when(cache.getRegion(PartitionedRegionHelper.PR_ROOT_REGION_NAME, true)).thenReturn(prRoot);
+    when(pr.getPartitionAttributes()).thenReturn(pa);
+    when(pr.getFullPath()).thenReturn("/region1");
+    when(pa.getColocatedWith()).thenReturn("region2");
+    when(((Region)prRoot).get("#region2")).thenReturn(prc);
+
+    PartitionedRegion colocatedPR = null;
+    boolean caughtIllegalStateException = false;
+    try {
+      colocatedPR = ColocationHelper.getColocatedRegion(pr);
+    } catch (Exception e) {
+      assertEquals("Expected IllegalStateException for missing colocated parent region", IllegalStateException.class, e.getClass());
+      assertTrue("Expected IllegalStateException to be thrown for missing colocated region",
+          e.getMessage().matches("Region specified in 'colocated-with' .* does not exist.*"));
+      caughtIllegalStateException = true;
+    }
+    assertTrue(caughtIllegalStateException);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/da29fb36/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/partitioned/PersistentColocatedPartitionedRegionDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/partitioned/PersistentColocatedPartitionedRegionDUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/partitioned/PersistentColocatedPartitionedRegionDUnitTest.java
index d8b3514..b701b70 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/partitioned/PersistentColocatedPartitionedRegionDUnitTest.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/partitioned/PersistentColocatedPartitionedRegionDUnitTest.java
@@ -17,20 +17,39 @@
 package com.gemstone.gemfire.internal.cache.partitioned;
 
 import org.junit.experimental.categories.Category;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.Logger;
 import org.junit.Test;
 
 import static org.junit.Assert.*;
 
+import static com.jayway.awaitility.Awaitility.await;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.times;
+
 import com.gemstone.gemfire.test.dunit.cache.internal.JUnit4CacheTestCase;
 import com.gemstone.gemfire.test.dunit.internal.JUnit4DistributedTestCase;
 import com.gemstone.gemfire.test.junit.categories.DistributedTest;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 
+import com.jayway.awaitility.core.ConditionTimeoutException;
 import org.junit.experimental.categories.Category;
 
 import com.gemstone.gemfire.admin.internal.AdminDistributedSystemImpl;
@@ -50,6 +69,7 @@ import com.gemstone.gemfire.distributed.internal.DistributionMessage;
 import com.gemstone.gemfire.distributed.internal.DistributionMessageObserver;
 import com.gemstone.gemfire.distributed.internal.InternalDistributedSystem;
 import com.gemstone.gemfire.internal.FileUtil;
+import com.gemstone.gemfire.internal.cache.ColocationLogger;
 import com.gemstone.gemfire.internal.cache.InitialImageOperation.RequestImageMessage;
 import com.gemstone.gemfire.internal.cache.PartitionedRegion;
 import com.gemstone.gemfire.internal.cache.control.InternalResourceManager;
@@ -68,8 +88,15 @@ import com.gemstone.gemfire.test.junit.categories.FlakyTest;
 @Category(DistributedTest.class)
 public class PersistentColocatedPartitionedRegionDUnitTest extends PersistentPartitionedRegionTestBase {
 
+  private static final String PATTERN_FOR_MISSING_CHILD_LOG = "(?s)Persistent data recovery for region .*is prevented by offline colocated region.*";
   private static final int NUM_BUCKETS = 15;
   private static final int MAX_WAIT = 30 * 1000;
+  private static final int DEFAULT_NUM_EXPECTED_LOG_MESSAGES = 1;
+  private static int numExpectedLogMessages = DEFAULT_NUM_EXPECTED_LOG_MESSAGES;
+  private static int numChildPRs = 1;
+  private static int numChildPRGenerations = 2;
+  // Default region creation delay long enough for the initial cycle of logger warnings
+  private static int delayForChildCreation = MAX_WAIT * 2 / 3;
 
   public PersistentColocatedPartitionedRegionDUnitTest() {
     super();
@@ -232,6 +259,950 @@ public class PersistentColocatedPartitionedRegionDUnitTest extends PersistentPar
 
   }
 
+  private void createPR(String regionName, boolean persistent) {
+    createPR(regionName, null, persistent, "disk");
+  }
+
+  private void createPR(String regionName, String colocatedWith, boolean persistent) {
+    createPR(regionName, colocatedWith, persistent, "disk");
+  }
+
+  private void createPR(String regionName, String colocatedRegionName, boolean persistent, String diskName) {
+    Cache cache = getCache();
+
+    DiskStore ds = cache.findDiskStore(diskName);
+    if (ds == null) {
+      ds = cache.createDiskStoreFactory().setDiskDirs(getDiskDirs())
+          .create(diskName);
+    }
+    AttributesFactory af = new AttributesFactory();
+    PartitionAttributesFactory paf = new PartitionAttributesFactory();
+    paf.setRedundantCopies(0);
+    if (colocatedRegionName != null) {
+      paf.setColocatedWith(colocatedRegionName);
+    }
+    af.setPartitionAttributes(paf.create());
+    if (persistent) {
+      af.setDataPolicy(DataPolicy.PERSISTENT_PARTITION);
+      af.setDiskStoreName(diskName);
+    } else {
+      af.setDataPolicy(DataPolicy.PARTITION);
+      af.setDiskStoreName(null);
+    }
+    cache.createRegion(regionName, af.create());
+  }
+
+  private SerializableRunnable createPRsColocatedPairThread = new SerializableRunnable("create2PRs") {
+    public void run() {
+      createPR(PR_REGION_NAME, true);
+      createPR("region2", PR_REGION_NAME, true);
+    }
+  };
+
+  private SerializableRunnable createMultipleColocatedChildPRs = new SerializableRunnable("create multiple child PRs") {
+    @Override
+    public void run() throws Exception {
+      createPR(PR_REGION_NAME, true);
+      for (int i = 2; i < numChildPRs +2; ++i) {
+        createPR("region" + i, PR_REGION_NAME, true);
+      }
+    }
+  };
+
+  private SerializableRunnable createPRColocationHierarchy = new SerializableRunnable("create PR colocation hierarchy") {
+    @Override
+    public void run() throws Exception {
+      createPR(PR_REGION_NAME, true);
+      createPR("region2", PR_REGION_NAME, true);
+      for (int i = 3; i < numChildPRGenerations +2; ++i) {
+        createPR("region" + i, "region" + (i-1), true);
+      }
+    }
+  };
+
+  private SerializableCallable createPRsMissingParentRegionThread = new SerializableCallable("createPRsMissingParentRegion") {
+    public Object call() throws Exception {
+      String exClass = "";
+      Exception ex = null;
+      try {
+        // Skip creation of first region - expect region2 creation to fail
+        //createPR(PR_REGION_NAME, true);
+        createPR("region2", PR_REGION_NAME, true);
+      } catch (Exception e) {
+        ex = e;
+        exClass = e.getClass().toString();
+      } finally {
+        return ex;
+      }
+    }
+  };
+
+  private SerializableCallable delayedCreatePRsMissingParentRegionThread = new SerializableCallable("delayedCreatePRsMissingParentRegion") {
+    public Object call() throws Exception {
+      String exClass = "";
+      Exception ex = null;
+      // To ensure that the targeted code paths in ColocationHelper.getColocatedRegion is taken, this 
+      // thread delays the attempted creation on the local member of colocated child region when parent doesn't exist.
+      // The delay is so that both parent and child regions will be created on another member and the PR root config 
+      // will have an entry for the parent region.
+      try {
+        await().pollDelay(50, TimeUnit.MILLISECONDS).atMost(100, TimeUnit.MILLISECONDS).until(() -> {return false;});
+      } catch (Exception e) {
+      }
+      try {
+        // Skip creation of first region - expect region2 creation to fail
+        //createPR(PR_REGION_NAME, true);
+        createPR("region2", PR_REGION_NAME, true);
+      } catch (Exception e) {
+        ex = e;
+        exClass = e.getClass().toString();
+      } finally {
+        return ex;
+      }
+    }
+  };
+
+  private SerializableCallable createPRsMissingChildRegionThread = new SerializableCallable("createPRsMissingChildRegion") {
+    Appender mockAppender;
+    ArgumentCaptor<LogEvent> loggingEventCaptor;
+
+    public Object call() throws Exception {
+      // Setup for capturing logger messages
+      Appender mockAppender = mock(Appender.class);
+      when(mockAppender.getName()).thenReturn("MockAppender");
+      when(mockAppender.isStarted()).thenReturn(true);
+      when(mockAppender.isStopped()).thenReturn(false);
+      Logger logger = (Logger) LogManager.getLogger(ColocationLogger.class);
+      logger.addAppender(mockAppender);
+      logger.setLevel(Level.WARN);
+      loggingEventCaptor = ArgumentCaptor.forClass(LogEvent.class);
+
+      // Logger interval may have been hooked by the test, so adjust test delays here
+      int logInterval = ColocationLogger.getLogInterval();
+      List<LogEvent> logEvents = Collections.emptyList();
+
+      AtomicBoolean isDone = new AtomicBoolean(false);
+      try {
+        createPR(PR_REGION_NAME, true);
+        // Let this thread continue running long enough for the missing region to be logged a couple times.
+        // Child regions do not get created by this thread. (1.5*logInterval < delay < 2*logInterval)
+        await().atMost((int)(1.75 * logInterval), TimeUnit.MILLISECONDS).until(() -> {verify(mockAppender, times(numExpectedLogMessages)).append(loggingEventCaptor.capture());});
+        //createPR("region2", PR_REGION_NAME, true);  // This child region is never created
+      } catch (Exception e) {
+        e.printStackTrace();
+      } finally {
+        logEvents = loggingEventCaptor.getAllValues();
+        assertEquals(String.format("Expected %d messages to be logged, got %d.", numExpectedLogMessages, logEvents.size()), numExpectedLogMessages, logEvents.size());
+        String logMsg = logEvents.get(0).getMessage().getFormattedMessage();
+        logger.removeAppender(mockAppender);
+        numExpectedLogMessages = 1;
+        return logMsg;
+      }
+    }
+  };
+
+  private SerializableCallable createPRsMissingChildRegionDelayedStartThread = new SerializableCallable("createPRsMissingChildRegionDelayedStart") {
+    Appender mockAppender;
+    ArgumentCaptor<LogEvent> loggingEventCaptor;
+
+    public Object call() throws Exception {
+      // Setup for capturing logger messages
+      mockAppender = mock(Appender.class);
+      when(mockAppender.getName()).thenReturn("MockAppender");
+      when(mockAppender.isStarted()).thenReturn(true);
+      when(mockAppender.isStopped()).thenReturn(false);
+      Logger logger = (Logger) LogManager.getLogger(ColocationLogger.class);
+      logger.addAppender(mockAppender);
+      logger.setLevel(Level.WARN);
+      loggingEventCaptor = ArgumentCaptor.forClass(LogEvent.class);
+
+      // Logger interval may have been hooked by the test, so adjust test delays here
+      int logInterval = ColocationLogger.getLogInterval();
+      List<LogEvent> logEvents = Collections.emptyList();
+
+      try {
+        createPR(PR_REGION_NAME, true);
+        // Delay creation of second (i.e child) region to see missing colocated region log message (logInterval/2 < delay < logInterval)
+        await().atMost((int)(.75 * logInterval), TimeUnit.MILLISECONDS).until(() -> {verify(mockAppender, times(1)).append(loggingEventCaptor.capture());});
+        logEvents = loggingEventCaptor.getAllValues();
+        createPR("region2", PR_REGION_NAME, true);
+        // Another delay before exiting the thread to make sure that missing region logging doesn't continue after 
+        // missing region is created (delay > logInterval)
+        await().atMost((int)(1.25 * logInterval), TimeUnit.MILLISECONDS).until(() -> {verifyNoMoreInteractions(mockAppender);});
+        fail("Unexpected missing colocated region log message");
+      } finally {
+        assertEquals(String.format("Expected %d messages to be logged, got %d.", numExpectedLogMessages, logEvents.size()), numExpectedLogMessages, logEvents.size());
+        String logMsg = logEvents.get(0).getMessage().getFormattedMessage();
+        logger.removeAppender(mockAppender);
+        numExpectedLogMessages = 1;
+        mockAppender = null;
+        return logMsg;
+      }
+    }
+  };
+
+  private SerializableCallable createPRsSequencedChildrenCreationThread = new SerializableCallable("createPRsSequencedChildrenCreation") {
+    Appender mockAppender;
+    ArgumentCaptor<LogEvent> loggingEventCaptor;
+
+    public Object call() throws Exception {
+      // Setup for capturing logger messages
+      mockAppender = mock(Appender.class);
+      when(mockAppender.getName()).thenReturn("MockAppender");
+      when(mockAppender.isStarted()).thenReturn(true);
+      when(mockAppender.isStopped()).thenReturn(false);
+      Logger logger = (Logger) LogManager.getLogger(ColocationLogger.class);
+      logger.addAppender(mockAppender);
+      logger.setLevel(Level.WARN);
+      loggingEventCaptor = ArgumentCaptor.forClass(LogEvent.class);
+
+      // Logger interval may have been hooked by the test, so adjust test delays here
+      int logInterval = ColocationLogger.getLogInterval();
+      List<LogEvent> logEvents = Collections.emptyList();
+      int numLogEvents = 0;
+
+      createPR(PR_REGION_NAME, true);
+      // Delay creation of child generation regions to see missing colocated region log message (logInterval/2 < delay < logInterval)
+      // parent region is generation 1, child region is generation 2, grandchild is 3, etc.
+      for (int generation = 2; generation < (numChildPRGenerations + 2); ++generation) {
+        String childPRName = "region" + generation;
+        String colocatedWithRegionName = generation == 2 ? PR_REGION_NAME : "region" + (generation - 1);
+        loggingEventCaptor = ArgumentCaptor.forClass(LogEvent.class);
+
+        // delay between starting generations of child regions
+        try {
+        await().atMost(delayForChildCreation, TimeUnit.MILLISECONDS).until(() -> {return false;});
+        } catch (ConditionTimeoutException e) {
+        }
+        // check for log messages
+        verify(mockAppender,  times((generation - 1) * generation / 2)).append(loggingEventCaptor.capture());
+        logEvents = loggingEventCaptor.getAllValues();
+        String logMsg = loggingEventCaptor.getValue().getMessage().getFormattedMessage();
+
+        // Start the child region
+        createPR(childPRName, colocatedWithRegionName, true);
+      }
+      // Another delay before exiting the thread to make sure that missing region logging doesn't continue after 
+      // all regions created (delay > logInterval)
+      String logMsg = "";
+          try {
+        await().atMost((int)(delayForChildCreation * 1.2), TimeUnit.MILLISECONDS).until(() -> {return false;});
+      } catch (ConditionTimeoutException e) {
+      } finally {
+        logEvents = loggingEventCaptor.getAllValues();
+        assertEquals(String.format("Expected warning messages to be logged."), numExpectedLogMessages, logEvents.size());
+        logMsg = logEvents.get(0).getMessage().getFormattedMessage();
+        logger.removeAppender(mockAppender);
+      }
+      numExpectedLogMessages = 1;
+      mockAppender = null;
+      return logMsg;
+    }
+  };
+
+  private SerializableCallable createMultipleColocatedChildPRsWithSequencedStart = new SerializableCallable("createPRsMultipleSequencedChildrenCreation") {
+    Appender mockAppender;
+    ArgumentCaptor<LogEvent> loggingEventCaptor;
+
+    public Object call() throws Exception {
+      // Setup for capturing logger messages
+      mockAppender = mock(Appender.class);
+      when(mockAppender.getName()).thenReturn("MockAppender");
+      when(mockAppender.isStarted()).thenReturn(true);
+      when(mockAppender.isStopped()).thenReturn(false);
+      Logger logger = (Logger) LogManager.getLogger(ColocationLogger.class);
+      logger.addAppender(mockAppender);
+      logger.setLevel(Level.WARN);
+      loggingEventCaptor = ArgumentCaptor.forClass(LogEvent.class);
+
+      // Logger interval may have been hooked by the test, so adjust test delays here
+      int logInterval = ColocationLogger.getLogInterval();
+      List<LogEvent> logEvents = Collections.emptyList();
+      int numLogEvents = 0;
+
+      createPR(PR_REGION_NAME, true);
+      // Delay creation of child generation regions to see missing colocated region log message (logInterval/2 < delay < logInterval)
+      for (int regionNum = 2; regionNum < (numChildPRs + 2); ++regionNum) {
+        String childPRName = "region" + regionNum;
+        loggingEventCaptor = ArgumentCaptor.forClass(LogEvent.class);
+
+        // delay between starting generations of child regions
+        try {
+          await().atMost(delayForChildCreation, TimeUnit.MILLISECONDS).until(() -> {return false;});
+        } catch (ConditionTimeoutException e) {
+        }
+        // check for log messages
+        verify(mockAppender,  times(regionNum - 1)).append(loggingEventCaptor.capture());
+        logEvents = loggingEventCaptor.getAllValues();
+        String logMsg = loggingEventCaptor.getValue().getMessage().getFormattedMessage();
+        numLogEvents = logEvents.size();
+        assertEquals("Expected warning messages to be logged.", regionNum - 1, numLogEvents);
+
+        // Start the child region
+        try {
+          createPR(childPRName, PR_REGION_NAME, true);
+        } catch (Exception e) {
+        }
+      }
+      // Another delay before exiting the thread to make sure that missing region logging doesn't continue after 
+      // all regions created (delay > logInterval)
+      try {
+        await().atMost((int)(delayForChildCreation * 1.2), TimeUnit.MILLISECONDS).until(() -> {return false;});
+      } catch (ConditionTimeoutException e) {
+      }
+      String logMsg;
+      try {
+        logEvents = loggingEventCaptor.getAllValues();
+        assertEquals(String.format("Expected warning messages to be logged."), numExpectedLogMessages, logEvents.size());
+        logMsg = logEvents.get(0).getMessage().getFormattedMessage();
+      } finally {
+        logger.removeAppender(mockAppender);
+        numExpectedLogMessages = 1;
+        mockAppender = null;
+      }
+      return logMsg;
+    }
+  };
+
+  private SerializableCallable createPRsMissingGrandchildRegionThread = new SerializableCallable("createPRsMissingChildRegion") {
+    Appender mockAppender;
+    ArgumentCaptor<LogEvent> loggingEventCaptor;
+
+    public Object call() throws Exception {
+      // Setup for capturing logger messages
+      Appender mockAppender = mock(Appender.class);
+      when(mockAppender.getName()).thenReturn("MockAppender");
+      when(mockAppender.isStarted()).thenReturn(true);
+      when(mockAppender.isStopped()).thenReturn(false);
+      Logger logger = (Logger) LogManager.getLogger(ColocationLogger.class);
+      logger.addAppender(mockAppender);
+      logger.setLevel(Level.WARN);
+      loggingEventCaptor = ArgumentCaptor.forClass(LogEvent.class);
+
+      // Logger interval may have been hooked by the test, so adjust test delays here
+      int logInterval = ColocationLogger.getLogInterval();
+      List<LogEvent> logEvents = Collections.emptyList();
+
+      try {
+        createPR(PR_REGION_NAME, true);
+        createPR("region2", PR_REGION_NAME, true);  // This child region is never created
+        // Let this thread continue running long enough for the missing region to be logged a couple times.
+        // Grandchild region does not get created by this thread. (1.5*logInterval < delay < 2*logInterval)
+        await().atMost((int)(1.75 * logInterval), TimeUnit.MILLISECONDS).until(() -> {verify(mockAppender, times(numExpectedLogMessages)).append(loggingEventCaptor.capture());});
+        //createPR("region3", PR_REGION_NAME, true);  // This child region is never created
+      } finally {
+        logEvents = loggingEventCaptor.getAllValues();
+        assertEquals(String.format("Expected %d messages to be logged, got %d.", numExpectedLogMessages, logEvents.size()), numExpectedLogMessages, logEvents.size());
+        String logMsg = logEvents.get(0).getMessage().getFormattedMessage();
+        logger.removeAppender(mockAppender);
+        numExpectedLogMessages = 1;
+        return logMsg;
+      }
+    }
+  };
+
+  private class ColocationLoggerIntervalSetter extends SerializableRunnable {
+    private int logInterval;
+
+    ColocationLoggerIntervalSetter(int newInterval) {
+      this.logInterval = newInterval;
+    }
+    @Override
+    public void run() throws Exception {
+      ColocationLogger.testhookSetLogInterval(logInterval);
+    }
+  }
+
+  private class ColocationLoggerIntervalResetter extends SerializableRunnable {
+    private int logInterval;
+
+    @Override
+    public void run() throws Exception {
+      ColocationLogger.testhookResetLogInterval();
+    }
+  }
+
+  private class ExpectedNumLogMessageSetter extends SerializableRunnable {
+    private int numMsgs;
+
+    ExpectedNumLogMessageSetter(int num) {
+      this.numMsgs = num;
+    }
+    @Override
+    public void run() throws Exception {
+      numExpectedLogMessages = numMsgs;
+    }
+  }
+
+  private class ExpectedNumLogMessageResetter extends SerializableRunnable {
+    private int numMsgs;
+
+    ExpectedNumLogMessageResetter() {
+      this.numMsgs = DEFAULT_NUM_EXPECTED_LOG_MESSAGES;
+    }
+    @Override
+    public void run() throws Exception {
+      numExpectedLogMessages = numMsgs;
+    }
+  }
+
+  private class NumChildPRsSetter extends SerializableRunnable {
+    private int numChildren;
+
+    NumChildPRsSetter(int num) {
+      this.numChildren = num;
+    }
+    @Override
+    public void run() throws Exception {
+      numChildPRs = numChildren;
+    }
+  }
+
+  private class NumChildPRGenerationsSetter extends SerializableRunnable {
+    private int numGenerations;
+
+    NumChildPRGenerationsSetter(int num) {
+      this.numGenerations = num;
+    }
+    @Override
+    public void run() throws Exception {
+      numChildPRGenerations = numGenerations;
+    }
+  }
+
+  private class DelayForChildCreationSetter extends SerializableRunnable {
+    private int delay;
+
+    DelayForChildCreationSetter(int millis) {
+      this.delay = millis;
+    }
+    @Override
+    public void run() throws Exception {
+      delayForChildCreation = delay;
+    }
+  }
+
+  /**
+   * Testing that missing colocated persistent PRs are logged as warning
+   */
+  @Test
+  public void testMissingColocatedParentPR() throws Throwable {
+    Host host = Host.getHost(0);
+    VM vm0 = host.getVM(0);
+    VM vm1 = host.getVM(1);
+
+    vm0.invoke(createPRsColocatedPairThread);
+    vm1.invoke(createPRsColocatedPairThread);
+
+    createData(vm0, 0, NUM_BUCKETS, "a");
+    createData(vm0, 0, NUM_BUCKETS, "b", "region2");
+
+    Set<Integer> vm0Buckets = getBucketList(vm0, PR_REGION_NAME);
+    assertFalse(vm0Buckets.isEmpty());
+    assertEquals(vm0Buckets, getBucketList(vm0, "region2"));
+    Set<Integer> vm1Buckets = getBucketList(vm1, PR_REGION_NAME);
+    assertEquals(vm1Buckets, getBucketList(vm1, "region2"));
+
+    closeCache(vm0);
+    closeCache(vm1);
+
+    Object remoteException = null;
+    remoteException = vm0.invoke(createPRsMissingParentRegionThread);
+    assertEquals("Expected IllegalState Exception for missing colocated parent region", IllegalStateException.class, remoteException.getClass());
+    assertTrue("Expected IllegalState Exception for missing colocated parent region", remoteException.toString().matches(
+        "java.lang.IllegalStateException: Region specified in 'colocated-with'.*"));
+  }
+
+  /**
+   * Testing that parent colocated persistent PRs only missing on local member throws exception 
+   */
+  @Test
+  public void testMissingColocatedParentPRWherePRConfigExists() throws Throwable {
+    Host host = Host.getHost(0);
+    VM vm0 = host.getVM(0);
+    VM vm1 = host.getVM(1);
+
+    vm0.invoke(createPRsColocatedPairThread);
+    vm1.invoke(createPRsColocatedPairThread);
+
+    createData(vm0, 0, NUM_BUCKETS, "a");
+    createData(vm0, 0, NUM_BUCKETS, "b", "region2");
+
+    Set<Integer> vm0Buckets = getBucketList(vm0, PR_REGION_NAME);
+    assertFalse(vm0Buckets.isEmpty());
+    assertEquals(vm0Buckets, getBucketList(vm0, "region2"));
+    Set<Integer> vm1Buckets = getBucketList(vm1, PR_REGION_NAME);
+    assertEquals(vm1Buckets, getBucketList(vm1, "region2"));
+
+    closeCache(vm0);
+    closeCache(vm1);
+
+    AsyncInvocation async0 = vm0.invokeAsync(createPRsColocatedPairThread);
+
+    Object logMsg = "";
+    Object remoteException = null;
+    AsyncInvocation async1 = vm1.invokeAsync(delayedCreatePRsMissingParentRegionThread);
+    remoteException = async1.get(MAX_WAIT, TimeUnit.MILLISECONDS);
+    assertEquals("Expected IllegalState Exception for missing colocated parent region", IllegalStateException.class, remoteException.getClass());
+    assertTrue("Expected IllegalState Exception for missing colocated parent region", remoteException.toString().matches(
+        "java.lang.IllegalStateException: Region specified in 'colocated-with'.*"));
+  }
+
+  /**
+   * Testing that missing colocated child persistent PRs are logged as warning
+   */
+  @Test
+  public void testMissingColocatedChildPRDueToDelayedStart() throws Throwable {
+    int loggerTestInterval = 4000; // millis
+    Host host = Host.getHost(0);
+    VM vm0 = host.getVM(0);
+    VM vm1 = host.getVM(1);
+
+    vm0.invoke(createPRsColocatedPairThread);
+    vm1.invoke(createPRsColocatedPairThread);
+
+    createData(vm0, 0, NUM_BUCKETS, "a");
+    createData(vm0, 0, NUM_BUCKETS, "b", "region2");
+
+    Set<Integer> vm0Buckets = getBucketList(vm0, PR_REGION_NAME);
+    assertFalse(vm0Buckets.isEmpty());
+    assertEquals(vm0Buckets, getBucketList(vm0, "region2"));
+    Set<Integer> vm1Buckets = getBucketList(vm1, PR_REGION_NAME);
+    assertEquals(vm1Buckets, getBucketList(vm1, "region2"));
+
+    closeCache(vm0);
+    closeCache(vm1);
+
+    vm0.invoke(new ColocationLoggerIntervalSetter(loggerTestInterval));
+    vm1.invoke(new ColocationLoggerIntervalSetter(loggerTestInterval));
+    vm0.invoke(new ExpectedNumLogMessageSetter(1));
+    vm1.invoke(new ExpectedNumLogMessageSetter(1));
+
+    Object logMsg = "";
+    AsyncInvocation async0 = vm0.invokeAsync(createPRsMissingChildRegionDelayedStartThread);
+    AsyncInvocation async1 = vm1.invokeAsync(createPRsMissingChildRegionDelayedStartThread);
+    logMsg = async1.get(MAX_WAIT, TimeUnit.MILLISECONDS);
+    vm0.invoke(new ExpectedNumLogMessageResetter());
+    vm1.invoke(new ExpectedNumLogMessageResetter());
+    vm0.invoke(new ColocationLoggerIntervalResetter());
+    vm1.invoke(new ColocationLoggerIntervalResetter());
+    assertTrue("Expected missing colocated region warning on remote. Got message \"" + logMsg + "\"",
+        logMsg.toString().matches(PATTERN_FOR_MISSING_CHILD_LOG));
+  }
+
+  /**
+   * Testing that missing colocated child persistent PRs are logged as warning
+   */
+  @Test
+  public void testMissingColocatedChildPR() throws Throwable {
+    int loggerTestInterval = 4000; // millis
+    Host host = Host.getHost(0);
+    VM vm0 = host.getVM(0);
+    VM vm1 = host.getVM(1);
+
+    vm0.invoke(createPRsColocatedPairThread);
+    vm1.invoke(createPRsColocatedPairThread);
+
+    createData(vm0, 0, NUM_BUCKETS, "a");
+    createData(vm0, 0, NUM_BUCKETS, "b", "region2");
+
+    Set<Integer> vm0Buckets = getBucketList(vm0, PR_REGION_NAME);
+    assertFalse(vm0Buckets.isEmpty());
+    assertEquals(vm0Buckets, getBucketList(vm0, "region2"));
+    Set<Integer> vm1Buckets = getBucketList(vm1, PR_REGION_NAME);
+    assertEquals(vm1Buckets, getBucketList(vm1, "region2"));
+
+    closeCache(vm0);
+    closeCache(vm1);
+
+    vm0.invoke(new ColocationLoggerIntervalSetter(loggerTestInterval));
+    vm1.invoke(new ColocationLoggerIntervalSetter(loggerTestInterval));
+    vm0.invoke(new ExpectedNumLogMessageSetter(2));
+    vm1.invoke(new ExpectedNumLogMessageSetter(2));
+
+    Object logMsg = "";
+    AsyncInvocation async0 = vm0.invokeAsync(createPRsMissingChildRegionThread);
+    AsyncInvocation async1 = vm1.invokeAsync(createPRsMissingChildRegionThread);
+    logMsg = async1.get(MAX_WAIT, TimeUnit.MILLISECONDS);
+    async0.get(MAX_WAIT, TimeUnit.MILLISECONDS);
+    vm0.invoke(new ExpectedNumLogMessageResetter());
+    vm1.invoke(new ExpectedNumLogMessageResetter());
+    vm0.invoke(new ColocationLoggerIntervalResetter());
+    vm1.invoke(new ColocationLoggerIntervalResetter());
+    assertTrue("Expected missing colocated region warning on remote. Got message \"" + logMsg + "\"",
+        logMsg.toString().matches(PATTERN_FOR_MISSING_CHILD_LOG));
+  }
+
+  /**
+   * Test that when there is more than one missing colocated child persistent PRs for a region all missing regions are logged
+   * in the warning. 
+   */
+  @Test
+  public void testMultipleColocatedChildPRsMissing() throws Throwable {
+    int loggerTestInterval = 4000; // millis
+    int numChildPRs = 2;
+    Host host = Host.getHost(0);
+    VM vm0 = host.getVM(0);
+    VM vm1 = host.getVM(1);
+
+    vm0.invoke(new NumChildPRsSetter(numChildPRs));
+    vm1.invoke(new NumChildPRsSetter(numChildPRs));
+    vm0.invoke(createMultipleColocatedChildPRs);
+    vm1.invoke(createMultipleColocatedChildPRs);
+
+    createData(vm0, 0, NUM_BUCKETS, "a");
+    createData(vm0, 0, NUM_BUCKETS, "b", "region2");
+    createData(vm0, 0, NUM_BUCKETS, "c", "region2");
+
+    Set<Integer> vm0Buckets = getBucketList(vm0, PR_REGION_NAME);
+    assertFalse(vm0Buckets.isEmpty());
+    Set<Integer> vm1Buckets = getBucketList(vm1, PR_REGION_NAME);
+    assertFalse(vm1Buckets.isEmpty());
+    for (int i = 2;i < numChildPRs + 2; ++i) {
+      String childName = "region" + i;
+      assertEquals(vm0Buckets, getBucketList(vm0, childName));
+      assertEquals(vm1Buckets, getBucketList(vm1, childName));
+    }
+
+    closeCache(vm0);
+    closeCache(vm1);
+
+    vm0.invoke(new ColocationLoggerIntervalSetter(loggerTestInterval));
+    vm1.invoke(new ColocationLoggerIntervalSetter(loggerTestInterval));
+    vm0.invoke(new ExpectedNumLogMessageSetter(2));
+    vm1.invoke(new ExpectedNumLogMessageSetter(2));
+
+    Object logMsg = "";
+    AsyncInvocation async0 = vm0.invokeAsync(createPRsMissingChildRegionThread);
+    AsyncInvocation async1 = vm1.invokeAsync(createPRsMissingChildRegionThread);
+    logMsg = async1.get(MAX_WAIT, TimeUnit.MILLISECONDS);
+    async0.get(MAX_WAIT, TimeUnit.MILLISECONDS);
+    vm0.invoke(new ExpectedNumLogMessageResetter());
+    vm1.invoke(new ExpectedNumLogMessageResetter());
+    vm0.invoke(new ColocationLoggerIntervalResetter());
+    vm1.invoke(new ColocationLoggerIntervalResetter());
+    assertTrue("Expected missing colocated region warning on remote. Got message \"" + logMsg + "\"",
+        logMsg.toString().matches(PATTERN_FOR_MISSING_CHILD_LOG));
+  }
+
+  /**
+   * Test that when there is more than one missing colocated child persistent PRs for a region all missing regions are logged
+   * in the warning. Verifies that as regions are created they no longer appear in the warning.
+   */
+  @Test
+  public void testMultipleColocatedChildPRsMissingWithSequencedStart() throws Throwable {
+    int loggerTestInterval = 4000; // millis
+    int numChildPRs = 2;
+    Host host = Host.getHost(0);
+    VM vm0 = host.getVM(0);
+    VM vm1 = host.getVM(1);
+
+    vm0.invoke(new NumChildPRsSetter(numChildPRs));
+    vm1.invoke(new NumChildPRsSetter(numChildPRs));
+    vm0.invoke(createMultipleColocatedChildPRs);
+    vm1.invoke(createMultipleColocatedChildPRs);
+
+    createData(vm0, 0, NUM_BUCKETS, "a");
+    createData(vm0, 0, NUM_BUCKETS, "b", "region2");
+    createData(vm0, 0, NUM_BUCKETS, "c", "region2");
+
+    Set<Integer> vm0Buckets = getBucketList(vm0, PR_REGION_NAME);
+    assertFalse(vm0Buckets.isEmpty());
+    Set<Integer> vm1Buckets = getBucketList(vm1, PR_REGION_NAME);
+    assertFalse(vm1Buckets.isEmpty());
+    for (int i = 2;i < numChildPRs + 2; ++i) {
+      String childName = "region" + i;
+      assertEquals(vm0Buckets, getBucketList(vm0, childName));
+      assertEquals(vm1Buckets, getBucketList(vm1, childName));
+    }
+
+    closeCache(vm0);
+    closeCache(vm1);
+
+    vm0.invoke(new ColocationLoggerIntervalSetter(loggerTestInterval));
+    vm1.invoke(new ColocationLoggerIntervalSetter(loggerTestInterval));
+    vm0.invoke(new ExpectedNumLogMessageSetter(2));
+    vm1.invoke(new ExpectedNumLogMessageSetter(2));
+    vm0.invoke(new DelayForChildCreationSetter((int)(loggerTestInterval)));
+    vm1.invoke(new DelayForChildCreationSetter((int)(loggerTestInterval)));
+
+    Object logMsg = "";
+    AsyncInvocation async0 = vm0.invokeAsync(createMultipleColocatedChildPRsWithSequencedStart);
+    AsyncInvocation async1 = vm1.invokeAsync(createMultipleColocatedChildPRsWithSequencedStart);
+    logMsg = async1.get(MAX_WAIT, TimeUnit.MILLISECONDS);
+    async0.get(MAX_WAIT, TimeUnit.MILLISECONDS);
+    vm0.invoke(new ExpectedNumLogMessageResetter());
+    vm1.invoke(new ExpectedNumLogMessageResetter());
+    vm0.invoke(new ColocationLoggerIntervalResetter());
+    vm1.invoke(new ColocationLoggerIntervalResetter());
+    assertTrue("Expected missing colocated region warning on remote. Got message \"" + logMsg + "\"",
+        logMsg.toString().matches(PATTERN_FOR_MISSING_CHILD_LOG));
+  }
+
+  /**
+   * Testing that all missing persistent PRs in a colocation hierarchy are logged as warnings
+   */
+  @Test
+  public void testHierarchyOfColocatedChildPRsMissing() throws Throwable {
+    int loggerTestInterval = 4000; // millis
+    int numChildGenerations = 2;
+    Host host = Host.getHost(0);
+    VM vm0 = host.getVM(0);
+    VM vm1 = host.getVM(1);
+
+    vm0.invoke(new NumChildPRGenerationsSetter(numChildGenerations));
+    vm1.invoke(new NumChildPRGenerationsSetter(numChildGenerations));
+    vm0.invoke(createPRColocationHierarchy);
+    vm1.invoke(createPRColocationHierarchy);
+
+    createData(vm0, 0, NUM_BUCKETS, "a");
+    createData(vm0, 0, NUM_BUCKETS, "b", "region2");
+    createData(vm0, 0, NUM_BUCKETS, "c", "region3");
+
+    Set<Integer> vm0Buckets = getBucketList(vm0, PR_REGION_NAME);
+    assertFalse(vm0Buckets.isEmpty());
+    Set<Integer> vm1Buckets = getBucketList(vm1, PR_REGION_NAME);
+    assertFalse(vm1Buckets.isEmpty());
+    for (int i = 2;i < numChildGenerations + 2; ++i) {
+      String childName = "region" + i;
+      assertEquals(vm0Buckets, getBucketList(vm0, childName));
+      assertEquals(vm1Buckets, getBucketList(vm1, childName));
+    }
+
+    closeCache(vm0);
+    closeCache(vm1);
+
+    vm0.invoke(new ColocationLoggerIntervalSetter(loggerTestInterval));
+    vm1.invoke(new ColocationLoggerIntervalSetter(loggerTestInterval));
+    // Expected warning logs only on the child region, because without the child there's nothing known about the remaining hierarchy
+    vm0.invoke(new ExpectedNumLogMessageSetter(numChildGenerations));
+    vm1.invoke(new ExpectedNumLogMessageSetter(numChildGenerations));
+
+    Object logMsg = "";
+    AsyncInvocation async0 = vm0.invokeAsync(createPRsMissingChildRegionThread);
+    AsyncInvocation async1 = vm1.invokeAsync(createPRsMissingChildRegionThread);
+    logMsg = async1.get(MAX_WAIT, TimeUnit.MILLISECONDS);
+    async0.get(MAX_WAIT, TimeUnit.MILLISECONDS);
+    vm0.invoke(new ExpectedNumLogMessageResetter());
+    vm1.invoke(new ExpectedNumLogMessageResetter());
+    vm0.invoke(new ColocationLoggerIntervalResetter());
+    vm1.invoke(new ColocationLoggerIntervalResetter());
+    assertTrue("Expected missing colocated region warning on remote. Got message \"" + logMsg + "\"",
+        logMsg.toString().matches(PATTERN_FOR_MISSING_CHILD_LOG));
+  }
+
+  /**
+   * Testing that all missing persistent PRs in a colocation hierarchy are logged as warnings
+   */
+  @Test
+  public void testHierarchyOfColocatedChildPRsMissingGrandchild() throws Throwable {
+    int loggerTestInterval = 4000; // millis
+    int numChildGenerations = 3;
+    Host host = Host.getHost(0);
+    VM vm0 = host.getVM(0);
+    VM vm1 = host.getVM(1);
+
+    vm0.invoke(new NumChildPRGenerationsSetter(numChildGenerations));
+    vm1.invoke(new NumChildPRGenerationsSetter(numChildGenerations));
+    vm0.invoke(createPRColocationHierarchy);
+    vm1.invoke(createPRColocationHierarchy);
+
+    createData(vm0, 0, NUM_BUCKETS, "a");
+    createData(vm0, 0, NUM_BUCKETS, "b", "region2");
+    createData(vm0, 0, NUM_BUCKETS, "c", "region3");
+
+    Set<Integer> vm0Buckets = getBucketList(vm0, PR_REGION_NAME);
+    assertFalse(vm0Buckets.isEmpty());
+    Set<Integer> vm1Buckets = getBucketList(vm1, PR_REGION_NAME);
+    assertFalse(vm1Buckets.isEmpty());
+    for (int i = 2;i < numChildGenerations + 2; ++i) {
+      String childName = "region" + i;
+      assertEquals(vm0Buckets, getBucketList(vm0, childName));
+      assertEquals(vm1Buckets, getBucketList(vm1, childName));
+    }
+
+    closeCache(vm0);
+    closeCache(vm1);
+
+    vm0.invoke(new ColocationLoggerIntervalSetter(loggerTestInterval));
+    vm1.invoke(new ColocationLoggerIntervalSetter(loggerTestInterval));
+    // Expected warning logs only on the child region, because without the child there's nothing known about the remaining hierarchy
+    vm0.invoke(new ExpectedNumLogMessageSetter(numChildGenerations * (numChildGenerations + 1) / 2));
+    vm1.invoke(new ExpectedNumLogMessageSetter(numChildGenerations * (numChildGenerations + 1) / 2));
+    vm0.invoke(new DelayForChildCreationSetter((int)(loggerTestInterval)));
+    vm1.invoke(new DelayForChildCreationSetter((int)(loggerTestInterval)));
+
+    Object logMsg = "";
+    AsyncInvocation async0 = vm0.invokeAsync(createPRsSequencedChildrenCreationThread);
+    AsyncInvocation async1 = vm1.invokeAsync(createPRsSequencedChildrenCreationThread);
+    logMsg = async1.get(MAX_WAIT, TimeUnit.MILLISECONDS);
+    async0.get(MAX_WAIT, TimeUnit.MILLISECONDS);
+    vm0.invoke(new ExpectedNumLogMessageResetter());
+    vm1.invoke(new ExpectedNumLogMessageResetter());
+    vm0.invoke(new ColocationLoggerIntervalResetter());
+    vm1.invoke(new ColocationLoggerIntervalResetter());
+    System.out.println(logMsg);
+    assertTrue("Expected missing colocated region warning on remote. Got message \"" + logMsg + "\"",
+        logMsg.toString().matches(PATTERN_FOR_MISSING_CHILD_LOG));
+  }
+
+  private SerializableRunnable createPRColocationTree = new SerializableRunnable("create PR colocation hierarchy") {
+    @Override
+    public void run() throws Exception {
+      createPR("Parent", true);
+      createPR("Gen1_C1", "Parent", true);
+      createPR("Gen1_C2", "Parent", true);
+      createPR("Gen2_C1_1", "Gen1_C1", true);
+      createPR("Gen2_C1_2", "Gen1_C1", true);
+      createPR("Gen2_C2_1", "Gen1_C2", true);
+      createPR("Gen2_C2_2", "Gen1_C2", true);
+    }
+  };
+
+  /**
+   * The colocation tree has the regions started in a specific order so that the logging is predictable.
+   * For each entry in the list, the array values are:
+   * <pre> 
+   *   [0] - the region name
+   *   [1] - the name of that region's parent
+   *   [2] - the number of warnings that will be logged after the region is created (1 warning for
+   *         each region in the tree that exists that still has 1 or more missing children.)
+   * </pre>
+   */
+  private static final List<Object[]> CHILD_REGION_RESTART_ORDER = new ArrayList<Object[]>();
+  static {
+    CHILD_REGION_RESTART_ORDER.add(new Object[]{"Gen1_C1", "Parent", 2});
+    CHILD_REGION_RESTART_ORDER.add(new Object[]{"Gen2_C1_1", "Gen1_C1", 2});
+    CHILD_REGION_RESTART_ORDER.add(new Object[]{"Gen1_C2", "Parent", 3});
+    CHILD_REGION_RESTART_ORDER.add(new Object[]{"Gen2_C1_2", "Gen1_C1", 2});
+    CHILD_REGION_RESTART_ORDER.add(new Object[]{"Gen2_C2_1", "Gen1_C2", 2});
+    CHILD_REGION_RESTART_ORDER.add(new Object[]{"Gen2_C2_2", "Gen1_C2", 0});
+  }
+
+  private SerializableCallable createPRsSequencedColocationTreeCreationThread = new SerializableCallable("createPRsSequencedColocationTreeCreation") {
+    Appender mockAppender;
+    ArgumentCaptor<LogEvent> loggingEventCaptor;
+
+    public Object call() throws Exception {
+      // Setup for capturing logger messages
+      mockAppender = mock(Appender.class);
+      when(mockAppender.getName()).thenReturn("MockAppender");
+      when(mockAppender.isStarted()).thenReturn(true);
+      when(mockAppender.isStopped()).thenReturn(false);
+      Logger logger = (Logger) LogManager.getLogger(ColocationLogger.class);
+      logger.addAppender(mockAppender);
+      logger.setLevel(Level.WARN);
+      loggingEventCaptor = ArgumentCaptor.forClass(LogEvent.class);
+
+      // Logger interval may have been hooked by the test, so adjust test delays here
+      int logInterval = ColocationLogger.getLogInterval();
+      List<LogEvent> logEvents = Collections.emptyList();
+      int nLogEvents = 0;
+      int nExpectedLogs = 1;
+
+      createPR("Parent", true);
+      // Delay creation of descendant regions in the hierarchy to see missing colocated region 
+      // log messages (logInterval/2 < delay < logInterval)
+      for (Object[] regionInfo:CHILD_REGION_RESTART_ORDER) {
+        loggingEventCaptor = ArgumentCaptor.forClass(LogEvent.class);
+        String childPRName = (String) regionInfo[0];
+        String colocatedWithRegionName = (String) regionInfo[1];
+        int nLogsAfterRegionCreation = (int) regionInfo[2];
+
+        // delay between starting generations of child regions
+        try {
+        await().atMost(delayForChildCreation, TimeUnit.MILLISECONDS).until(() -> {return false;});
+        } catch (ConditionTimeoutException e) {
+        }
+        // check for log messages that occurred during the delay
+        verify(mockAppender,  times(nExpectedLogs)).append(loggingEventCaptor.capture());
+        nExpectedLogs += nLogsAfterRegionCreation;
+        logEvents = loggingEventCaptor.getAllValues();
+        String logMsg = loggingEventCaptor.getValue().getMessage().getFormattedMessage();
+
+        // Finally start the next child region
+        createPR(childPRName, colocatedWithRegionName, true);
+      }
+      // Another delay before exiting the thread to make sure that missing region logging doesn't continue after 
+      // all regions created (delay > logInterval)
+      String logMsg = "";
+          try {
+        await().atMost((int)(delayForChildCreation * 1.2), TimeUnit.MILLISECONDS).until(() -> {return false;});
+      } catch (ConditionTimeoutException e) {
+      } finally {
+        logEvents = loggingEventCaptor.getAllValues();
+        assertEquals(String.format("Expected warning messages to be logged."), nExpectedLogs, logEvents.size());
+        logMsg = logEvents.get(0).getMessage().getFormattedMessage();
+        logger.removeAppender(mockAppender);
+      }
+      numExpectedLogMessages = 1;
+      mockAppender = null;
+      return logMsg;
+    }
+  };
+
+  /**
+   * Testing that all missing persistent PRs in a colocation tree hierarchy are logged as warnings.
+   * This test is a combines the "multiple children" and "hierarchy of children" tests.
+   * This is the colocation tree for this test
+   * <pre>
+   *                  Parent
+   *                /         \
+   *             /               \
+   *         Gen1_C1            Gen1_C2
+   *         /    \              /    \
+   *  Gen2_C1_1  Gen2_C1_2  Gen2_C2_1  Gen2_C2_2
+   * </pre>
+   */
+  @Test
+  public void testFullTreeOfColocatedChildPRsWithMissingRegions() throws Throwable {
+    int loggerTestInterval = 4000; // millis
+    int numChildPRs = 2;
+    int numChildGenerations = 2;
+    Host host = Host.getHost(0);
+    VM vm0 = host.getVM(0);
+    VM vm1 = host.getVM(1);
+
+    vm0.invoke(createPRColocationTree);
+    vm1.invoke(createPRColocationTree);
+
+    createData(vm0, 0, NUM_BUCKETS, "a", "Parent");
+    createData(vm0, 0, NUM_BUCKETS, "b", "Gen1_C1");
+    createData(vm0, 0, NUM_BUCKETS, "c", "Gen1_C2");
+    createData(vm0, 0, NUM_BUCKETS, "c", "Gen2_C1_1");
+    createData(vm0, 0, NUM_BUCKETS, "c", "Gen2_C1_2");
+    createData(vm0, 0, NUM_BUCKETS, "c", "Gen2_C2_1");
+    createData(vm0, 0, NUM_BUCKETS, "c", "Gen2_C2_2");
+
+    Set<Integer> vm0Buckets = getBucketList(vm0, "Parent");
+    assertFalse(vm0Buckets.isEmpty());
+    Set<Integer> vm1Buckets = getBucketList(vm1, "Parent");
+    assertFalse(vm1Buckets.isEmpty());
+    for (String region:new String[]{"Gen1_C1", "Gen1_C2", "Gen2_C1_1", "Gen2_C1_2", "Gen2_C2_1", "Gen2_C2_2"}) {
+      assertEquals(vm0Buckets, getBucketList(vm0, region));
+      assertEquals(vm1Buckets, getBucketList(vm1, region));
+    }
+
+    closeCache(vm0);
+    closeCache(vm1);
+
+    vm0.invoke(new ColocationLoggerIntervalSetter(loggerTestInterval));
+    vm1.invoke(new ColocationLoggerIntervalSetter(loggerTestInterval));
+    vm0.invoke(new DelayForChildCreationSetter((int)(loggerTestInterval)));
+    vm1.invoke(new DelayForChildCreationSetter((int)(loggerTestInterval)));
+
+    Object logMsg = "";
+    AsyncInvocation async0 = vm0.invokeAsync(createPRsSequencedColocationTreeCreationThread);
+    AsyncInvocation async1 = vm1.invokeAsync(createPRsSequencedColocationTreeCreationThread);
+    logMsg = async1.get(MAX_WAIT, TimeUnit.MILLISECONDS);
+    async0.get(MAX_WAIT, TimeUnit.MILLISECONDS);
+    vm0.invoke(new ColocationLoggerIntervalResetter());
+    vm1.invoke(new ColocationLoggerIntervalResetter());
+    // Expected warning logs only on the child region, because without the child there's nothing known about the remaining hierarchy
+    assertTrue("Expected missing colocated region warning on remote. Got message \"" + logMsg + "\"",
+        logMsg.toString().matches(PATTERN_FOR_MISSING_CHILD_LOG));
+  }
+
   /**
    * Testing what happens we we recreate colocated persistent PRs by creating
    * one PR everywhere and then the other PR everywhere.
@@ -1580,7 +2551,7 @@ public class PersistentColocatedPartitionedRegionDUnitTest extends PersistentPar
       }
     });
   }
-  
+
   private static class PRObserver extends PartitionedRegionObserverAdapter {
     private CountDownLatch rebalanceDone = new CountDownLatch(1);
     private CountDownLatch bucketCreateStarted  = new CountDownLatch(3);

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/da29fb36/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/partitioned/PersistentPartitionedRegionTestBase.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/partitioned/PersistentPartitionedRegionTestBase.java b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/partitioned/PersistentPartitionedRegionTestBase.java
index 692378c..a6411ed 100644
--- a/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/partitioned/PersistentPartitionedRegionTestBase.java
+++ b/geode-core/src/test/java/com/gemstone/gemfire/internal/cache/partitioned/PersistentPartitionedRegionTestBase.java
@@ -239,10 +239,14 @@ public abstract class PersistentPartitionedRegionTestBase extends JUnit4CacheTes
   }
 
   protected void closePR(VM vm0) {
-    SerializableRunnable close = new SerializableRunnable("Close Cache") {
+    closePR(vm0, PR_REGION_NAME);
+  }
+
+  protected void closePR(VM vm0, String regionName) {
+    SerializableRunnable close = new SerializableRunnable("Close PR") {
       public void run() {
         Cache cache = getCache();
-        Region region = cache.getRegion(PR_REGION_NAME);
+        Region region = cache.getRegion(regionName);
         region.close();
       }
     };
@@ -251,10 +255,14 @@ public abstract class PersistentPartitionedRegionTestBase extends JUnit4CacheTes
   }
 
   protected void destroyPR(VM vm0) {
-    SerializableRunnable destroy = new SerializableRunnable() {
+    destroyPR(vm0,PR_REGION_NAME);
+  }
+
+  protected void destroyPR(VM vm0, String regionName) {
+    SerializableRunnable destroy = new SerializableRunnable("Destroy PR") {
       public void run() {
         Cache cache = getCache();
-        Region region = cache.getRegion(PR_REGION_NAME);
+        Region region = cache.getRegion(regionName);
         region.localDestroyRegion();
       }
     };
@@ -490,6 +498,7 @@ public abstract class PersistentPartitionedRegionTestBase extends JUnit4CacheTes
       }
     };
     
+    vm.invoke(getBuckets);
   }
   
   protected Set<Integer> getPrimaryBucketList(VM vm0) {


Mime
View raw message