geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kl...@apache.org
Subject [02/16] incubator-geode git commit: Moving a distributed lock service unit test to open-source.
Date Tue, 10 May 2016 21:17:03 GMT
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2c148caa/geode-core/src/test/java/com/gemstone/gemfire/distributed/DistributedLockServiceDUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/com/gemstone/gemfire/distributed/DistributedLockServiceDUnitTest.java b/geode-core/src/test/java/com/gemstone/gemfire/distributed/DistributedLockServiceDUnitTest.java
new file mode 100755
index 0000000..42c3f01
--- /dev/null
+++ b/geode-core/src/test/java/com/gemstone/gemfire/distributed/DistributedLockServiceDUnitTest.java
@@ -0,0 +1,3246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.gemstone.gemfire.distributed;
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import junit.framework.Assert;
+import junit.framework.AssertionFailedError;
+
+import com.gemstone.gemfire.SystemFailure;
+import com.gemstone.gemfire.distributed.internal.DistributionManager;
+import com.gemstone.gemfire.distributed.internal.DistributionMessage;
+import com.gemstone.gemfire.distributed.internal.DistributionMessageObserver;
+import com.gemstone.gemfire.distributed.internal.locks.DLockGrantor;
+import com.gemstone.gemfire.distributed.internal.locks.DLockRemoteToken;
+import com.gemstone.gemfire.distributed.internal.locks.DLockRequestProcessor;
+import com.gemstone.gemfire.distributed.internal.locks.DLockRequestProcessor.DLockRequestMessage;
+import com.gemstone.gemfire.distributed.internal.locks.DLockRequestProcessor.DLockResponseMessage;
+import com.gemstone.gemfire.distributed.internal.locks.DLockService;
+import com.gemstone.gemfire.distributed.internal.locks.DLockToken;
+import com.gemstone.gemfire.distributed.internal.locks.RemoteThread;
+import com.gemstone.gemfire.distributed.internal.membership.InternalDistributedMember;
+import com.gemstone.gemfire.internal.util.StopWatch;
+import com.gemstone.gemfire.test.dunit.AsyncInvocation;
+import com.gemstone.gemfire.test.dunit.DistributedTestCase;
+import com.gemstone.gemfire.test.dunit.Host;
+import com.gemstone.gemfire.test.dunit.Invoke;
+import com.gemstone.gemfire.test.dunit.LogWriterUtils;
+import com.gemstone.gemfire.test.dunit.RMIException;
+import com.gemstone.gemfire.test.dunit.SerializableCallable;
+import com.gemstone.gemfire.test.dunit.SerializableRunnable;
+import com.gemstone.gemfire.test.dunit.ThreadUtils;
+import com.gemstone.gemfire.test.dunit.VM;
+import com.gemstone.gemfire.test.dunit.Wait;
+import com.gemstone.gemfire.test.dunit.WaitCriterion;
+
+/**
+ * This class tests distributed ownership via the DistributedLockService api.
+ */
+public class DistributedLockServiceDUnitTest extends DistributedTestCase {
+  
+	protected static DistributedSystem dlstSystem;
+  private static DistributedLockBlackboard blackboard;
+  protected static Object monitor = new Object();
+
+  private int hits = 0;
+  private int completes = 0;
+  private boolean done;
+  private boolean got;
+
+
+  public DistributedLockServiceDUnitTest(String name) {
+    super(name);
+    if (blackboard == null) {
+      try {
+        blackboard = DistributedLockBlackboardImpl.getInstance();
+      } catch (Exception e) {
+        throw new RuntimeException("initialization error", e);
+      }
+    }
+  }
+  
+  /////////// Test lifecycle //////////
+ 
+  public static void caseSetUp() throws Exception {
+    disconnectAllFromDS();
+  }
+  
+  public static void caseTearDown() throws Exception {
+    disconnectAllFromDS();
+  }
+  
+  /**
+   * Returns a previously created (or new, if this is the first
+   * time this method is called in this VM) distributed system
+   * which is somewhat configurable via hydra test parameters.
+   */
+  @Override
+  public final void postSetUp() throws Exception {
+    // Create a DistributedSystem in every VM
+    connectDistributedSystem();
+
+    for (int h = 0; h < Host.getHostCount(); h++) {
+      Host host = Host.getHost(h);
+
+      for (int v = 0; v < host.getVMCount(); v++) {
+        host.getVM(v).invoke(
+          DistributedLockServiceDUnitTest.class, "connectDistributedSystem", null);
+      }
+    }
+  }
+
+  @Override
+  public final void preTearDown() throws Exception {
+    Invoke.invokeInEveryVM(DistributedLockServiceDUnitTest.class,
+                    "destroyAllDLockServices"); 
+//    invokeInEveryVM(DistributedLockServiceDUnitTest.class,
+//                    "remoteDumpAllDLockServices"); 
+                    
+    //InternalDistributedLockService.destroyAll();
+    
+//     // Disconnects the DistributedSystem in every VM - since
+//     // each test randomly chooses whether shared memory is used
+//     disconnectAllFromDS();
+    
+    this.lockGrantor = null;
+  }
+  
+  public static void destroyAllDLockServices() {
+    DLockService.destroyAll();
+    dlstSystem = null;
+  }
+
+  public static void remoteDumpAllDLockServices() {
+    DLockService.dumpAllServices();
+  }
+
+  ///////// Remote setup/teardown support
+
+  /**
+   * Connects a DistributedSystem, saves it in static variable "system"
+   */
+  protected static void connectDistributedSystem() {
+    dlstSystem = (new DistributedLockServiceDUnitTest("dummy")).getSystem();
+  }
+
+  /////////  Public test methods
+  
+  public void testBasic() {
+    String serviceName = getUniqueName();
+    String objectName = "object";
+    
+    // Create service
+    DistributedLockService service = DistributedLockService.create(serviceName, dlstSystem);
+    
+    // Not locked initially
+    assertFalse(service.isHeldByCurrentThread(objectName));
+
+    // Get lock
+    assertTrue(service.lock(objectName, 3000, -1));
+    assertTrue(service.isHeldByCurrentThread(objectName));
+    assertTrue(service.lock(objectName, 3000, -1));
+    assertTrue(service.isHeldByCurrentThread(objectName));
+    
+    // Release lock
+    service.unlock(objectName);
+    assertTrue(service.isHeldByCurrentThread(objectName));
+    service.unlock(objectName);
+    assertFalse(service.isHeldByCurrentThread(objectName));
+    
+    // Destroy service
+    DistributedLockService.destroy(serviceName);
+  }
+  
+  public void testCreateDestroy() throws Exception {
+    final String serviceName = getUniqueName();
+    final String abc = "abc";
+    
+    // create and destroy dls
+    assertNull(DistributedLockService.getServiceNamed(serviceName));
+    DistributedLockService service = DistributedLockService.create(serviceName, getSystem());
+    assertSame(service, DistributedLockService.getServiceNamed(serviceName));
+    DistributedLockService.destroy(serviceName);
+    
+    // assert attempt to use dls throws LockServiceDestroyedException
+    try {
+      service.lock(abc, -1, -1);
+      fail("didn't get LockServiceDestroyedException");
+    } catch (LockServiceDestroyedException ex) {
+    }
+    
+    // assert that destroyed dls is no longer available
+    service = DistributedLockService.getServiceNamed(serviceName);
+    assertNull("" + service, service);
+    
+    // recreate the dls
+    service = DistributedLockService.create(serviceName, getSystem());
+    assertTrue(!((DLockService)service).isDestroyed());
+    ((DLockService)service).checkDestroyed();
+    
+    // get the same dls from another thread and hold a lock
+    Thread thread = new Thread(new Runnable() {
+      public void run() {
+        DistributedLockService dls = 
+            DistributedLockService.getServiceNamed(serviceName);
+        assertTrue(!((DLockService)dls).isDestroyed());
+        ((DLockService)dls).checkDestroyed();
+        dls.lock(abc, -1, -1); // get lock on abc and hold it
+      }
+    });
+    thread.start();
+    ThreadUtils.join(thread, 30 * 1000);
+    
+    // start a new thread to wait for lock on abc
+    AsyncInvocation remoteWaitingThread = 
+      Host.getHost(0).getVM(0).invokeAsync(new SerializableRunnable() {
+      public void run() {
+        DistributedLockService dls = 
+            DistributedLockService.create(serviceName, getSystem());
+        try {
+          dls.lock(abc, -1, -1); // waiting to get lock abc
+          fail("remoteWaitingThread got lock after dls destroyed");
+        }
+        catch (LockServiceDestroyedException expected) {
+          return;
+        }
+        fail("remoteWaitingThread lock failed to throw LockServiceDestroyedException");
+      }
+    });
+    
+    // loop will handle race condition with 1 sec sleep and retry
+    int retry = 10;
+    for (int i = 0; i < retry; i++) {
+      try {
+        // destroy DLS and free up remoteWaitingThread
+        Host.getHost(0).getVM(0).invoke(new SerializableRunnable() {
+          public void run() {
+            DistributedLockService.destroy(serviceName);
+          }
+        });
+      }
+      catch (RMIException e) {
+        // race condition: remoteWaitingThread probably hasn't created DLS yet
+        if (i < retry && e.getCause() instanceof IllegalArgumentException) {
+          sleep(1000);
+          continue;
+        }
+        else {
+          throw e;
+        }
+      }
+      break; // completed so break out of loop
+    }
+    
+    DistributedLockService.destroy(serviceName);
+    
+    // make sure remoteWaitingThread stopped waiting and threw LockServiceDestroyedException
+    ThreadUtils.join(remoteWaitingThread, 10 * 1000);
+    if (remoteWaitingThread.exceptionOccurred()) {
+      Throwable e = remoteWaitingThread.getException();
+      com.gemstone.gemfire.test.dunit.Assert.fail(e.getMessage(), e);
+    }
+    
+    // make sure LockServiceDestroyedException is thrown
+    try {
+      service.lock(abc, -1, -1);
+      fail("didn't get LockServiceDestroyedException");
+    } catch (LockServiceDestroyedException ex) {
+    }
+    
+    // make sure getServiceNamed returns null
+    service = DistributedLockService.getServiceNamed(serviceName);
+    assertNull("" + service, service);
+  }
+
+  protected static DistributedLockService dls_testFairness;
+  protected static int count_testFairness[] = new int[16];
+  protected static volatile boolean stop_testFairness;
+  protected static volatile boolean[] done_testFairness = new boolean[16];
+  static { Arrays.fill(done_testFairness, true); }
+  public void testFairness() throws Exception {
+    final String serviceName = "testFairness_" + getUniqueName();
+    final Object lock = "lock";
+    
+    // get the lock and hold it until all threads are ready to go
+    DistributedLockService service = 
+        DistributedLockService.create(serviceName, dlstSystem);
+    assertTrue(service.lock(lock, -1, -1));
+    
+    final int[] vmThreads = new int[] { 1, 4, 8, 16 };
+    forNumVMsInvoke(vmThreads.length,
+                    "remoteCreateService", 
+                    new Object[] { serviceName });
+    sleep(100);
+
+    // line up threads for the fairness race...
+    for (int i = 0; i < vmThreads.length; i++) {
+      final int vm = i;
+      LogWriterUtils.getLogWriter().info("[testFairness] lining up " + vmThreads[vm] + 
+                          " threads in vm " + vm); 
+                          
+      for (int j = 0; j < vmThreads[vm]; j++) {
+        final int thread = j;
+        /*getLogWriter().info("[testFairness] setting up thread " + thread +
+                            " in vm " + vm);*/
+        
+        Host.getHost(0).getVM(vm).invokeAsync(new SerializableRunnable() {
+          public void run() {
+            // lock, inc count, and unlock until stop_testFairness is set true
+            try {
+              done_testFairness[thread] = false;
+              dls_testFairness = DistributedLockService.getServiceNamed(serviceName);
+              while (!stop_testFairness) {
+                assertTrue(dls_testFairness.lock(lock, -1, -1));
+                count_testFairness[thread]++;
+                dls_testFairness.unlock(lock);
+              }
+              done_testFairness[thread] = true;
+            }
+            catch (VirtualMachineError e) {
+              SystemFailure.initiateFailure(e);
+              throw e;
+            }
+            catch (Throwable t) {
+              LogWriterUtils.getLogWriter().warning(t);
+              fail(t.getMessage());
+            }
+          }
+        });
+      }
+    }
+    sleep(500); // 500 ms
+    
+    // start the race!
+    service.unlock(lock);
+    sleep(1000 * 5); // 5 seconds
+    assertTrue(service.lock(lock, -1, -1));
+    
+    // stop the race...
+    for (int i = 0; i < vmThreads.length; i++) {
+      final int vm = i;
+      Host.getHost(0).getVM(vm).invoke(new SerializableRunnable() {
+        public void run() {
+          stop_testFairness = true;
+        }
+      });
+    }
+    service.unlock(lock);
+    for (int i = 0; i < vmThreads.length; i++) {
+      final int vm = i;
+      Host.getHost(0).getVM(vm).invoke(new SerializableRunnable() {
+        public void run() {
+          try {
+            boolean testIsDone = false;
+            while (!stop_testFairness || !testIsDone) {
+              testIsDone = true;
+              for (int i2 = 0; i2 < done_testFairness.length; i2++) {
+                if (!done_testFairness[i2]) testIsDone = false;
+              }
+            }
+            DistributedLockService.destroy(serviceName);
+          }
+          catch (VirtualMachineError e) {
+            SystemFailure.initiateFailure(e);
+            throw e;
+          }
+          catch (Throwable t) {
+            fail(t.getMessage());
+          }
+        }
+      });
+    }
+    
+    // calc total locks granted...
+    int totalLocks = 0;
+    int minLocks = Integer.MAX_VALUE;
+    int maxLocks = 0;
+
+    // add up total locks across all vms and threads...
+    int numThreads = 0;
+    for (int i = 0; i < vmThreads.length; i++) {
+      final int vm = i;
+      for (int j = 0; j < vmThreads[vm]; j++) {
+        final int thread = j;
+        Integer count = (Integer)Host.getHost(0).getVM(vm).invoke(() -> DistributedLockServiceDUnitTest.get_count_testFairness( new Integer(thread) ));
+        int numLocks = count.intValue();
+        if (numLocks < minLocks) minLocks = numLocks;
+        if (numLocks > maxLocks) maxLocks = numLocks;
+        totalLocks = totalLocks + numLocks;
+        numThreads++;
+      }
+    }
+
+    LogWriterUtils.getLogWriter().info("[testFairness] totalLocks=" + totalLocks + 
+                        " minLocks=" + minLocks +
+                        " maxLocks=" + maxLocks);
+
+    int expectedLocks = (totalLocks / numThreads) + 1;
+    
+    int deviation = (int)(expectedLocks * 0.3);
+    int lowThreshold = expectedLocks - deviation;
+    int highThreshold = expectedLocks + deviation;
+
+    LogWriterUtils.getLogWriter().info("[testFairness] deviation=" + deviation +
+                        " expectedLocks=" + expectedLocks + 
+                        " lowThreshold=" + lowThreshold +
+                        " highThreshold=" + highThreshold);
+                        
+    assertTrue("minLocks is less than lowThreshold",
+               minLocks >= lowThreshold);
+    assertTrue("maxLocks is greater than highThreshold",
+               maxLocks <= highThreshold);
+  }
+  
+  /**
+   * Accessed by reflection.  DO NOT REMOVE
+   * @param i
+   * @return
+   */
+  public static Integer get_count_testFairness(Integer i) {
+    return new Integer(count_testFairness[i.intValue()]);
+  }
+  
+  public void testOneGetsAndOthersTimeOut() throws Exception {
+    doOneGetsAndOthersTimeOut(1, 1);
+//     doOneGetsAndOthersTimeOut(2, 2);
+//     doOneGetsAndOthersTimeOut(3, 2);
+    doOneGetsAndOthersTimeOut(4, 3);
+  }
+  
+  private InternalDistributedMember lockGrantor;
+  private synchronized void assertGrantorIsConsistent(InternalDistributedMember id) {
+    if (this.lockGrantor == null) {
+      this.lockGrantor = id;
+    } else {
+      assertEquals("assertGrantorIsConsistent failed", lockGrantor, id);
+    }
+  }
+  
+  /**
+   * Accessed via reflection.  DO NOT REMOVE
+   * @param serviceName
+   * @return
+   */
+  public static InternalDistributedMember identifyLockGrantor(String serviceName) {
+    DLockService service = (DLockService)
+        DistributedLockService.getServiceNamed(serviceName);
+    assertNotNull(service);
+    InternalDistributedMember grantor = service.getLockGrantorId().getLockGrantorMember();
+    assertNotNull(grantor);
+    logInfo("In identifyLockGrantor - grantor is " + grantor);
+    return grantor;
+  }
+  
+  /**
+   * Accessed via reflection.  DO NOT REMOVE.
+   * @param serviceName
+   * @return
+   */
+  public static Boolean isLockGrantor(String serviceName) {
+    DLockService service = (DLockService)
+        DistributedLockService.getServiceNamed(serviceName);
+    assertNotNull(service);
+    Boolean result = Boolean.valueOf(service.isLockGrantor());
+    logInfo("In isLockGrantor: " + result);
+    return result;
+  }
+
+  /**
+   * Accessed via reflection.  DO NOT REMOVE.
+   * @param serviceName
+   */
+  protected static void becomeLockGrantor(String serviceName) {
+    DLockService service = (DLockService)
+        DistributedLockService.getServiceNamed(serviceName);
+    assertNotNull(service);
+    logInfo("About to call becomeLockGrantor...");
+    service.becomeLockGrantor();
+  }
+  
+  public void testGrantorSelection() {
+    // TODO change distributedCreateService usage to be concurrent threads
+    
+    // bring up 4 members and make sure all identify one as grantor
+    int numVMs = 4;
+    final String serviceName = "testGrantorSelection_" + getUniqueName();
+    distributedCreateService(numVMs, serviceName);
+    try {
+      Thread.sleep(100);
+    }
+    catch (InterruptedException ignore) {fail("interrupted");}
+
+    final Object[] args = new Object[] {serviceName};
+    final Host host = Host.getHost(0);
+    for (int vm=0; vm<numVMs; vm++) {
+      final int finalvm = vm;
+      logInfo("VM " + finalvm + " in " + serviceName + " about to invoke");
+      InternalDistributedMember id = (InternalDistributedMember)host.getVM(finalvm).invoke(
+          DistributedLockServiceDUnitTest.class, 
+          "identifyLockGrantor", 
+          args);
+      logInfo("VM " + finalvm + " in " + serviceName + " got " + id);
+      assertGrantorIsConsistent(id);
+    }
+  }
+  
+  public void testBasicGrantorRecovery() {
+    //DLockGrantor.setUncleanDestroyEnabled(true);
+//    try {
+      // 1) start up 4 VM members...
+      int numVMs = 4;
+      final String serviceName = "testBasicGrantorRecovery_" + getUniqueName();
+      distributedCreateService(numVMs, serviceName);
+      try {
+        Thread.sleep(100);
+      }
+      catch (InterruptedException ignore) {fail("interrupted");}
+      
+      final Object[] args = new Object[] {serviceName};
+      final Host host = Host.getHost(0);
+      
+      int originalGrantor = 3;
+      host.getVM(originalGrantor).invoke(
+          DistributedLockServiceDUnitTest.class, "identifyLockGrantor", args);
+            
+      // 2) find the grantor and disconnect him...
+      int originalVM = -1;
+      InternalDistributedMember oldGrantor = null;
+      for (int vm = 0; vm < numVMs; vm++) {
+        final int finalvm = vm;
+        Boolean isGrantor = (Boolean)host.getVM(finalvm).invoke(
+            DistributedLockServiceDUnitTest.class, "isLockGrantor", args);
+        if (isGrantor.booleanValue()) {
+          originalVM = vm;
+          oldGrantor = (InternalDistributedMember)host.getVM(finalvm).invoke(
+            DistributedLockServiceDUnitTest.class, "identifyLockGrantor", args);
+          break;
+        }
+      }
+      
+      assertTrue(originalVM == originalGrantor);
+      
+      host.getVM(originalVM).invoke(new SerializableRunnable() {
+        public void run() {
+          disconnectFromDS();
+        }
+      });
+      
+      try {
+        Thread.sleep(100);
+      }
+      catch (InterruptedException ignore) {fail("interrupted");}
+        
+      // 3) verify that another member recovers for grantor
+      int attempts = 3;
+      for (int attempt = 0; attempt < attempts; attempt++) {
+        try {
+          for (int vm = 0; vm < numVMs; vm++) {
+            if (vm == originalVM) continue; // skip because he's disconnected
+            final int finalvm = vm;
+            logInfo("[testBasicGrantorRecovery] VM " + finalvm + 
+                    " in " + serviceName + " about to invoke");
+            InternalDistributedMember id = (InternalDistributedMember)host.getVM(finalvm).invoke(
+                DistributedLockServiceDUnitTest.class, 
+                "identifyLockGrantor", 
+                args);
+            logInfo("[testBasicGrantorRecovery] VM " + finalvm + 
+                    " in " + serviceName + " got " + id);
+            assertGrantorIsConsistent(id);
+            logInfo("[testBasicGrantorRecovery] new grantor " + id + 
+                    " is not old grantor " + oldGrantor);
+            assertEquals("New grantor must not equal the old grantor", true,
+                         !id.equals(oldGrantor)); // new grantor != old grantor
+          } // loop thru vms
+          logInfo("[testBasicGrantorRecovery] succeeded attempt " + attempt);
+          break; // success
+        }
+        catch (AssertionFailedError e) {
+          logInfo("[testBasicGrantorRecovery] failed attempt " + attempt);
+          if (attempt == attempts-1) throw e;
+        }
+      } // loop thru attempts
+//    }
+//    finally {
+      //DLockGrantor.setUncleanDestroyEnabled(false);
+//    }
+  }
+  
+  public void testLockFailover() {
+    final int originalGrantorVM = 0;
+    final int oneVM = 1;
+    final int twoVM = 2;
+    final String serviceName = "testLockFailover-" + getUniqueName();
+    
+    // create lock services...
+    LogWriterUtils.getLogWriter().fine("[testLockFailover] create services");
+    
+    Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.remoteCreateService( serviceName ));
+    
+    Host.getHost(0).getVM(oneVM).invoke(() -> DistributedLockServiceDUnitTest.remoteCreateService( serviceName ));
+    
+    Host.getHost(0).getVM(twoVM).invoke(() -> DistributedLockServiceDUnitTest.remoteCreateService( serviceName ));
+    
+    Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.identifyLockGrantor( serviceName ));
+        
+    Boolean isGrantor = (Boolean) Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.isLockGrantor( serviceName ));
+    assertEquals("First member calling getLockGrantor failed to become grantor", 
+                 Boolean.TRUE, isGrantor);
+
+    // get locks...
+    LogWriterUtils.getLogWriter().fine("[testLockFailover] get lock");
+    
+    Boolean locked = (Boolean) Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "KEY-"+originalGrantorVM ));
+    assertEquals("Failed to get lock in testLockFailover", 
+                 Boolean.TRUE, locked);
+                 
+    locked = (Boolean) Host.getHost(0).getVM(twoVM).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "KEY-"+twoVM ));
+    assertEquals("Failed to get lock in testLockFailover", 
+                 Boolean.TRUE, locked);
+                 
+    locked = (Boolean) Host.getHost(0).getVM(oneVM).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "KEY-"+oneVM ));
+    assertEquals("Failed to get lock in testLockFailover", 
+                 Boolean.TRUE, locked);
+                 
+    // disconnect originalGrantorVM...
+    LogWriterUtils.getLogWriter().fine("[testLockFailover] disconnect originalGrantorVM");
+
+    Host.getHost(0).getVM(originalGrantorVM).invoke(new SerializableRunnable() {
+      public void run() {
+        disconnectFromDS();
+      }
+    });
+    
+    try {
+      Thread.sleep(100);
+    }
+    catch (InterruptedException ignore) {fail("interrupted");}
+                 
+    // verify locks by unlocking...
+    LogWriterUtils.getLogWriter().fine("[testLockFailover] release locks");
+                 
+    Boolean unlocked = (Boolean) Host.getHost(0).getVM(twoVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+twoVM ));
+    assertEquals("Failed to release lock in testLockFailover", 
+                 Boolean.TRUE, unlocked);
+
+    unlocked = (Boolean) Host.getHost(0).getVM(oneVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+oneVM ));
+    assertEquals("Failed to release lock in testLockFailover", 
+                 Boolean.TRUE, unlocked);
+
+    // switch locks...
+    locked = (Boolean) Host.getHost(0).getVM(oneVM).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "KEY-"+twoVM ));
+    assertEquals("Failed to get lock in testLockFailover", 
+                 Boolean.TRUE, locked);
+                 
+    locked = (Boolean) Host.getHost(0).getVM(twoVM).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "KEY-"+oneVM ));
+    assertEquals("Failed to get lock in testLockFailover", 
+                 Boolean.TRUE, locked);
+                 
+    unlocked = (Boolean) Host.getHost(0).getVM(oneVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+twoVM ));
+    assertEquals("Failed to release lock in testLockFailover", 
+                 Boolean.TRUE, unlocked);
+
+    unlocked = (Boolean) Host.getHost(0).getVM(twoVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+oneVM ));
+    assertEquals("Failed to release lock in testLockFailover", 
+                 Boolean.TRUE, unlocked);
+                 
+    // verify grantor is unique...
+    LogWriterUtils.getLogWriter().fine("[testLockFailover] verify grantor identity");
+    
+    InternalDistributedMember oneID = (InternalDistributedMember)Host.getHost(0).getVM(oneVM).invoke(() -> DistributedLockServiceDUnitTest.identifyLockGrantor(serviceName));
+    InternalDistributedMember twoID = (InternalDistributedMember)Host.getHost(0).getVM(twoVM).invoke(() -> DistributedLockServiceDUnitTest.identifyLockGrantor(serviceName));
+    assertTrue("Failed to identifyLockGrantor in testLockFailover",
+               oneID != null && twoID != null);
+    assertEquals("Failed grantor uniqueness in testLockFailover", 
+                 oneID, twoID);
+  }
+  
+  public void testLockThenBecomeLockGrantor() {
+    final int originalGrantorVM = 0;
+    final int becomeGrantorVM = 1;
+    final int thirdPartyVM = 2;
+    final String serviceName = "testLockThenBecomeLockGrantor-" + getUniqueName();
+    
+    // create lock services...
+    LogWriterUtils.getLogWriter().fine("[testLockThenBecomeLockGrantor] create services");
+    
+    Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.remoteCreateService( serviceName ));
+    
+    try {
+      Thread.sleep(20);
+    } catch (InterruptedException ignore) {fail("interrupted");}
+
+    Host.getHost(0).getVM(becomeGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.remoteCreateService( serviceName ));
+    
+    Host.getHost(0).getVM(thirdPartyVM).invoke(() -> DistributedLockServiceDUnitTest.remoteCreateService( serviceName ));
+    
+    Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.identifyLockGrantor( serviceName ));
+        
+    Boolean isGrantor = (Boolean) Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.isLockGrantor( serviceName ));
+    assertEquals("First member calling getLockGrantor failed to become grantor", 
+                 Boolean.TRUE, isGrantor);
+
+    // control...
+    LogWriterUtils.getLogWriter().fine("[testLockThenBecomeLockGrantor] check control");
+    Boolean check = (Boolean) Host.getHost(0).getVM(becomeGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+becomeGrantorVM ));
+    assertEquals("Check of control failed... unlock succeeded but nothing locked", 
+                 Boolean.FALSE, check);
+                 
+    // get locks...
+    LogWriterUtils.getLogWriter().fine("[testLockThenBecomeLockGrantor] get lock");
+    
+    Boolean locked = (Boolean) Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "KEY-"+originalGrantorVM ));
+    assertEquals("Failed to get lock in testLockThenBecomeLockGrantor", 
+                 Boolean.TRUE, locked);
+                 
+    locked = (Boolean) Host.getHost(0).getVM(thirdPartyVM).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "KEY-"+thirdPartyVM ));
+    assertEquals("Failed to get lock in testLockThenBecomeLockGrantor", 
+                 Boolean.TRUE, locked);
+                 
+    locked = (Boolean) Host.getHost(0).getVM(becomeGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "KEY-"+becomeGrantorVM ));
+    assertEquals("Failed to get lock in testLockThenBecomeLockGrantor", 
+                 Boolean.TRUE, locked);
+                 
+    // become lock grantor...
+    LogWriterUtils.getLogWriter().fine("[testLockThenBecomeLockGrantor] become lock grantor");
+    
+    Host.getHost(0).getVM(becomeGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.becomeLockGrantor( serviceName ));
+        
+    try {
+      Thread.sleep(20);
+    } catch (InterruptedException ignore) {fail("interrupted");}
+    
+    isGrantor = (Boolean) Host.getHost(0).getVM(becomeGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.isLockGrantor( serviceName ));
+    assertEquals("Failed to become lock grantor", 
+                 Boolean.TRUE, isGrantor);    
+
+    // verify locks by unlocking...
+    LogWriterUtils.getLogWriter().fine("[testLockThenBecomeLockGrantor] release locks");
+                 
+    Boolean unlocked = (Boolean) Host.getHost(0).getVM(originalGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+originalGrantorVM ));
+    assertEquals("Failed to release lock in testLockThenBecomeLockGrantor", 
+                 Boolean.TRUE, unlocked);
+                 
+    unlocked = (Boolean) Host.getHost(0).getVM(thirdPartyVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+thirdPartyVM ));
+    assertEquals("Failed to release lock in testLockThenBecomeLockGrantor", 
+                 Boolean.TRUE, unlocked);
+
+    unlocked = (Boolean) Host.getHost(0).getVM(becomeGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+becomeGrantorVM ));
+    assertEquals("Failed to release lock in testLockThenBecomeLockGrantor", 
+                 Boolean.TRUE, unlocked);
+
+    // test for bug in which transferred token gets re-entered causing lock recursion
+    unlocked = (Boolean) Host.getHost(0).getVM(becomeGrantorVM).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY-"+becomeGrantorVM ));
+    assertEquals("Transfer of tokens caused lock recursion in held lock", 
+                 Boolean.FALSE, unlocked);
+  }
+  
+  public void testBecomeLockGrantor() {
+    // create lock services...
+    int numVMs = 4;
+    final String serviceName = "testBecomeLockGrantor-" + getUniqueName();
+    distributedCreateService(numVMs, serviceName);
+    
+    // each one gets a lock...
+    for (int vm = 0; vm < numVMs; vm++) {
+      final int finalvm = vm;
+      Boolean locked = Host.getHost(0).getVM(finalvm).invoke(() -> DistributedLockServiceDUnitTest.lock( serviceName, "obj-"+finalvm ));
+      assertEquals("Failed to get lock in testBecomeLockGrantor", 
+                   Boolean.TRUE, locked);
+    }
+    
+    // find the grantor...
+    final Object[] args = new Object[] {serviceName};
+    int originalVM = -1;
+    InternalDistributedMember oldGrantor = null;
+    for (int vm = 0; vm < numVMs; vm++) {
+      final int finalvm = vm;
+      Boolean isGrantor = (Boolean)Host.getHost(0).getVM(finalvm).invoke(
+          DistributedLockServiceDUnitTest.class, "isLockGrantor", args);
+      if (isGrantor.booleanValue()) {
+        originalVM = vm;
+        oldGrantor = (InternalDistributedMember)Host.getHost(0).getVM(finalvm).invoke(
+          DistributedLockServiceDUnitTest.class, "identifyLockGrantor", args);
+        break;
+      }
+    }
+    
+    LogWriterUtils.getLogWriter().fine("[testBecomeLockGrantor] original grantor is " + oldGrantor);
+    
+    // have one call becomeLockGrantor
+    for (int vm = 0; vm < numVMs; vm++) {
+      if (vm != originalVM) {
+        final int finalvm = vm;
+        Host.getHost(0).getVM(finalvm).invoke(
+            DistributedLockServiceDUnitTest.class, "becomeLockGrantor", args);
+        Boolean isGrantor = (Boolean)Host.getHost(0).getVM(finalvm).invoke(
+            DistributedLockServiceDUnitTest.class, "isLockGrantor", args);
+        assertEquals("isLockGrantor is false after calling becomeLockGrantor", 
+                     Boolean.TRUE, isGrantor);
+        break;
+      }
+    }
+    
+    LogWriterUtils.getLogWriter().fine("[testBecomeLockGrantor] one vm has called becomeLockGrantor...");
+    
+    InternalDistributedMember newGrantor = null;
+    for (int vm = 0; vm < numVMs; vm++) {
+      final int finalvm = vm;
+      Boolean isGrantor = (Boolean)Host.getHost(0).getVM(finalvm).invoke(
+          DistributedLockServiceDUnitTest.class, "isLockGrantor", args);
+      if (isGrantor.booleanValue()) {
+        newGrantor = (InternalDistributedMember)Host.getHost(0).getVM(finalvm).invoke(
+          DistributedLockServiceDUnitTest.class, "identifyLockGrantor", args);
+        break;
+      }
+    }
+    LogWriterUtils.getLogWriter().fine("[testBecomeLockGrantor] new Grantor is " + newGrantor);
+    assertEquals(false, newGrantor.equals(oldGrantor));
+    
+    // verify locks still held by unlocking
+    // each one unlocks...
+    for (int vm = 0; vm < numVMs; vm++) {
+      final int finalvm = vm;
+      Boolean unlocked = (Boolean) Host.getHost(0).getVM(finalvm).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "obj-"+finalvm ));
+      assertEquals("Failed to unlock in testBecomeLockGrantor", 
+                   Boolean.TRUE, unlocked);
+    }
+    
+    LogWriterUtils.getLogWriter().fine("[testBecomeLockGrantor] finished");
+    
+    // verify that pending requests are granted by unlocking them also
+  }
+
+  public void testTryLock() {
+    final Long waitMillis = new Long(100);
+    
+    // create lock services...
+    LogWriterUtils.getLogWriter().fine("[testTryLock] create lock services");
+    final String serviceName = "testTryLock-" + getUniqueName();
+    distributedCreateService(4, serviceName);
+    
+    // all 4 vms scramble to get tryLock but only one should succeed...
+    LogWriterUtils.getLogWriter().fine("[testTryLock] attempt to get tryLock");
+    int lockCount = 0;
+    for (int vm = 0; vm < 4; vm++) {
+      final int finalvm = vm;
+      Boolean locked = (Boolean) Host.getHost(0).getVM(finalvm).invoke(() -> DistributedLockServiceDUnitTest.tryLock( serviceName, "KEY", waitMillis ));
+      if (locked.booleanValue()) lockCount++;
+    }
+
+    assertEquals("More than one vm acquired the tryLock", 
+                 1, lockCount);    
+
+    LogWriterUtils.getLogWriter().fine("[testTryLock] unlock tryLock");
+    int unlockCount = 0;
+    for (int vm = 0; vm < 4; vm++) {
+      final int finalvm = vm;
+      Boolean unlocked = (Boolean) Host.getHost(0).getVM(finalvm).invoke(() -> DistributedLockServiceDUnitTest.unlock( serviceName, "KEY" ));
+      if (unlocked.booleanValue()) unlockCount++;
+    }
+
+    assertEquals("More than one vm unlocked the tryLock", 
+                 1, unlockCount);    
+  }
+  
+  public void testOneGetsThenOtherGets()  throws Exception  { // (numVMs, numThreadsPerVM)
+    doOneGetsThenOtherGets(1, 1);
+//     doOneGetsThenOtherGets(2, 2);
+//     doOneGetsThenOtherGets(3, 3);
+    doOneGetsThenOtherGets(4, 3);
+  }
+  
+  public void testLockDifferentNames() throws Exception  {
+    String serviceName = getUniqueName();
+
+    // Same VM
+    remoteCreateService(serviceName);
+    DistributedLockService service = DistributedLockService.getServiceNamed(serviceName);
+    assertTrue(service.lock("obj1", -1, -1));
+    assertTrue(service.lock("obj2", -1, -1));
+    service.unlock("obj1");
+    service.unlock("obj2");
+    
+    // Different VMs
+    VM vm = Host.getHost(0).getVM(0);
+    vm.invoke(() -> this.remoteCreateService(serviceName));
+    assertTrue(service.lock("masterVMobj", -1, -1));
+    
+    assertEquals(Boolean.TRUE, vm.invoke(() -> this.getLockAndIncrement(
+        serviceName, "otherVMobj", new Integer(-1), new Integer(0)
+      )));
+    
+    service.unlock("masterVMobj");
+  }
+  
+  public void testLocalGetLockAndIncrement()  throws Exception {
+    String serviceName = getUniqueName();
+    remoteCreateService(serviceName);
+    DistributedLockService.getServiceNamed(serviceName);
+    assertEquals(Boolean.TRUE, 
+                 getLockAndIncrement(serviceName, "localVMobj", -1, 0));
+  }
+  public void testRemoteGetLockAndIncrement() {
+    String serviceName = getUniqueName();
+    VM vm = Host.getHost(0).getVM(0);
+    vm.invoke(() -> this.remoteCreateService(serviceName));
+    assertEquals(Boolean.TRUE, vm.invoke(() -> this.getLockAndIncrement(
+        serviceName, "remoteVMobj", new Integer(-1), new Integer(0)
+      )));
+  }
+  
+  public void testLockSameNameDifferentService() 
+  {
+    String serviceName1 = getUniqueName() + "_1";
+    String serviceName2 = getUniqueName() + "_2";
+    String objName = "obj";
+
+    // Same VM
+    remoteCreateService(serviceName1);
+    remoteCreateService(serviceName2);
+    DistributedLockService service1 = DistributedLockService.getServiceNamed(serviceName1);
+    DistributedLockService service2 = DistributedLockService.getServiceNamed(serviceName2);
+    assertTrue(service1.lock(objName, -1, -1));
+    assertTrue(service2.lock(objName, -1, -1));
+    service1.unlock(objName);
+    service2.unlock(objName);
+    
+    // Different VMs
+    VM vm = Host.getHost(0).getVM(0);
+    vm.invoke(() -> this.remoteCreateService(serviceName1));
+    vm.invoke(() -> this.remoteCreateService(serviceName2));
+    assertTrue(service1.lock(objName, -1, -1));
+    assertEquals(Boolean.TRUE, vm.invoke(() -> this.getLockAndIncrement(
+        serviceName2, objName, new Integer(-1), new Integer(0)
+      )));
+    service1.unlock(objName);
+  }
+  
+  public void testLeaseDoesntExpire()
+  throws InterruptedException
+  {
+    String serviceName = getUniqueName();
+    final Object objName = new Integer(3);
+
+    // Same VM
+    remoteCreateService(serviceName);
+    final DistributedLockService service = DistributedLockService.getServiceNamed(serviceName);
+    // lock objName with a sufficiently long lease
+    assertTrue(service.lock(objName, -1, 60000));
+    // try to lock in another thread, with a timeout shorter than above lease
+    final boolean[] resultHolder = new boolean[] { false };
+    Thread thread = new Thread(new Runnable() {
+      public void run() {
+        resultHolder[0] = !service.lock(objName, 1000, -1);
+      }
+    });
+    thread.start();
+    ThreadUtils.join(thread, 30 * 1000);
+    assertTrue(resultHolder[0]);
+    // the unlock should succeed without throwing LeaseExpiredException
+    service.unlock(objName);
+    
+    // Different VM
+    VM vm = Host.getHost(0).getVM(0);
+    vm.invoke(() -> this.remoteCreateService(serviceName));
+    // lock objName in this VM with a sufficiently long lease
+    assertTrue(service.lock(objName, -1, 60000));
+    // try to lock in another VM, with a timeout shorter than above lease
+    assertEquals(Boolean.FALSE, vm.invoke(() -> this.getLockAndIncrement(
+        serviceName, objName, new Long(1000), new Long(0)
+      )));
+    // the unlock should succeed without throwing LeaseExpiredException
+    service.unlock(objName);
+  }
+
+  public void testLockUnlock() {
+    String serviceName = getUniqueName();
+    Object objName = new Integer(42);
+    
+    remoteCreateService(serviceName);
+    DistributedLockService service = 
+        DistributedLockService.getServiceNamed(serviceName);
+
+    Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+    
+    service.lock(objName, -1, -1);
+    Assert.assertTrue(service.isHeldByCurrentThread(objName));
+    
+    service.unlock(objName);
+    Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+  }
+  
+  public void testLockExpireUnlock() {
+    long leaseMs = 200;
+    long waitBeforeLockingMs = 210;
+    
+    String serviceName = getUniqueName();
+    Object objName = new Integer(42);
+    
+    remoteCreateService(serviceName);
+    DistributedLockService service = 
+        DistributedLockService.getServiceNamed(serviceName);
+
+    assertTrue(!service.isHeldByCurrentThread(objName));
+    
+    assertTrue(service.lock(objName, -1, leaseMs));
+    assertTrue(service.isHeldByCurrentThread(objName));
+    
+    sleep(waitBeforeLockingMs); // should expire...
+    assertTrue(!service.isHeldByCurrentThread(objName));
+   
+    try {
+      service.unlock(objName);
+      fail("unlock should have thrown LeaseExpiredException");
+    } catch (LeaseExpiredException ex) {
+    }
+  }
+  
+  public void testLockRecursion() {
+    String serviceName = getUniqueName();
+    Object objName = new Integer(42);
+    
+    remoteCreateService(serviceName);
+    DistributedLockService service = 
+        DistributedLockService.getServiceNamed(serviceName);
+
+    Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+    
+    // initial lock...
+    Assert.assertTrue(service.lock(objName, -1, -1));
+    Assert.assertTrue(service.isHeldByCurrentThread(objName));
+
+    // recursion +1...
+    Assert.assertTrue(service.lock(objName, -1, -1));
+
+    // recursion -1...    
+    service.unlock(objName);
+    Assert.assertTrue(service.isHeldByCurrentThread(objName));
+
+    // and unlock...    
+    service.unlock(objName);
+    Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+  }
+  
+  public void testLockRecursionWithExpiration() {
+    long leaseMs = 500;
+    long waitBeforeLockingMs = 750;
+    
+    String serviceName = getUniqueName();
+    Object objName = new Integer(42);
+    
+    remoteCreateService(serviceName);
+    DistributedLockService service = 
+        DistributedLockService.getServiceNamed(serviceName);
+
+    Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+    
+    // initial lock...
+    Assert.assertTrue(service.lock(objName, -1, leaseMs));
+    Assert.assertTrue(service.isHeldByCurrentThread(objName));
+
+    // recursion +1...
+    Assert.assertTrue(service.lock(objName, -1, leaseMs));
+    Assert.assertTrue(service.isHeldByCurrentThread(objName));
+    
+    // expire...
+    sleep(waitBeforeLockingMs);
+    Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+
+    // should fail...
+    try {
+      service.unlock(objName);
+      fail("unlock should have thrown LeaseExpiredException");
+    } catch (LeaseExpiredException ex) {
+    }
+    
+    // relock it...
+    Assert.assertTrue(service.lock(objName, -1, leaseMs));
+    Assert.assertTrue(service.isHeldByCurrentThread(objName));
+
+    // and unlock to verify no recursion...    
+    service.unlock(objName);
+    Assert.assertTrue(!service.isHeldByCurrentThread(objName)); // throws failure!!
+    
+    // go thru again in different order...
+    Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+    
+    // initial lock...
+    Assert.assertTrue(service.lock(objName, -1, leaseMs));
+    Assert.assertTrue(service.isHeldByCurrentThread(objName));
+
+    // expire...
+    sleep(waitBeforeLockingMs);
+    Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+
+    // relock it...
+    Assert.assertTrue(service.lock(objName, -1, leaseMs));
+    Assert.assertTrue(service.isHeldByCurrentThread(objName));
+    
+    // and unlock to verify no recursion...    
+    service.unlock(objName);
+    Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+  }
+  
+  public void testLeaseExpiresBeforeOtherLocks()
+  throws InterruptedException 
+  {
+    leaseExpiresTest(false);
+  }
+  
+  public void testLeaseExpiresWhileOtherLocks()
+  throws InterruptedException 
+  {
+    leaseExpiresTest(true);
+  }
+  
+  private void leaseExpiresTest(boolean tryToLockBeforeExpiration)
+  throws InterruptedException
+  {
+    LogWriterUtils.getLogWriter().fine("[testLeaseExpires] prepping");
+    long leaseMs = 100;
+    long waitBeforeLockingMs = tryToLockBeforeExpiration ? 50 : 110;
+
+    final String serviceName = getUniqueName();
+    final Object objName = new Integer(3);
+
+    // Same VM
+    remoteCreateService(serviceName);
+    final DistributedLockService service = DistributedLockService.getServiceNamed(serviceName);
+    
+    LogWriterUtils.getLogWriter().fine("[testLeaseExpires] acquire first lock");
+    // lock objName with a short lease
+    assertTrue(service.lock(objName, -1, leaseMs));
+    sleep(waitBeforeLockingMs);
+
+    if (waitBeforeLockingMs > leaseMs) {
+      Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+    }
+    
+    LogWriterUtils.getLogWriter().fine("[testLeaseExpires] acquire lock that expired");
+    // try to lock in another thread - lease should have expired
+    final boolean[] resultHolder = new boolean[] { false };
+    Thread thread = new Thread(new Runnable() {
+      public void run() {
+        resultHolder[0] = service.lock(objName, -1, -1);
+        service.unlock(objName);
+        Assert.assertTrue(!service.isHeldByCurrentThread(objName));
+      }
+    });
+    thread.start();
+    ThreadUtils.join(thread, 30 * 1000);
+    assertTrue(resultHolder[0]);
+    
+    LogWriterUtils.getLogWriter().fine("[testLeaseExpires] unlock should throw LeaseExpiredException");
+    // this thread's unlock should throw LeaseExpiredException
+    try {
+      service.unlock(objName);
+      fail("unlock should have thrown LeaseExpiredException");
+    } catch (LeaseExpiredException ex) {
+    }
+    
+    LogWriterUtils.getLogWriter().fine("[testLeaseExpires] create service in other vm");
+    // Different VM
+    VM vm = Host.getHost(0).getVM(0);
+    vm.invoke(() -> this.remoteCreateService(serviceName));
+    
+    LogWriterUtils.getLogWriter().fine("[testLeaseExpires] acquire lock again and expire");
+    // lock objName in this VM with a short lease
+    assertTrue(service.lock(objName, -1, leaseMs));
+    sleep(waitBeforeLockingMs);
+    
+    LogWriterUtils.getLogWriter().fine("[testLeaseExpires] succeed lock in other vm");
+    // try to lock in another VM - should succeed
+    assertEquals(Boolean.TRUE, vm.invoke(() -> this.getLockAndIncrement(
+        serviceName, objName, new Long(-1), new Long(0)
+      )));
+      
+    LogWriterUtils.getLogWriter().fine("[testLeaseExpires] unlock should throw LeaseExpiredException again");
+    // this VMs unlock should throw LeaseExpiredException
+    try {
+      service.unlock(objName);
+      fail("unlock should have thrown LeaseExpiredException");
+    } catch (LeaseExpiredException ex) {
+    }
+  }
+
+  public void testSuspendLockingAfterExpiration() throws Exception {
+    LogWriterUtils.getLogWriter().fine("[leaseExpiresThenSuspendTest]");
+    
+    final long leaseMillis = 100;
+    final long suspendWaitMillis = 10000;
+
+    final String serviceName = getUniqueName();
+    final Object key = new Integer(3);
+
+    // controller locks key and then expires - controller is grantor
+    
+    DistributedLockService dls 
+        = DistributedLockService.create(serviceName, getSystem());
+    
+    assertTrue(dls.lock(key, -1, leaseMillis));
+    
+    // wait for expiration
+    sleep(leaseMillis*2);
+    
+    LogWriterUtils.getLogWriter().fine("[leaseExpiresThenSuspendTest] unlock should throw LeaseExpiredException");
+    // this thread's unlock should throw LeaseExpiredException
+    try {
+      dls.unlock(key);
+      fail("unlock should have thrown LeaseExpiredException");
+    } catch (LeaseExpiredException ex) {
+    }
+    
+    // other vm calls suspend 
+    
+    LogWriterUtils.getLogWriter().fine("[leaseExpiresThenSuspendTest] call to suspend locking");
+    Host.getHost(0).getVM(0).invoke(new SerializableRunnable() {
+      public void run() {
+        final DistributedLockService dlock 
+            = DistributedLockService.create(serviceName, getSystem());
+        dlock.suspendLocking(suspendWaitMillis);
+        dlock.resumeLocking();
+        assertTrue(dlock.lock(key, -1, leaseMillis));
+        dlock.unlock(key);
+      }
+    });
+  }
+  
+  volatile boolean started = false;
+  volatile boolean gotLock = false;
+  volatile Throwable exception = null;
+  volatile Throwable throwable = null;
+
+  public void testLockInterruptiblyIsInterruptible() {
+    started = false;
+    gotLock = false;
+    exception = null;
+    throwable = null;
+
+    // Lock entire service in first thread
+    LogWriterUtils.getLogWriter().info("[testLockInterruptiblyIsInterruptible] get and hold the lock");
+    final String serviceName = getUniqueName();
+    final DistributedLockService service = 
+      DistributedLockService.create(serviceName, dlstSystem);
+    service.becomeLockGrantor();
+    assertTrue(service.lock("obj", 1000, -1));
+
+    // Start second thread that tries to lock in second thread
+    LogWriterUtils.getLogWriter().info("[testLockInterruptiblyIsInterruptible] call lockInterruptibly");
+    Thread thread2 = new Thread(new Runnable() {
+      public void run() {
+        try {
+          started = true;
+          gotLock = service.lockInterruptibly("obj", -1, -1);
+        } catch (InterruptedException ex) {
+          exception = ex;
+        } 
+        catch (VirtualMachineError e) {
+          SystemFailure.initiateFailure(e);
+          throw e;
+        }
+        catch (Throwable t) {
+          throwable = t;
+        }
+      }
+    });
+    thread2.start();
+    
+    // Interrupt second thread
+    LogWriterUtils.getLogWriter().info("[testLockInterruptiblyIsInterruptible] interrupt calling thread");
+    while (!started) Thread.yield();
+    thread2.interrupt();
+    ThreadUtils.join(thread2, 20 * 1000);
+
+    // Expect it got InterruptedException and didn't lock the service
+    LogWriterUtils.getLogWriter().info("[testLockInterruptiblyIsInterruptible] verify failed to get lock");
+    assertFalse(gotLock);
+    if (throwable != null) {
+      LogWriterUtils.getLogWriter().warning(
+        "testLockInterruptiblyIsInterruptible threw unexpected Throwable", 
+        throwable);
+    }
+    assertNotNull(exception);
+
+    // Unlock "obj" in first thread
+    LogWriterUtils.getLogWriter().info("[testLockInterruptiblyIsInterruptible] unlock the lock");
+    service.unlock("obj");
+
+    // Make sure it didn't get locked by second thread
+    LogWriterUtils.getLogWriter().info("[testLockInterruptiblyIsInterruptible] try to get lock with timeout should not fail");
+    assertTrue(service.lock("obj", 5000, -1));
+    DistributedLockService.destroy(serviceName);
+  }
+  
+  volatile boolean wasFlagSet = false;
+
+  public void testLockIsNotInterruptible() {
+    // Lock entire service in first thread
+    LogWriterUtils.getLogWriter().fine("[testLockIsNotInterruptible] lock in first thread");
+    started = false;
+    gotLock = false;
+    exception = null;
+    wasFlagSet = false;
+    
+    final String serviceName = getUniqueName();
+    final DistributedLockService service = 
+      DistributedLockService.create(serviceName, dlstSystem);
+    assertTrue(service.lock("obj", 1000, -1));
+
+    // Start second thread that tries to lock in second thread
+    LogWriterUtils.getLogWriter().fine("[testLockIsNotInterruptible] attempt lock in second thread");
+    Thread thread2 = new Thread(new Runnable() {
+      public void run() {
+        try {
+          started = true;
+          gotLock = service.lock("obj", -1, -1);
+          LogWriterUtils.getLogWriter().fine("[testLockIsNotInterruptible] thread2 finished lock() - got " + gotLock);
+        } 
+        catch (VirtualMachineError e) {
+          SystemFailure.initiateFailure(e);
+          throw e;
+        }
+        catch (Throwable ex) {
+          LogWriterUtils.getLogWriter().warning("[testLockIsNotInterruptible] Caught...", ex);
+          exception = ex;
+        }
+        wasFlagSet = Thread.currentThread().isInterrupted();
+      }
+    });
+    thread2.start();
+    
+    // Interrupt second thread
+    LogWriterUtils.getLogWriter().fine("[testLockIsNotInterruptible] interrupt second thread");
+    while (!started) Thread.yield();
+    sleep(500);
+    thread2.interrupt();
+    // Expect it didn't get an exception and didn't lock the service
+    sleep(500);
+    assertFalse(gotLock);
+    assertNull(exception);
+
+    // Unlock entire service in first thread
+    LogWriterUtils.getLogWriter().fine("[testLockIsNotInterruptible] unlock in first thread");
+    service.unlock("obj");
+    sleep(500);
+
+    // Expect that thread2 should now complete execution.
+    ThreadUtils.join(thread2, 20 * 1000);
+
+    // Now thread2 should have gotten the lock, not the exception, but the
+    // thread's flag should be set
+    LogWriterUtils.getLogWriter().fine("[testLockIsNotInterruptible] verify second thread got lock");
+    assertNull(exception);
+    assertTrue(gotLock);
+    assertTrue(wasFlagSet);
+  }
+  
+  /**
+   * Test DistributedLockService.acquireExclusiveLocking(), releaseExclusiveLocking()
+   */
+  public void testSuspendLockingBasic() 
+  throws InterruptedException {
+    final DistributedLockService service = 
+      DistributedLockService.create(getUniqueName(), dlstSystem);
+
+    try {
+      service.resumeLocking();
+      fail("Didn't throw LockNotHeldException");
+    } catch (LockNotHeldException ex) {
+      // expected
+    }
+
+    assertTrue(service.suspendLocking(-1));
+    service.resumeLocking();
+
+    // It's not reentrant
+    assertTrue(service.suspendLocking(1000));
+    try {
+      service.suspendLocking(1);
+      fail("didn't get IllegalStateException");
+    } catch (IllegalStateException ex) {
+      // expected
+    }
+    service.resumeLocking();
+    
+    // Get "false" if another thread is holding it
+    Thread thread = new Thread(new Runnable() {
+      public void run() {
+        logInfo("new thread about to suspendLocking()");
+        assertTrue(service.suspendLocking(1000));
+      }
+    });
+    thread.start();
+    ThreadUtils.join(thread, 30 * 1000);
+    logInfo("main thread about to suspendLocking");
+    assertTrue(!service.suspendLocking(1000));
+  }
+
+  /**
+   * Test that exlusive locking prohibits locking activity
+   */
+  public void testSuspendLockingProhibitsLocking() 
+  {
+    final String name = getUniqueName();
+    distributedCreateService(2, name);
+    DistributedLockService service = 
+      DistributedLockService.getServiceNamed(name);
+    
+    // Should be able to lock from other VM
+    VM vm1 = Host.getHost(0).getVM(1);
+    assertTrue(vm1.invoke(() -> DistributedLockServiceDUnitTest.tryToLock( name )));
+
+    assertTrue(service.suspendLocking(1000));
+    
+    // vm1 is the grantor... use debugHandleSuspendTimeouts
+    vm1.invoke(new SerializableRunnable("setDebugHandleSuspendTimeouts") {
+      public void run() {
+        DLockService dls = 
+          (DLockService) DistributedLockService.getServiceNamed(name);
+        assertTrue(dls.isLockGrantor());
+        DLockGrantor grantor = dls.getGrantorWithNoSync();
+        grantor.setDebugHandleSuspendTimeouts(5000);
+      }
+    });
+    
+    // Shouldn't be able to lock a name from another VM
+    assertTrue(!vm1.invoke(() -> DistributedLockServiceDUnitTest.tryToLock( name )));
+    
+    service.resumeLocking();
+
+    vm1.invoke(new SerializableRunnable("unsetDebugHandleSuspendTimeouts") {
+      public void run() {
+        DLockService dls = 
+          (DLockService) DistributedLockService.getServiceNamed(name);
+        assertTrue(dls.isLockGrantor());
+        DLockGrantor grantor = dls.getGrantorWithNoSync();
+        grantor.setDebugHandleSuspendTimeouts(0);
+      }
+    });
+    
+    // Should be able to lock again
+    assertTrue(vm1.invoke(() -> DistributedLockServiceDUnitTest.tryToLock( name )));
+    
+  }
+
+  /**
+   * Test that suspend locking behaves under various usage patterns. This
+   * ensures that suspend and regular locks behave as ReadWriteLocks and
+   * processing occurs in order.
+   */
+  public void notestSuspendLockingBehaves() throws Exception {
+    try {
+      doTestSuspendLockingBehaves();
+    }
+    finally {
+      Invoke.invokeInEveryVM(new SerializableRunnable() {
+        public void run() {
+          try {
+            if (suspendClientSuspendLockingBehaves != null) {
+              suspendClientSuspendLockingBehaves.stop();
+              suspendClientSuspendLockingBehaves = null;
+            }
+          }
+          catch (VirtualMachineError e) {
+            SystemFailure.initiateFailure(e);
+            throw e;
+          }
+          catch (Throwable t) {
+            LogWriterUtils.getLogWriter().error("Error in testSuspendLockingBehaves finally", t);
+          }
+          try {
+            if (lockClientSuspendLockingBehaves != null) {
+              lockClientSuspendLockingBehaves.stop();
+              lockClientSuspendLockingBehaves = null;
+            }
+          }
+          catch (VirtualMachineError e) {
+            SystemFailure.initiateFailure(e);
+            throw e;
+          }
+          catch (Throwable t) {
+            LogWriterUtils.getLogWriter().error("Error in testSuspendLockingBehaves finally", t);
+          }
+        }
+      });
+    }
+  }
+  private void doTestSuspendLockingBehaves() throws Exception {
+    final String dlsName = getUniqueName();
+    final VM vmGrantor = Host.getHost(0).getVM(0);
+    final VM vmOne = Host.getHost(0).getVM(1);
+    final VM vmTwo = Host.getHost(0).getVM(2);
+    final VM vmThree = Host.getHost(0).getVM(3);
+    final String key1 = "key1";
+    
+    // TODO: make sure suspend thread can get other locks
+    
+    // TODO: test local (in grantor) locks and suspends also
+    
+    // define some SerializableRunnables
+    final SerializableRunnable createDLS = 
+    new SerializableRunnable("Create "+dlsName) {
+      public void run() {
+        DistributedLockService.create(dlsName, getSystem());
+        lockClientSuspendLockingBehaves = new BasicLockClient(dlsName, key1);
+        suspendClientSuspendLockingBehaves = new BasicLockClient(dlsName, key1);
+        assertFalse(isLockGrantor(dlsName).booleanValue());
+      }
+    };
+    final SerializableRunnable suspendLocking = 
+    new SerializableRunnable("Suspend locking "+dlsName) {
+      public void run() {
+        suspendClientSuspendLockingBehaves.suspend();
+      }
+    };
+    final SerializableRunnable resumeLocking = 
+    new SerializableRunnable("Resume locking "+dlsName) {
+      public void run() {
+        suspendClientSuspendLockingBehaves.resume();
+      }
+    };
+    final SerializableRunnable lockKey = 
+    new SerializableRunnable("Get lock "+dlsName) {
+      public void run() {
+        lockClientSuspendLockingBehaves.lock();
+      }
+    };
+    final SerializableRunnable unlockKey = 
+    new SerializableRunnable("Unlock "+dlsName) {
+      public void run() {
+        lockClientSuspendLockingBehaves.unlock();
+      }
+    };
+    
+    // create grantor
+    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] Create grantor "+dlsName);
+    vmGrantor.invoke(new SerializableRunnable("Create grantor "+dlsName) {
+      public void run() {
+        DistributedLockService.create(dlsName, getSystem());
+        DistributedLockService.getServiceNamed(dlsName).lock(key1, -1, -1);
+        DistributedLockService.getServiceNamed(dlsName).unlock(key1);
+        assertTrue(isLockGrantor(dlsName).booleanValue());
+      }
+    });
+
+    // create dls in other vms
+//    getLogWriter().info("[testSuspendLockingBehaves] Create DLS in vmOne");
+    vmOne.invoke(createDLS);
+//    getLogWriter().info("[testSuspendLockingBehaves] Create DLS in vmTwo");
+    vmTwo.invoke(createDLS);
+//    getLogWriter().info("[testSuspendLockingBehaves] Create DLS in vmThree");
+    vmThree.invoke(createDLS);
+    
+    // get a lock
+    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] line up vms for lock");
+//    getLogWriter().info("[testSuspendLockingBehaves] vmOne lock");
+    vmOne.invoke(lockKey);
+//    getLogWriter().info("[testSuspendLockingBehaves] start vmTwoLocking");
+    AsyncInvocation vmTwoLocking =  vmTwo.invokeAsync(lockKey);
+    Wait.pause(2000); // make sure vmTwo is first in line
+//    getLogWriter().info("[testSuspendLockingBehaves] start vmThreeLocking");
+    AsyncInvocation vmThreeLocking = vmThree.invokeAsync(lockKey);
+    Wait.pause(2000);
+    
+    // make sure vmTwo and vmThree are still waiting for lock on key1
+//    getLogWriter().info("[testSuspendLockingBehaves] assert vmTwoLocking still alive");
+    Wait.pause(100);
+    assertTrue(vmTwoLocking.isAlive());
+//    getLogWriter().info("[testSuspendLockingBehaves] assert vmThreeLocking still alive");
+    Wait.pause(100);
+    assertTrue(vmThreeLocking.isAlive());
+    
+    // let vmTwo get key
+    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] unlock so vmTwo can get key");
+//    getLogWriter().info("[testSuspendLockingBehaves] vmOne unlock");
+    vmOne.invoke(unlockKey);
+    ThreadUtils.join(vmTwoLocking, 10 * 1000);
+
+    // start suspending in vmOne and vmTwo
+    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] start suspending requests");
+//    getLogWriter().info("[testSuspendLockingBehaves] start vmOneSuspending");
+    AsyncInvocation vmOneSuspending = vmOne.invokeAsync(suspendLocking);
+    Wait.pause(2000); // make sure vmOne is first in line
+//    getLogWriter().info("[testSuspendLockingBehaves] start vmTwoSuspending");
+    AsyncInvocation vmTwoSuspending  = vmTwo.invokeAsync(suspendLocking);
+    Wait.pause(2000); 
+    
+    // let vmThree finish locking key
+    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] unlock so vmThree can get key");
+//    getLogWriter().info("[testSuspendLockingBehaves] vmTwo unlock");
+    vmTwo.invoke(unlockKey);
+    ThreadUtils.join(vmThreeLocking, 10 * 1000);
+    
+    // have vmOne get back in line for locking key
+    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] start another lock request");
+//    getLogWriter().info("[testSuspendLockingBehaves] start vmOneLockingAgain");
+    AsyncInvocation vmOneLockingAgain = vmOne.invokeAsync(lockKey);
+    Wait.pause(2000); 
+    
+    // let vmOne suspend locking
+    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] let vmOne suspend locking");
+//    getLogWriter().info("[testSuspendLockingBehaves] assert vmOneSuspending still alive");
+    Wait.pause(100);
+    assertTrue(vmOneSuspending.isAlive());
+//    getLogWriter().info("[testSuspendLockingBehaves] vmThree unlock");
+    vmThree.invoke(unlockKey);
+    ThreadUtils.join(vmOneSuspending, 10 * 1000);
+    
+    // start suspending in vmThree
+    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] line up vmThree for suspending");
+//    getLogWriter().info("[testSuspendLockingBehaves] start vmThreeSuspending");
+    AsyncInvocation vmThreeSuspending  = vmThree.invokeAsync(suspendLocking);
+    Wait.pause(2000); 
+
+    // let vmTwo suspend locking
+    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] let vmTwo suspend locking");
+//    getLogWriter().info("[testSuspendLockingBehaves] assert vmTwoSuspending still alive");
+    Wait.pause(100);
+    assertTrue(vmTwoSuspending.isAlive());
+//    getLogWriter().info("[testSuspendLockingBehaves] vmOne resumes locking");
+    vmOne.invoke(resumeLocking);
+    ThreadUtils.join(vmTwoSuspending, 10 * 1000);
+    
+    // let vmOne get that lock
+    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] let vmOne get that lock");
+//    getLogWriter().info("[testSuspendLockingBehaves] assert vmOneLockingAgain still alive");
+    Wait.pause(100);
+    assertTrue(vmOneLockingAgain.isAlive());
+//    getLogWriter().info("[testSuspendLockingBehaves] vmTwo resumes locking");
+    vmTwo.invoke(resumeLocking);
+    ThreadUtils.join(vmOneLockingAgain, 10 * 1000);
+    
+    // let vmThree suspend locking
+    LogWriterUtils.getLogWriter().info("[testSuspendLockingBehaves] let vmThree suspend locking");
+//    getLogWriter().info("[testSuspendLockingBehaves] assert vmThreeSuspending still alive");
+    Wait.pause(100);
+    assertTrue(vmThreeSuspending.isAlive());
+//    getLogWriter().info("[testSuspendLockingBehaves] vmOne unlocks again");
+    vmOne.invoke(unlockKey);
+    ThreadUtils.join(vmThreeSuspending, 10 * 1000);
+
+    // done
+//    getLogWriter().info("[testSuspendLockingBehaves] vmThree resumes locking");
+    vmThree.invoke(resumeLocking);
+  }
+  protected static BasicLockClient suspendClientSuspendLockingBehaves;
+  protected static BasicLockClient lockClientSuspendLockingBehaves;
+  
+  /**
+   * Test that exlusive locking prohibits locking activity
+   */
+  public void testSuspendLockingBlocksUntilNoLocks() 
+  throws InterruptedException
+  {
+
+    final String name = getUniqueName();
+    distributedCreateService(2, name);
+    final DistributedLockService service = 
+      DistributedLockService.getServiceNamed(name);
+    
+    // Get lock from other VM.  Since same thread needs to lock and unlock,
+    // invoke asynchronously, get lock, wait to be notified, then unlock.
+    VM vm1 = Host.getHost(0).getVM(1);
+    vm1.invokeAsync(new SerializableRunnable("Lock & unlock in vm1") {
+      public void run() {
+        DistributedLockService service2 = 
+          DistributedLockService.getServiceNamed(name);
+        assertTrue(service2.lock("lock", -1, -1));
+        synchronized (monitor) {
+          try {
+            monitor.wait();
+          } catch (InterruptedException ex) {
+            System.out.println("Unexpected InterruptedException");
+            fail("interrupted");
+          }
+        }
+        service2.unlock("lock");
+      }
+    });
+    // Let vm1's thread get the lock and go into wait()
+    Thread.sleep(100);
+    
+    Thread thread = new Thread(new Runnable() {
+      public void run() {
+        setGot(service.suspendLocking(-1));
+        setDone(true);
+        service.resumeLocking();
+      }
+    });
+    setGot(false);
+    setDone(false);
+    thread.start();
+    
+    // Let thread start, make sure it's blocked in suspendLocking
+    Thread.sleep(100);
+    assertFalse("Before release, got: " + getGot() + ", done: " + getDone(), getGot() || getDone());
+    
+    vm1.invoke(new SerializableRunnable("notify vm1 to unlock") {
+      public void run() {
+        synchronized (monitor) {
+          monitor.notify();
+        }
+      }
+    });
+
+    // Let thread finish, make sure it successfully suspended and is done
+    WaitCriterion ev = new WaitCriterion() {
+      public boolean done() {
+        return getDone();
+      }
+      public String description() {
+        return null;
+      }
+    };
+    Wait.waitForCriterion(ev, 30 * 1000, 200, true);
+    if (!getGot() || !getDone()) {
+      ThreadUtils.dumpAllStacks();
+    }
+    assertTrue("After release, got: " + getGot() + ", done: " + getDone(), getGot() && getDone());
+    
+  }
+  
+  public void testSuspendLockingInterruptiblyIsInterruptible() {
+    
+    started = false;
+    gotLock = false;
+    exception = null;
+
+    // Lock entire service in first thread
+    final String name = getUniqueName();
+    final DistributedLockService service = 
+      DistributedLockService.create(name, dlstSystem);
+    assertTrue(service.suspendLocking(1000));
+
+    // Start second thread that tries to lock in second thread
+    Thread thread2 = new Thread(new Runnable() {
+      public void run() {
+        try {
+          started = true;
+          gotLock = service.suspendLockingInterruptibly(-1);
+        } catch (InterruptedException ex) {
+          exception = ex;
+        }
+      }
+    });
+    thread2.start();
+    
+    // Interrupt second thread
+    while (!started) Thread.yield();
+    thread2.interrupt();
+    ThreadUtils.join(thread2, 20 * 1000);
+
+    // Expect it got InterruptedException and didn't lock the service
+    sleep(500);
+    assertFalse(gotLock);
+    assertNotNull(exception);
+
+    // Unlock entire service in first thread
+    service.resumeLocking();
+    sleep(500);
+
+    // Make sure it didn't get locked by second thread
+    assertTrue(service.suspendLocking(1000));
+    DistributedLockService.destroy(name);
+  }
+  
+  public void testSuspendLockingIsNotInterruptible() {
+    
+    started = false;
+    gotLock = false;
+    exception = null;
+    wasFlagSet = false;
+
+    // Lock entire service in first thread
+    final String name = getUniqueName();
+    final DistributedLockService service = 
+      DistributedLockService.create(name, dlstSystem);
+    assertTrue(service.suspendLocking(1000));
+
+    // Start second thread that tries to lock in second thread
+    Thread thread2 = new Thread(new Runnable() {
+      public void run() {
+        try {
+          started = true;
+          gotLock = service.suspendLocking(-1);
+        } 
+        catch (VirtualMachineError e) {
+          SystemFailure.initiateFailure(e);
+          throw e;
+        }
+        catch (Throwable ex) {
+          exception = ex;
+        }
+        wasFlagSet = Thread.currentThread().isInterrupted();
+     }
+    });
+    thread2.start();
+    
+    // Interrupt second thread
+    while (!started) Thread.yield();
+    thread2.interrupt();
+    // Expect it didn't get an exception and didn't lock the service
+    sleep(500);
+    assertFalse(gotLock);
+    assertNull(exception);
+
+    // Unlock entire service in first thread
+    service.resumeLocking();
+    ThreadUtils.join(thread2, 20 * 1000);
+
+    // Now thread2 should have gotten the lock, not the exception, but the
+    // thread's flag should be set
+    LogWriterUtils.getLogWriter().info("[testSuspendLockingIsNotInterruptible]" +
+      " gotLock=" + gotLock +
+      " wasFlagSet=" + wasFlagSet +
+      " exception=" + exception, exception);
+    assertTrue(gotLock);
+    assertNull(exception);
+    assertTrue(wasFlagSet);
+  }
+  
+  /**
+   * Tests what happens when you attempt to lock a name on a lock
+   * service that has been destroyed.
+   *
+   * @author David Whitlock
+   */
+  public void testLockDestroyedService() {
+    String serviceName = this.getUniqueName();
+    DistributedLockService service =
+      DistributedLockService.create(serviceName, dlstSystem);
+    DistributedLockService.destroy(serviceName);
+    try {
+      boolean locked = service.lock("TEST", -1, -1);
+      fail("Lock of destroyed service returned: " + locked);
+
+    } catch (LockServiceDestroyedException ex) {
+      // pass...
+    }
+  }
+  
+  public void testDepartedLastOwnerWithLease() {
+    final String serviceName = this.getUniqueName();
+
+    // Create service in this VM
+    DistributedLockService service =
+      DistributedLockService.create(serviceName, dlstSystem);
+    assertTrue(service.lock("key", -1, -1));
+    service.unlock("key");
+    
+    // Create service in other VM
+    VM otherVm = Host.getHost(0).getVM(0);
+    otherVm.invoke(new SerializableRunnable() {
+      public void run() {
+        DistributedLockService service2 =
+          DistributedLockService.create(serviceName, dlstSystem);
+        service2.lock("key", -1, 360000);
+        service2.unlock("key");
+        // Wait for asynchronous messaging to complete
+        try {
+          Thread.sleep(100);
+        } catch (InterruptedException ex) {
+          fail("interrupted");
+        }
+        disconnectFromDS();
+      }
+    });
+    
+    // Now lock back in this VM
+    assertTrue(service.lock("key", -1, -1));
+    
+  }
+  
+  public void testDepartedLastOwnerNoLease() {
+    final String serviceName = this.getUniqueName();
+
+    // Create service in this VM
+    DistributedLockService service =
+      DistributedLockService.create(serviceName, dlstSystem);
+    assertTrue(service.lock("key", -1, -1));
+    service.unlock("key");
+    
+    // Create service in other VM
+    VM otherVm = Host.getHost(0).getVM(0);
+    otherVm.invoke(new SerializableRunnable() {
+      public void run() {
+        DistributedLockService service2 =
+          DistributedLockService.create(serviceName, dlstSystem);
+        service2.lock("key", -1, -1);
+        service2.unlock("key");
+        // Wait for asynchronous messaging to complete
+        try {
+          Thread.sleep(100);
+        } catch (InterruptedException ex) {
+          fail("interrupted");
+        }
+        disconnectFromDS();
+      }
+    });
+    
+    // Now lock back in this VM
+    assertTrue(service.lock("key", -1, -1));
+    
+  }
+  
+  /** 
+   * Tests for 32461 R3 StuckLocks can occur on locks with an expiration lease
+   * <p>
+   * VM-A locks/unlocks "lock", VM-B leases "lock" and disconnects, VM-C
+   * attempts to lock "lock" and old dlock throws StuckLockException. VM-C
+   * should now succeed in acquiring the lock.
+   */
+  public void testBug32461() throws Exception {
+    LogWriterUtils.getLogWriter().fine("[testBug32461] prepping");
+
+    final String serviceName = getUniqueName();
+    final Object objName = "32461";
+    final int VM_A = 0;
+    final int VM_B = 1;
+    final int VM_C = 2;
+
+    // VM-A locks/unlocks "lock"...
+    LogWriterUtils.getLogWriter().fine("[testBug32461] VM-A locks/unlocks '32461'");
+    
+    Host.getHost(0).getVM(VM_A).invoke(new SerializableRunnable() {
+      public void run() {
+        remoteCreateService(serviceName);
+        final DistributedLockService service = 
+            DistributedLockService.getServiceNamed(serviceName);
+        assertTrue(service.lock(objName, -1, Long.MAX_VALUE));
+        service.unlock(objName);
+      }
+    });
+    
+    // VM-B leases "lock" and disconnects,
+    LogWriterUtils.getLogWriter().fine("[testBug32461] VM_B leases '32461' and disconnects");
+    
+    Host.getHost(0).getVM(VM_B).invoke(new SerializableRunnable() {
+      public void run() {
+        remoteCreateService(serviceName);
+        final DistributedLockService service = 
+            DistributedLockService.getServiceNamed(serviceName);
+        assertTrue(service.lock(objName, -1, Long.MAX_VALUE));
+        DistributedLockService.destroy(serviceName);
+        disconnectFromDS();
+      }
+    });
+    
+    LogWriterUtils.getLogWriter().fine("[testBug32461] VM_C attempts to lock '32461'");
+
+    Host.getHost(0).getVM(VM_C).invoke(new SerializableRunnable() {
+      public void run() {
+        remoteCreateService(serviceName);
+        final DistributedLockService service = 
+            DistributedLockService.getServiceNamed(serviceName);
+        assertTrue(service.lock(objName, -1, -1));
+        service.unlock(objName);
+      }
+    });
+  }
+  
+  public void testNoStuckLock() {
+    final String serviceName = this.getUniqueName();
+    final Object keyWithLease = "key-with-lease";
+    final Object keyNoLease = "key-no-lease";
+
+    // Create service in this VM
+    DistributedLockService service =
+      DistributedLockService.create(serviceName, dlstSystem);
+
+    assertTrue(service.lock(keyWithLease, -1, -1));
+    service.unlock(keyWithLease);
+
+    assertTrue(service.lock(keyNoLease, -1, -1));
+    service.unlock(keyNoLease);
+    
+    // Create service in other VM
+    VM otherVm = Host.getHost(0).getVM(0);
+    otherVm.invoke(new SerializableRunnable() {
+      public void run() {
+        DistributedLockService service2 =
+          DistributedLockService.create(serviceName, dlstSystem);
+        service2.lock(keyWithLease, -1, 360000);
+        service2.lock(keyNoLease, -1, -1);
+        disconnectFromDS();
+      }
+    });
+    
+    // Now lock back in this VM... no stuck locks anymore
+    assertTrue(service.lock(keyWithLease, -1, -1));
+    service.unlock(keyWithLease);
+    assertTrue(service.lock(keyNoLease, -1, -1));
+    service.unlock(keyNoLease);
+  }
+  
+  volatile boolean startedThread1_testReleaseOrphanedGrant;
+  volatile boolean releaseThread1_testReleaseOrphanedGrant;
+  volatile boolean startedThread2_testReleaseOrphanedGrant;
+  volatile boolean gotLockThread2_testReleaseOrphanedGrant;
+  /**
+   * Client requests lock and then interrupts lock request before processing
+   * the grant reply. This causes the Client to send a release msg to the
+   * grantor.
+   */
+  public void testReleaseOrphanedGrant_Local() {
+    DLockRequestProcessor.setDebugReleaseOrphanedGrant(true);
+    DLockRequestProcessor.setWaitToProcessDLockResponse(false);
+    try {
+      startedThread2_testReleaseOrphanedGrant = false;
+      gotLockThread2_testReleaseOrphanedGrant = false;
+      releaseThread1_testReleaseOrphanedGrant = false;
+      
+      LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Local] create lock service");
+      final String serviceName = getUniqueName();
+      final DistributedLockService service = 
+        DistributedLockService.create(serviceName, dlstSystem);
+      
+      // thread to get lock and wait and then unlock
+      final Thread thread1 = new Thread(new Runnable() {
+        public void run() {
+          LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Local] get the lock");
+          assertTrue(service.lock("obj", -1, -1));
+          DLockRequestProcessor.setWaitToProcessDLockResponse(true);
+          startedThread1_testReleaseOrphanedGrant = true;
+          synchronized(Thread.currentThread()) {
+            while (!releaseThread1_testReleaseOrphanedGrant) {
+              try {
+                Thread.currentThread().wait();
+              } catch (InterruptedException ignore) {fail("interrupted");}
+            }
+          }
+          LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Local] unlock the lock");
+          service.unlock("obj");
+        }
+      });
+      thread1.start();
+      while (!startedThread1_testReleaseOrphanedGrant) {
+        Thread.yield();
+      }
+      
+      // thread to interrupt lockInterruptibly call to cause zombie grant
+      final Thread thread2 = new Thread(new Runnable() {
+        public void run() {
+          try {
+            LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Local] call lockInterruptibly");
+            startedThread2_testReleaseOrphanedGrant = true;
+            assertFalse(service.lockInterruptibly("obj", -1, -1));
+          } catch (InterruptedException expected) {
+            Thread.currentThread().interrupt();
+          }
+        }
+      });
+      thread2.start();
+      while (!startedThread2_testReleaseOrphanedGrant) {
+        Thread.yield();
+      }
+          
+      // release first thread to unlock
+      LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Local] release 1st thread");
+      sleep(500);
+      synchronized(thread1) {
+        releaseThread1_testReleaseOrphanedGrant = true;
+        thread1.notifyAll();
+      }
+      sleep(500);
+  
+      // while first thread is stuck on waitToProcessDLockResponse,
+      //   interrupt 2nd thread
+      LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Local] interrupt 2nd thread");
+      thread2.interrupt();
+      ThreadUtils.join(thread2, 20 * 1000);
+  
+      // release waitToProcessDLockResponse
+      LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Local] process lock response");
+      sleep(500);
+      DLockRequestProcessor.setWaitToProcessDLockResponse(false);
+      
+      // relock obj to make sure zombie release worked
+      LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Local] verify lock not held");
+      assertTrue(service.lock("obj", 1000, -1));
+    }
+    finally {
+      DLockRequestProcessor.setDebugReleaseOrphanedGrant(false);
+      DLockRequestProcessor.setWaitToProcessDLockResponse(false);
+    }
+  }
+  
+  static volatile Thread threadVM1_testReleaseOrphanedGrant_Remote;
+  static volatile Thread threadVM2_testReleaseOrphanedGrant_Remote;
+  static volatile boolean startedThreadVM1_testReleaseOrphanedGrant_Remote;
+  static volatile boolean releaseThreadVM1_testReleaseOrphanedGrant_Remote;
+  static volatile boolean unlockedThreadVM1_testReleaseOrphanedGrant_Remote;
+  static volatile boolean startedThreadVM2_testReleaseOrphanedGrant_Remote;
+  static volatile boolean gotLockThreadVM2_testReleaseOrphanedGrant_Remote;
+  public void testReleaseOrphanedGrant_Remote() {
+    doTestReleaseOrphanedGrant_Remote(false);
+  }
+  public void testReleaseOrphanedGrant_RemoteWithDestroy() {
+    doTestReleaseOrphanedGrant_Remote(true);
+  }
+  public void doTestReleaseOrphanedGrant_Remote(final boolean destroyLockService) {
+    final VM vm1 = Host.getHost(0).getVM(0);
+    final VM vm2 = Host.getHost(0).getVM(1);
+
+    try {
+      LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] create lock service");
+      final String serviceName = getUniqueName();
+      final DistributedLockService service = 
+        DistributedLockService.create(serviceName, dlstSystem);
+        
+      // lock and unlock to make sure this vm is grantor
+      assertTrue(service.lock("obj", -1, -1));
+      service.unlock("obj");
+      
+      // thread to get lock and wait and then unlock
+      vm1.invokeAsync(new SerializableRunnable() {
+        public void run() {
+          LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] get the lock");
+          threadVM1_testReleaseOrphanedGrant_Remote = Thread.currentThread();
+          connectDistributedSystem();
+          DistributedLockService service_vm1 =
+            DistributedLockService.create(serviceName, getSystem());
+          assertTrue(service_vm1.lock("obj", -1, -1));
+          synchronized(threadVM1_testReleaseOrphanedGrant_Remote) {
+            while (!releaseThreadVM1_testReleaseOrphanedGrant_Remote) {
+              try {
+                startedThreadVM1_testReleaseOrphanedGrant_Remote = true;
+                Thread.currentThread().wait();
+              } catch (InterruptedException ignore) {fail("interrupted");}
+            }
+          }
+          LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] unlock the lock");
+          service_vm1.unlock("obj");
+          unlockedThreadVM1_testReleaseOrphanedGrant_Remote = true;
+        }
+      });
+      vm1.invoke(new SerializableRunnable() {
+        public void run() {
+          while (!startedThreadVM1_testReleaseOrphanedGrant_Remote) {
+            Thread.yield();
+          }
+        }
+      });
+      sleep(500);
+      
+      // thread to interrupt lockInterruptibly call to cause zombie grant
+      vm2.invokeAsync(new SerializableRunnable() {
+        public void run() {
+          LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] call lockInterruptibly");
+          threadVM2_testReleaseOrphanedGrant_Remote = Thread.currentThread();
+          DistributedLockService service_vm2 =
+            DistributedLockService.create(serviceName, getSystem());
+          startedThreadVM2_testReleaseOrphanedGrant_Remote = true;
+          try {
+            DLockRequestProcessor.setDebugReleaseOrphanedGrant(true);
+            DLockRequestProcessor.setWaitToProcessDLockResponse(true);
+            assertFalse(service_vm2.lockInterruptibly("obj", -1, -1));
+          } catch (InterruptedException expected) {Thread.currentThread().interrupt();}
+        }
+      });
+      vm2.invoke(new SerializableRunnable() {
+        public void run() {
+          while (!startedThreadVM2_testReleaseOrphanedGrant_Remote) {
+            Thread.yield();
+          }
+        }
+      });
+      sleep(500);
+          
+      // release first thread to unlock
+      vm1.invoke(new SerializableRunnable() {
+        public void run() {
+          LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] release 1st thread");
+          synchronized(threadVM1_testReleaseOrphanedGrant_Remote) {
+            releaseThreadVM1_testReleaseOrphanedGrant_Remote = true;
+            threadVM1_testReleaseOrphanedGrant_Remote.notifyAll();
+          }
+        }
+      });
+      sleep(500); // lock is being released, grantor will grant lock to vm2
+  
+      // while first thread is stuck on waitToProcessDLockResponse,
+      //   interrupt 2nd thread
+      vm2.invoke(new SerializableRunnable() {
+        public void run() {
+          LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] interrupt 2nd thread");
+          threadVM2_testReleaseOrphanedGrant_Remote.interrupt();
+          ThreadUtils.join(threadVM2_testReleaseOrphanedGrant_Remote, 
+              5 * 60 * 1000);
+          if (destroyLockService) {
+            LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] destroy lock service");
+            DistributedLockService.destroy(serviceName);
+            assertNull(DistributedLockService.getServiceNamed(serviceName));
+          }
+        }
+      });
+      sleep(500); // grant is blocked while reply processor is being destroyed
+  
+      // release waitToProcessDLockResponse
+      vm2.invoke(new SerializableRunnable() {
+        public void run() {
+          LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] process lock response");
+          DLockRequestProcessor.setWaitToProcessDLockResponse(false);
+        }
+      });
+      sleep(500); // process grant and send zombie release to grantor
+      
+      // relock obj to make sure zombie release worked
+      LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] verify lock not held");
+      assertTrue(service.lock("obj", 1000, -1));
+    }
+    finally {
+      vm2.invoke(new SerializableRunnable() {
+        public void run() {
+          LogWriterUtils.getLogWriter().info("[testReleaseOrphanedGrant_Remote] clean up DebugReleaseOrphanedGrant");
+          DLockRequestProcessor.setDebugReleaseOrphanedGrant(false);
+          DLockRequestProcessor.setWaitToProcessDLockResponse(false);
+        }
+      });
+    }
+  }
+  
+  public void testDestroyLockServiceAfterGrantResponse() throws Throwable {
+    Host host = Host.getHost(0);
+    VM vm0 = host.getVM(0);
+    
+    final String serviceName = getUniqueName();
+
+    vm0.invoke(new SerializableRunnable("Create the grantor") {
+      
+      public void run() {
+        connectDistributedSystem();
+        final DistributedLockService service = 
+          DistributedLockService.create(serviceName, dlstSystem);
+          
+        // lock and unlock to make sure this vm is grantor
+        assertTrue(service.lock("obj", -1, -1));
+        service.unlock("obj");
+      }
+    });
+    
+    DistributionMessageObserver.setInstance(new DistributionMessageObserver() {
+
+      @Override
+      public void beforeProcessMessage(DistributionManager dm,
+          DistributionMessage message) {
+        if(message instanceof DLockResponseMessage) {
+          DistributedLockService.destroy(serviceName);  
+        }
+      }
+    });
+
+    connectDistributedSystem();
+    final DistributedLockService service = 
+      DistributedLockService.create(serviceName, dlstSystem);
+    try {
+      service.lock("obj", -1, -1);
+      fail("The lock service should have been destroyed");
+    } catch(LockServiceDestroyedException expected) {
+      //Do nothing
+    }
+
+    vm0.invoke(new SerializableRunnable("check to make sure the lock is not orphaned") {
+
+      public void run() {
+        final DistributedLockService service = 
+          DistributedLockService.getServiceNamed(serviceName);
+
+        // lock and unlock to make sure this vm is grantor
+        assertTrue(service.lock("obj", -1, -1));
+        service.unlock("obj");


<TRUNCATED>


Mime
View raw message