manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1560796 - in /manifoldcf/branches/CONNECTORS-867/framework/core/src: main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java test/java/org/apache/manifoldcf/core/lockmanager/TestZooKeeperLocks.java
Date Thu, 23 Jan 2014 20:12:20 GMT
Author: kwright
Date: Thu Jan 23 20:12:20 2014
New Revision: 1560796

URL: http://svn.apache.org/r1560796
Log:
Debug zookeeper no-busy-wait locks

Modified:
    manifoldcf/branches/CONNECTORS-867/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java
    manifoldcf/branches/CONNECTORS-867/framework/core/src/test/java/org/apache/manifoldcf/core/lockmanager/TestZooKeeperLocks.java

Modified: manifoldcf/branches/CONNECTORS-867/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-867/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java?rev=1560796&r1=1560795&r2=1560796&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-867/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java
(original)
+++ manifoldcf/branches/CONNECTORS-867/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java
Thu Jan 23 20:12:20 2014
@@ -212,7 +212,6 @@ public class ZooKeeperConnection
           }
         }
       }
-      System.out.println("...done");
     }
     catch (KeeperException e)
     {
@@ -251,28 +250,55 @@ public class ZooKeeperConnection
     {
       // Assert that we want a write lock
       lockNode = createSequentialChild(lockPath,WRITE_PREFIX);
-      String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 + WRITE_PREFIX.length());
-      // See if we got it
-      List<String> children = zookeeper.getChildren(lockPath,false);
-      for (String x : children)
-      {
-        String otherLock;
-        if (x.startsWith(WRITE_PREFIX))
-          otherLock = x.substring(WRITE_PREFIX.length());
-        else if (x.startsWith(NONEXWRITE_PREFIX))
-          otherLock = x.substring(NONEXWRITE_PREFIX.length());
-        else if (x.startsWith(READ_PREFIX))
-          otherLock = x.substring(READ_PREFIX.length());
-        else
-          continue;
-        if (otherLock.compareTo(lockSequenceNumber) < 0)
+      try
+      {
+        String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 + WRITE_PREFIX.length());
+        // See if we got it
+        List<String> children = zookeeper.getChildren(lockPath,false);
+        for (String x : children)
         {
-          // We didn't get the lock.  Clean up and exit
-          zookeeper.delete(lockNode,-1);
-          lockNode = null;
-          return false;
+          String otherLock;
+          if (x.startsWith(WRITE_PREFIX))
+            otherLock = x.substring(WRITE_PREFIX.length());
+          else if (x.startsWith(NONEXWRITE_PREFIX))
+            otherLock = x.substring(NONEXWRITE_PREFIX.length());
+          else if (x.startsWith(READ_PREFIX))
+            otherLock = x.substring(READ_PREFIX.length());
+          else
+            continue;
+          if (otherLock.compareTo(lockSequenceNumber) < 0)
+          {
+            // We didn't get the lock.  Clean up and exit
+            zookeeper.delete(lockNode,-1);
+            lockNode = null;
+            return false;
+          }
         }
       }
+      catch (KeeperException e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (InterruptedException e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (Error e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (RuntimeException e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
       // We got it!
       return true;
     }
@@ -288,12 +314,102 @@ public class ZooKeeperConnection
   public void obtainWriteLock(String lockPath)
     throws ManifoldCFException, InterruptedException
   {
-    // MHL to do this the proper ZooKeeper way
-    while (true)
+    if (lockNode != null)
+      throw new IllegalStateException("Already have a lock in place: '"+lockNode+"'; can't
also write lock '"+lockPath+"'");
+
+    try
     {
-      if (obtainWriteLockNoWait(lockPath))
-        return;
-      ManifoldCF.sleep(10L);
+      // Assert that we want a write lock
+      lockNode = createSequentialChild(lockPath,WRITE_PREFIX);
+      try
+      {
+        long lockSequenceNumber = new Long(lockNode.substring(lockPath.length() + 1 + WRITE_PREFIX.length())).longValue();
+        //System.out.println("Trying to get write lock for '"+lockSequenceNumber+"'");
+        while (true)
+        {
+          //System.out.println("Assessing whether we got lock for '"+lockNode+"'...");
+          // See if we got it
+          List<String> children = zookeeper.getChildren(lockPath,false);
+          String previousLock = null;
+          boolean gotLock = true;
+          long highestPreviousLockIndex = -1L;
+          for (String x : children)
+          {
+            String otherLock;
+            if (x.startsWith(WRITE_PREFIX))
+              otherLock = x.substring(WRITE_PREFIX.length());
+            else if (x.startsWith(NONEXWRITE_PREFIX))
+              otherLock = x.substring(NONEXWRITE_PREFIX.length());
+            else if (x.startsWith(READ_PREFIX))
+              otherLock = x.substring(READ_PREFIX.length());
+            else
+              continue;
+            long otherLockSequenceNumber = new Long(otherLock).longValue();
+            //System.out.println("Saw other child sequence number "+otherLockSequenceNumber);
+            if (otherLockSequenceNumber < lockSequenceNumber)
+            {
+              // We didn't get the lock.  But keep going because we're looking for the node
right before the
+              // one we just asserted.
+              gotLock = false;
+              if (otherLockSequenceNumber > highestPreviousLockIndex)
+              {
+                previousLock = x;
+                highestPreviousLockIndex = otherLockSequenceNumber;
+              }
+            }
+          }
+
+          if (gotLock)
+          {
+            // We got it!
+            //System.out.println("Got write lock for '"+lockSequenceNumber+"'");
+            return;
+          }
+
+          // There SHOULD be a previous node immediately prior to the one we asserted.  If
we didn't find one, go back around;
+          // the previous lock was probably created and destroyed before we managed to get
the children.
+          if (previousLock != null)
+          {
+            //System.out.println(" Waiting on '"+previousLock+"' for write lock '"+lockSequenceNumber+"'");
+            // Create an exists() watch on the previous node, and wait until we are awakened
by that watch firing.
+            ExistsWatcher w = new ExistsWatcher();
+            Stat s = zookeeper.exists(lockPath+"/"+previousLock, w);
+            if (s != null)
+              w.waitForEvent();
+          }
+          //else
+          //  System.out.println(" Retrying for write lock '"+lockSequenceNumber+"'");
+        }
+      }
+      catch (KeeperException e)
+      {
+        //System.out.println("Unexpected keeper exception: "+e.getMessage());
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (InterruptedException e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (Error e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (RuntimeException e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+    }
+    catch (KeeperException e)
+    {
+      throw new ManifoldCFException(e.getMessage(),e);
     }
   }
 
@@ -311,26 +427,53 @@ public class ZooKeeperConnection
     {
       // Assert that we want a read lock
       lockNode = createSequentialChild(lockPath,NONEXWRITE_PREFIX);
-      String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 + NONEXWRITE_PREFIX.length());
-      // See if we got it
-      List<String> children = zookeeper.getChildren(lockPath,false);
-      for (String x : children)
-      {
-        String otherLock;
-        if (x.startsWith(WRITE_PREFIX))
-          otherLock = x.substring(WRITE_PREFIX.length());
-        else if (x.startsWith(READ_PREFIX))
-          otherLock = x.substring(READ_PREFIX.length());
-        else
-          continue;
-        if (otherLock.compareTo(lockSequenceNumber) < 0)
+      try
+      {
+        String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 + NONEXWRITE_PREFIX.length());
+        // See if we got it
+        List<String> children = zookeeper.getChildren(lockPath,false);
+        for (String x : children)
         {
-          // We didn't get the lock.  Clean up and exit
-          zookeeper.delete(lockNode,-1);
-          lockNode = null;
-          return false;
+          String otherLock;
+          if (x.startsWith(WRITE_PREFIX))
+            otherLock = x.substring(WRITE_PREFIX.length());
+          else if (x.startsWith(READ_PREFIX))
+            otherLock = x.substring(READ_PREFIX.length());
+          else
+            continue;
+          if (otherLock.compareTo(lockSequenceNumber) < 0)
+          {
+            // We didn't get the lock.  Clean up and exit
+            zookeeper.delete(lockNode,-1);
+            lockNode = null;
+            return false;
+          }
         }
       }
+      catch (KeeperException e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (InterruptedException e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (Error e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (RuntimeException e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
       // We got it!
       return true;
     }
@@ -346,12 +489,91 @@ public class ZooKeeperConnection
   public void obtainNonExWriteLock(String lockPath)
     throws ManifoldCFException, InterruptedException
   {
-    // MHL to do this the proper ZooKeeper way
-    while (true)
+    if (lockNode != null)
+      throw new IllegalStateException("Already have a lock in place: '"+lockNode+"'; can't
also non-ex write lock '"+lockPath+"'");
+
+    try
     {
-      if (obtainNonExWriteLockNoWait(lockPath))
-        return;
-      ManifoldCF.sleep(10L);
+      // Assert that we want a read lock
+      lockNode = createSequentialChild(lockPath,NONEXWRITE_PREFIX);
+      try
+      {
+        long lockSequenceNumber = new Long(lockNode.substring(lockPath.length() + 1 + NONEXWRITE_PREFIX.length())).longValue();
+        while (true)
+        {
+          // See if we got it
+          List<String> children = zookeeper.getChildren(lockPath,false);
+          String previousLock = null;
+          boolean gotLock = true;
+          long highestPreviousLockIndex = -1L;
+          for (String x : children)
+          {
+            String otherLock;
+            if (x.startsWith(WRITE_PREFIX))
+              otherLock = x.substring(WRITE_PREFIX.length());
+            else if (x.startsWith(READ_PREFIX))
+              otherLock = x.substring(READ_PREFIX.length());
+            else
+              continue;
+            long otherLockSequenceNumber = new Long(otherLock).longValue();
+            //System.out.println("Saw other child sequence number "+otherLockSequenceNumber);
+            if (otherLockSequenceNumber < lockSequenceNumber)
+            {
+              // We didn't get the lock.  But keep going because we're looking for the node
right before the
+              // one we just asserted.
+              gotLock = false;
+              if (otherLockSequenceNumber > highestPreviousLockIndex)
+              {
+                previousLock = x;
+                highestPreviousLockIndex = otherLockSequenceNumber;
+              }
+            }
+          }
+          
+          if (gotLock)
+            // We got it!
+            return;
+
+          // There SHOULD be a previous node immediately prior to the one we asserted.  If
we didn't find one, go back around;
+          // the previous lock was probably created and destroyed before we managed to get
the children.
+          if (previousLock != null)
+          {
+            // Create an exists() watch on the previous node, and wait until we are awakened
by that watch firing.
+            ExistsWatcher w = new ExistsWatcher();
+            Stat s = zookeeper.exists(lockPath+"/"+previousLock, w);
+            if (s != null)
+              w.waitForEvent();
+          }
+        }
+      }
+      catch (KeeperException e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (InterruptedException e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (Error e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (RuntimeException e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+    }
+    catch (KeeperException e)
+    {
+      throw new ManifoldCFException(e.getMessage(),e);
     }
   }
 
@@ -369,26 +591,53 @@ public class ZooKeeperConnection
     {
       // Assert that we want a read lock
       lockNode = createSequentialChild(lockPath,READ_PREFIX);
-      String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 + READ_PREFIX.length());
-      // See if we got it
-      List<String> children = zookeeper.getChildren(lockPath,false);
-      for (String x : children)
-      {
-        String otherLock;
-        if (x.startsWith(WRITE_PREFIX))
-          otherLock = x.substring(WRITE_PREFIX.length());
-        else if (x.startsWith(NONEXWRITE_PREFIX))
-          otherLock = x.substring(NONEXWRITE_PREFIX.length());
-        else
-          continue;
-        if (otherLock.compareTo(lockSequenceNumber) < 0)
+      try
+      {
+        String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 + READ_PREFIX.length());
+        // See if we got it
+        List<String> children = zookeeper.getChildren(lockPath,false);
+        for (String x : children)
         {
-          // We didn't get the lock.  Clean up and exit
-          zookeeper.delete(lockNode,-1);
-          lockNode = null;
-          return false;
+          String otherLock;
+          if (x.startsWith(WRITE_PREFIX))
+            otherLock = x.substring(WRITE_PREFIX.length());
+          else if (x.startsWith(NONEXWRITE_PREFIX))
+            otherLock = x.substring(NONEXWRITE_PREFIX.length());
+          else
+            continue;
+          if (otherLock.compareTo(lockSequenceNumber) < 0)
+          {
+            // We didn't get the lock.  Clean up and exit
+            zookeeper.delete(lockNode,-1);
+            lockNode = null;
+            return false;
+          }
         }
       }
+      catch (KeeperException e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (InterruptedException e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (Error e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (RuntimeException e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
       // We got it!
       return true;
     }
@@ -404,12 +653,100 @@ public class ZooKeeperConnection
   public void obtainReadLock(String lockPath)
     throws ManifoldCFException, InterruptedException
   {
-    // MHL to do this the proper ZooKeeper way
-    while (true)
+    if (lockNode != null)
+      throw new IllegalStateException("Already have a lock in place: '"+lockNode+"'; can't
also read lock '"+lockPath+"'");
+
+    try
+    {
+      // Assert that we want a read lock
+      lockNode = createSequentialChild(lockPath,READ_PREFIX);
+      try
+      {
+        long lockSequenceNumber = new Long(lockNode.substring(lockPath.length() + 1 + READ_PREFIX.length())).longValue();
+        //System.out.println("Trying to get read lock for '"+lockSequenceNumber+"'");
+        while (true)
+        {
+          // See if we got it
+          List<String> children = zookeeper.getChildren(lockPath,false);
+          String previousLock = null;
+          boolean gotLock = true;
+          long highestPreviousLockIndex = -1L;
+          for (String x : children)
+          {
+            String otherLock;
+            if (x.startsWith(WRITE_PREFIX))
+              otherLock = x.substring(WRITE_PREFIX.length());
+            else if (x.startsWith(NONEXWRITE_PREFIX))
+              otherLock = x.substring(NONEXWRITE_PREFIX.length());
+            else
+              continue;
+            long otherLockSequenceNumber = new Long(otherLock).longValue();
+            //System.out.println("Saw other child sequence number "+otherLockSequenceNumber);
+            if (otherLockSequenceNumber < lockSequenceNumber)
+            {
+              // We didn't get the lock.  But keep going because we're looking for the node
right before the
+              // one we just asserted.
+              gotLock = false;
+              if (otherLockSequenceNumber > highestPreviousLockIndex)
+              {
+                previousLock = x;
+                highestPreviousLockIndex = otherLockSequenceNumber;
+              }
+            }
+          }
+          
+          if (gotLock)
+          {
+            // We got it!
+            //System.out.println("Got read lock for '"+lockSequenceNumber+"'");
+            return;
+          }
+
+          // There SHOULD be a previous node immediately prior to the one we asserted.  If
we didn't find one, go back around;
+          // the previous lock was probably created and destroyed before we managed to get
the children.
+          if (previousLock != null)
+          {
+            //System.out.println(" Waiting on '"+previousLock+"' for read lock '"+lockSequenceNumber+"'");
+            // Create an exists() watch on the previous node, and wait until we are awakened
by that watch firing.
+            ExistsWatcher w = new ExistsWatcher();
+            Stat s = zookeeper.exists(lockPath+"/"+previousLock, w);
+            if (s != null)
+              w.waitForEvent();
+          }
+          //else
+          //  System.out.println(" Retrying for read lock '"+lockSequenceNumber+"'");
+
+        }
+      }
+      catch (KeeperException e)
+      {
+        //System.out.println("Unexpected keeper exception: "+e.getMessage());
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (InterruptedException e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (Error e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+      catch (RuntimeException e)
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        throw e;
+      }
+    }
+    catch (KeeperException e)
     {
-      if (obtainReadLockNoWait(lockPath))
-        return;
-      ManifoldCF.sleep(10L);
+      throw new ManifoldCFException(e.getMessage(),e);
     }
   }
   
@@ -420,6 +757,7 @@ public class ZooKeeperConnection
   {
     if (lockNode == null)
       throw new IllegalStateException("Can't release lock we don't hold");
+    //System.out.println("Releasing lock '"+lockNode+"'");
     try
     {
       zookeeper.delete(lockNode,-1);
@@ -647,10 +985,43 @@ public class ZooKeeperConnection
     {
     }
     
+    @Override
+    public void process(WatchedEvent event)
+    {
+    }
+
+  }
+
+  /** Watcher class for exists state changes, so we get notified about deletions of lock
request nodes. */
+  protected static class ExistsWatcher implements Watcher
+  {
+    protected boolean eventTriggered = false;
+
+    public ExistsWatcher()
+    {
+    }
+    
+    @Override
     public void process(WatchedEvent event)
     {
+      synchronized (this)
+      {
+        eventTriggered = true;
+        notifyAll();
+      }
     }
 
+    public void waitForEvent()
+      throws InterruptedException
+    {
+      synchronized (this)
+      {
+        if (eventTriggered)
+          return;
+        wait();
+      }
+    }
+    
   }
 
 }

Modified: manifoldcf/branches/CONNECTORS-867/framework/core/src/test/java/org/apache/manifoldcf/core/lockmanager/TestZooKeeperLocks.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/CONNECTORS-867/framework/core/src/test/java/org/apache/manifoldcf/core/lockmanager/TestZooKeeperLocks.java?rev=1560796&r1=1560795&r2=1560796&view=diff
==============================================================================
--- manifoldcf/branches/CONNECTORS-867/framework/core/src/test/java/org/apache/manifoldcf/core/lockmanager/TestZooKeeperLocks.java
(original)
+++ manifoldcf/branches/CONNECTORS-867/framework/core/src/test/java/org/apache/manifoldcf/core/lockmanager/TestZooKeeperLocks.java
Thu Jan 23 20:12:20 2014
@@ -287,14 +287,22 @@ public class TestZooKeeperLocks extends 
         // LockPool is a dummy
         LockPool lp = new LockPool(factory);
         LockGate lo;
-        // Take write locks but free them if read is what's active
+        // The reader threads require ALL of the readers to get into the protected area.
 If
+        // we try to write before that happens, ordering requirements produce a deadlock.
 So wait.
+        while (ai.get() < readerThreadCount)
+        {
+          Thread.sleep(100L);
+        }
+        
+        /*
+        // Take write locks but free them immediately if read is what's active
         while (true)
         {
           lo = lp.getObject(lockKey);
           enterWriteLock(threadID,lo);
-          System.out.println("Made it into write lock");
           try
           {
+            System.out.println("Made it into read-time write lock");
             // Check if we made it in during read cycle... that would be bad.
             if (ai.get() > 0 && ai.get() < readerThreadCount)
               throw new Exception("Was able to write even when readers were active");
@@ -303,12 +311,15 @@ public class TestZooKeeperLocks extends 
           }
           finally
           {
+            System.out.println("Leaving read-time write lock");
             leaveWriteLock(lo);
+            System.out.println("Left read-time write lock");
           }
-          Thread.sleep(100L);
+          Thread.sleep(1000L);
         }
+        */
         
-        // Get write lock, increment twice, and leave write lock
+        // Get write lock, increment twice, and leave write lock.  Meanwhile, reader threads
will be trying to gain access.
         lo = lp.getObject(lockKey);
         enterWriteLock(threadID,lo);
         try



Mime
View raw message