manifoldcf-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kwri...@apache.org
Subject svn commit: r1570364 - in /manifoldcf/trunk: CHANGES.txt framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java
Date Thu, 20 Feb 2014 22:19:46 GMT
Author: kwright
Date: Thu Feb 20 22:19:46 2014
New Revision: 1570364

URL: http://svn.apache.org/r1570364
Log:
Fix for CONNECTORS-898

Modified:
    manifoldcf/trunk/CHANGES.txt
    manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java

Modified: manifoldcf/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/CHANGES.txt?rev=1570364&r1=1570363&r2=1570364&view=diff
==============================================================================
--- manifoldcf/trunk/CHANGES.txt (original)
+++ manifoldcf/trunk/CHANGES.txt Thu Feb 20 22:19:46 2014
@@ -3,6 +3,10 @@ $Id$
 
 ======================= 1.6-dev =====================
 
+CONNECTORS-898: Handle zookeeper connection loss and session
+expiration conditions.
+(Graeme Seaton, Karl Wright)
+
 CONNECTORS-886: Add support for Parent folder security
 in SharedDriveConnector.
 (Shinichiro Abe)

Modified: manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java
URL: http://svn.apache.org/viewvc/manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java?rev=1570364&r1=1570363&r2=1570364&view=diff
==============================================================================
--- manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java
(original)
+++ manifoldcf/trunk/framework/core/src/main/java/org/apache/manifoldcf/core/lockmanager/ZooKeeperConnection.java
Thu Feb 20 22:19:46 2014
@@ -44,6 +44,9 @@ public class ZooKeeperConnection
 
   private static final String CHILD_PREFIX = "child-";
   
+  protected final String connectString;
+  protected final int sessionTimeout;
+  
   // Our zookeeper client
   protected ZooKeeper zookeeper = null;
   protected ZooKeeperWatcher zookeeperWatcher = null;
@@ -55,9 +58,17 @@ public class ZooKeeperConnection
   public ZooKeeperConnection(String connectString, int sessionTimeout)
     throws ManifoldCFException, InterruptedException
   {
+    this.connectString = connectString;
+    this.sessionTimeout = sessionTimeout;
+    zookeeperWatcher = new ZooKeeperWatcher();
+    createSession();
+  }
+  
+  protected void createSession()
+    throws ManifoldCFException, InterruptedException
+  {
     try
     {
-      zookeeperWatcher = new ZooKeeperWatcher();
       zookeeper = new ZooKeeper(connectString, sessionTimeout, zookeeperWatcher);
     }
     catch (InterruptedIOException e)
@@ -75,13 +86,17 @@ public class ZooKeeperConnection
   public void createNode(String nodePath, byte[] nodeData)
     throws ManifoldCFException, InterruptedException
   {
-    try
-    {
-      zookeeper.create(nodePath, nodeData, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
-    }
-    catch (KeeperException e)
+    while (true)
     {
-      throw new ManifoldCFException(e.getMessage(),e);
+      try
+      {
+        zookeeper.create(nodePath, nodeData, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
+        break;
+      }
+      catch (KeeperException e)
+      {
+        handleKeeperException(e);
+      }
     }
   }
   
@@ -92,13 +107,16 @@ public class ZooKeeperConnection
   public boolean checkNodeExists(String nodePath)
     throws ManifoldCFException, InterruptedException
   {
-    try
-    {
-      return (zookeeper.exists(nodePath,false) != null);
-    }
-    catch (KeeperException e)
+    while (true)
     {
-      throw new ManifoldCFException(e.getMessage(),e);
+      try
+      {
+        return (zookeeper.exists(nodePath,false) != null);
+      }
+      catch (KeeperException e)
+      {
+        handleKeeperException(e);
+      }
     }
   }
 
@@ -125,13 +143,17 @@ public class ZooKeeperConnection
   public void deleteNode(String nodePath)
     throws ManifoldCFException, InterruptedException
   {
-    try
-    {
-      zookeeper.delete(nodePath,-1);
-    }
-    catch (KeeperException e)
+    while (true)
     {
-      throw new ManifoldCFException(e.getMessage(),e);
+      try
+      {
+        zookeeper.delete(nodePath,-1);
+        return;
+      }
+      catch (KeeperException e)
+      {
+        handleKeeperException(e);
+      }
     }
   }
   
@@ -140,20 +162,25 @@ public class ZooKeeperConnection
   public void deleteNodeChildren(String nodePath)
     throws ManifoldCFException, InterruptedException
   {
-    try
+    while (true)
     {
-      List<String> children = zookeeper.getChildren(nodePath,false);
-      for (String child : children)
+      try
       {
-        zookeeper.delete(nodePath + "/" + child,-1);
+        List<String> children = zookeeper.getChildren(nodePath,false);
+        for (String child : children)
+        {
+          zookeeper.delete(nodePath + "/" + child,-1);
+        }
+        break;
+      }
+      catch (KeeperException.NoNodeException e)
+      {
+        break;
+      }
+      catch (KeeperException e)
+      {
+        handleKeeperException(e);
       }
-    }
-    catch (KeeperException.NoNodeException e)
-    {
-    }
-    catch (KeeperException e)
-    {
-      throw new ManifoldCFException(e.getMessage(),e);
     }
   }
   
@@ -163,26 +190,29 @@ public class ZooKeeperConnection
   public List<String> getChildren(String nodePath)
     throws ManifoldCFException, InterruptedException
   {
-    try
+    while (true)
     {
-      //System.out.println("Children of '"+nodePath+"':");
-      List<String> children = zookeeper.getChildren(nodePath,false);
-      List<String> rval = new ArrayList<String>();
-      for (String child : children)
+      try
       {
-        //System.out.println(" '"+child+"'");
-        if (child.startsWith(CHILD_PREFIX))
-          rval.add(child.substring(CHILD_PREFIX.length()));
+        //System.out.println("Children of '"+nodePath+"':");
+        List<String> children = zookeeper.getChildren(nodePath,false);
+        List<String> rval = new ArrayList<String>();
+        for (String child : children)
+        {
+          //System.out.println(" '"+child+"'");
+          if (child.startsWith(CHILD_PREFIX))
+            rval.add(child.substring(CHILD_PREFIX.length()));
+        }
+        return rval;
+      }
+      catch (KeeperException.NoNodeException e)
+      {
+        return new ArrayList<String>();
+      }
+      catch (KeeperException e)
+      {
+        handleKeeperException(e);
       }
-      return rval;
-    }
-    catch (KeeperException.NoNodeException e)
-    {
-      return new ArrayList<String>();
-    }
-    catch (KeeperException e)
-    {
-      throw new ManifoldCFException(e.getMessage(),e);
     }
   }
   
@@ -191,31 +221,35 @@ public class ZooKeeperConnection
   public void createChild(String nodePath, String childName)
     throws ManifoldCFException, InterruptedException
   {
-    try
+    while (true)
     {
-      //System.out.println("Creating child '"+childName+"' of nodepath '"+nodePath+"'");
-      while (true)
+      try
       {
-        try
-        {
-          zookeeper.create(nodePath + "/" + CHILD_PREFIX + childName, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
-          break;
-        }
-        catch (KeeperException.NoNodeException e)
+        //System.out.println("Creating child '"+childName+"' of nodepath '"+nodePath+"'");
+        while (true)
         {
           try
           {
-            zookeeper.create(nodePath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+            zookeeper.create(nodePath + "/" + CHILD_PREFIX + childName, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
+            break;
           }
-          catch (KeeperException.NodeExistsException e2)
+          catch (KeeperException.NoNodeException e)
           {
+            try
+            {
+              zookeeper.create(nodePath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+            }
+            catch (KeeperException.NodeExistsException e2)
+            {
+            }
           }
         }
+        break;
+      }
+      catch (KeeperException e)
+      {
+        handleKeeperException(e);
       }
-    }
-    catch (KeeperException e)
-    {
-      throw new ManifoldCFException(e.getMessage(),e);
     }
   }
   
@@ -224,15 +258,19 @@ public class ZooKeeperConnection
   public void deleteChild(String nodePath, String childName)
     throws ManifoldCFException, InterruptedException
   {
-    try
-    {
-      //System.out.println("Deleting child '"+childName+"' of nodePath '"+nodePath+"'");
-      zookeeper.delete(nodePath + "/" + CHILD_PREFIX + childName, -1);
-      //System.out.println("...done");
-    }
-    catch (KeeperException e)
+    while (true)
     {
-      throw new ManifoldCFException(e.getMessage(),e);
+      try
+      {
+        //System.out.println("Deleting child '"+childName+"' of nodePath '"+nodePath+"'");
+        zookeeper.delete(nodePath + "/" + CHILD_PREFIX + childName, -1);
+        //System.out.println("...done");
+        break;
+      }
+      catch (KeeperException e)
+      {
+        handleKeeperException(e);
+      }
     }
   }
   
@@ -246,65 +284,68 @@ public class ZooKeeperConnection
     if (lockNode != null)
       throw new IllegalStateException("Already have a lock in place: '"+lockNode+"'; can't
also write lock '"+lockPath+"'");
 
-    try
+    while (true)
     {
-      // Assert that we want a write lock
-      lockNode = createSequentialChild(lockPath,WRITE_PREFIX);
       try
       {
-        String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 + WRITE_PREFIX.length());
-        // See if we got it
-        List<String> children = zookeeper.getChildren(lockPath,false);
-        for (String x : children)
-        {
-          String otherLock;
-          if (x.startsWith(WRITE_PREFIX))
-            otherLock = x.substring(WRITE_PREFIX.length());
-          else if (x.startsWith(NONEXWRITE_PREFIX))
-            otherLock = x.substring(NONEXWRITE_PREFIX.length());
-          else if (x.startsWith(READ_PREFIX))
-            otherLock = x.substring(READ_PREFIX.length());
-          else
-            continue;
-          if (otherLock.compareTo(lockSequenceNumber) < 0)
-          {
-            // We didn't get the lock.  Clean up and exit
-            zookeeper.delete(lockNode,-1);
-            lockNode = null;
-            return false;
+        // Assert that we want a write lock
+        lockNode = createSequentialChild(lockPath,WRITE_PREFIX);
+        try
+        {
+          String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 + WRITE_PREFIX.length());
+          // See if we got it
+          List<String> children = zookeeper.getChildren(lockPath,false);
+          for (String x : children)
+          {
+            String otherLock;
+            if (x.startsWith(WRITE_PREFIX))
+              otherLock = x.substring(WRITE_PREFIX.length());
+            else if (x.startsWith(NONEXWRITE_PREFIX))
+              otherLock = x.substring(NONEXWRITE_PREFIX.length());
+            else if (x.startsWith(READ_PREFIX))
+              otherLock = x.substring(READ_PREFIX.length());
+            else
+              continue;
+            if (otherLock.compareTo(lockSequenceNumber) < 0)
+            {
+              // We didn't get the lock.  Clean up and exit
+              zookeeper.delete(lockNode,-1);
+              lockNode = null;
+              return false;
+            }
           }
         }
+        catch (KeeperException e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (InterruptedException e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (Error e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (RuntimeException e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        // We got it!
+        return true;
       }
       catch (KeeperException e)
       {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-      catch (InterruptedException e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
+        handleKeeperException(e);
       }
-      catch (Error e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-      catch (RuntimeException e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-      // We got it!
-      return true;
-    }
-    catch (KeeperException e)
-    {
-      throw new ManifoldCFException(e.getMessage(),e);
     }
   }
   
@@ -317,100 +358,103 @@ public class ZooKeeperConnection
     if (lockNode != null)
       throw new IllegalStateException("Already have a lock in place: '"+lockNode+"'; can't
also write lock '"+lockPath+"'");
 
-    try
+    while (true)
     {
-      // Assert that we want a write lock
-      lockNode = createSequentialChild(lockPath,WRITE_PREFIX);
       try
       {
-        long lockSequenceNumber = new Long(lockNode.substring(lockPath.length() + 1 + WRITE_PREFIX.length())).longValue();
-        //System.out.println("Trying to get write lock for '"+lockSequenceNumber+"'");
-        while (true)
+        // Assert that we want a write lock
+        lockNode = createSequentialChild(lockPath,WRITE_PREFIX);
+        try
         {
-          //System.out.println("Assessing whether we got lock for '"+lockNode+"'...");
-          // See if we got it
-          List<String> children = zookeeper.getChildren(lockPath,false);
-          String previousLock = null;
-          boolean gotLock = true;
-          long highestPreviousLockIndex = -1L;
-          for (String x : children)
+          long lockSequenceNumber = new Long(lockNode.substring(lockPath.length() + 1 + WRITE_PREFIX.length())).longValue();
+          //System.out.println("Trying to get write lock for '"+lockSequenceNumber+"'");
+          while (true)
           {
-            String otherLock;
-            if (x.startsWith(WRITE_PREFIX))
-              otherLock = x.substring(WRITE_PREFIX.length());
-            else if (x.startsWith(NONEXWRITE_PREFIX))
-              otherLock = x.substring(NONEXWRITE_PREFIX.length());
-            else if (x.startsWith(READ_PREFIX))
-              otherLock = x.substring(READ_PREFIX.length());
-            else
-              continue;
-            long otherLockSequenceNumber = new Long(otherLock).longValue();
-            //System.out.println("Saw other child sequence number "+otherLockSequenceNumber);
-            if (otherLockSequenceNumber < lockSequenceNumber)
-            {
-              // We didn't get the lock.  But keep going because we're looking for the node
right before the
-              // one we just asserted.
-              gotLock = false;
-              if (otherLockSequenceNumber > highestPreviousLockIndex)
+            //System.out.println("Assessing whether we got lock for '"+lockNode+"'...");
+            // See if we got it
+            List<String> children = zookeeper.getChildren(lockPath,false);
+            String previousLock = null;
+            boolean gotLock = true;
+            long highestPreviousLockIndex = -1L;
+            for (String x : children)
+            {
+              String otherLock;
+              if (x.startsWith(WRITE_PREFIX))
+                otherLock = x.substring(WRITE_PREFIX.length());
+              else if (x.startsWith(NONEXWRITE_PREFIX))
+                otherLock = x.substring(NONEXWRITE_PREFIX.length());
+              else if (x.startsWith(READ_PREFIX))
+                otherLock = x.substring(READ_PREFIX.length());
+              else
+                continue;
+              long otherLockSequenceNumber = new Long(otherLock).longValue();
+              //System.out.println("Saw other child sequence number "+otherLockSequenceNumber);
+              if (otherLockSequenceNumber < lockSequenceNumber)
               {
-                previousLock = x;
-                highestPreviousLockIndex = otherLockSequenceNumber;
+                // We didn't get the lock.  But keep going because we're looking for the
node right before the
+                // one we just asserted.
+                gotLock = false;
+                if (otherLockSequenceNumber > highestPreviousLockIndex)
+                {
+                  previousLock = x;
+                  highestPreviousLockIndex = otherLockSequenceNumber;
+                }
               }
             }
-          }
 
-          if (gotLock)
-          {
-            // We got it!
-            //System.out.println("Got write lock for '"+lockSequenceNumber+"'");
-            return;
-          }
+            if (gotLock)
+            {
+              // We got it!
+              //System.out.println("Got write lock for '"+lockSequenceNumber+"'");
+              return;
+            }
 
-          // There SHOULD be a previous node immediately prior to the one we asserted.  If
we didn't find one, go back around;
-          // the previous lock was probably created and destroyed before we managed to get
the children.
-          if (previousLock != null)
-          {
-            //System.out.println(" Waiting on '"+previousLock+"' for write lock '"+lockSequenceNumber+"'");
-            // Create an exists() watch on the previous node, and wait until we are awakened
by that watch firing.
-            ExistsWatcher w = new ExistsWatcher();
-            Stat s = zookeeper.exists(lockPath+"/"+previousLock, w);
-            if (s != null)
-              w.waitForEvent();
+            // There SHOULD be a previous node immediately prior to the one we asserted.
 If we didn't find one, go back around;
+            // the previous lock was probably created and destroyed before we managed to
get the children.
+            if (previousLock != null)
+            {
+              //System.out.println(" Waiting on '"+previousLock+"' for write lock '"+lockSequenceNumber+"'");
+              // Create an exists() watch on the previous node, and wait until we are awakened
by that watch firing.
+              ExistsWatcher w = new ExistsWatcher();
+              Stat s = zookeeper.exists(lockPath+"/"+previousLock, w);
+              if (s != null)
+                w.waitForEvent();
+            }
+            //else
+            //  System.out.println(" Retrying for write lock '"+lockSequenceNumber+"'");
           }
-          //else
-          //  System.out.println(" Retrying for write lock '"+lockSequenceNumber+"'");
+        }
+        catch (KeeperException e)
+        {
+          //System.out.println("Unexpected keeper exception: "+e.getMessage());
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (InterruptedException e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (Error e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (RuntimeException e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
         }
       }
       catch (KeeperException e)
       {
-        //System.out.println("Unexpected keeper exception: "+e.getMessage());
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-      catch (InterruptedException e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-      catch (Error e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-      catch (RuntimeException e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
+        handleKeeperException(e);
       }
     }
-    catch (KeeperException e)
-    {
-      throw new ManifoldCFException(e.getMessage(),e);
-    }
   }
 
   /** Obtain a non-ex-write lock, with no wait.
@@ -423,63 +467,66 @@ public class ZooKeeperConnection
     if (lockNode != null)
       throw new IllegalStateException("Already have a lock in place: '"+lockNode+"'; can't
also non-ex write lock '"+lockPath+"'");
 
-    try
+    while (true)
     {
-      // Assert that we want a read lock
-      lockNode = createSequentialChild(lockPath,NONEXWRITE_PREFIX);
       try
       {
-        String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 + NONEXWRITE_PREFIX.length());
-        // See if we got it
-        List<String> children = zookeeper.getChildren(lockPath,false);
-        for (String x : children)
-        {
-          String otherLock;
-          if (x.startsWith(WRITE_PREFIX))
-            otherLock = x.substring(WRITE_PREFIX.length());
-          else if (x.startsWith(READ_PREFIX))
-            otherLock = x.substring(READ_PREFIX.length());
-          else
-            continue;
-          if (otherLock.compareTo(lockSequenceNumber) < 0)
-          {
-            // We didn't get the lock.  Clean up and exit
-            zookeeper.delete(lockNode,-1);
-            lockNode = null;
-            return false;
+        // Assert that we want a read lock
+        lockNode = createSequentialChild(lockPath,NONEXWRITE_PREFIX);
+        try
+        {
+          String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 + NONEXWRITE_PREFIX.length());
+          // See if we got it
+          List<String> children = zookeeper.getChildren(lockPath,false);
+          for (String x : children)
+          {
+            String otherLock;
+            if (x.startsWith(WRITE_PREFIX))
+              otherLock = x.substring(WRITE_PREFIX.length());
+            else if (x.startsWith(READ_PREFIX))
+              otherLock = x.substring(READ_PREFIX.length());
+            else
+              continue;
+            if (otherLock.compareTo(lockSequenceNumber) < 0)
+            {
+              // We didn't get the lock.  Clean up and exit
+              zookeeper.delete(lockNode,-1);
+              lockNode = null;
+              return false;
+            }
           }
         }
+        catch (KeeperException e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (InterruptedException e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (Error e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (RuntimeException e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        // We got it!
+        return true;
       }
       catch (KeeperException e)
       {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-      catch (InterruptedException e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-      catch (Error e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-      catch (RuntimeException e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
+        handleKeeperException(e);
       }
-      // We got it!
-      return true;
-    }
-    catch (KeeperException e)
-    {
-      throw new ManifoldCFException(e.getMessage(),e);
     }
   }
 
@@ -492,88 +539,91 @@ public class ZooKeeperConnection
     if (lockNode != null)
       throw new IllegalStateException("Already have a lock in place: '"+lockNode+"'; can't
also non-ex write lock '"+lockPath+"'");
 
-    try
+    while (true)
     {
-      // Assert that we want a read lock
-      lockNode = createSequentialChild(lockPath,NONEXWRITE_PREFIX);
       try
       {
-        long lockSequenceNumber = new Long(lockNode.substring(lockPath.length() + 1 + NONEXWRITE_PREFIX.length())).longValue();
-        while (true)
+        // Assert that we want a read lock
+        lockNode = createSequentialChild(lockPath,NONEXWRITE_PREFIX);
+        try
         {
-          // See if we got it
-          List<String> children = zookeeper.getChildren(lockPath,false);
-          String previousLock = null;
-          boolean gotLock = true;
-          long highestPreviousLockIndex = -1L;
-          for (String x : children)
+          long lockSequenceNumber = new Long(lockNode.substring(lockPath.length() + 1 + NONEXWRITE_PREFIX.length())).longValue();
+          while (true)
           {
-            String otherLock;
-            if (x.startsWith(WRITE_PREFIX))
-              otherLock = x.substring(WRITE_PREFIX.length());
-            else if (x.startsWith(READ_PREFIX))
-              otherLock = x.substring(READ_PREFIX.length());
-            else
-              continue;
-            long otherLockSequenceNumber = new Long(otherLock).longValue();
-            //System.out.println("Saw other child sequence number "+otherLockSequenceNumber);
-            if (otherLockSequenceNumber < lockSequenceNumber)
-            {
-              // We didn't get the lock.  But keep going because we're looking for the node
right before the
-              // one we just asserted.
-              gotLock = false;
-              if (otherLockSequenceNumber > highestPreviousLockIndex)
+            // See if we got it
+            List<String> children = zookeeper.getChildren(lockPath,false);
+            String previousLock = null;
+            boolean gotLock = true;
+            long highestPreviousLockIndex = -1L;
+            for (String x : children)
+            {
+              String otherLock;
+              if (x.startsWith(WRITE_PREFIX))
+                otherLock = x.substring(WRITE_PREFIX.length());
+              else if (x.startsWith(READ_PREFIX))
+                otherLock = x.substring(READ_PREFIX.length());
+              else
+                continue;
+              long otherLockSequenceNumber = new Long(otherLock).longValue();
+              //System.out.println("Saw other child sequence number "+otherLockSequenceNumber);
+              if (otherLockSequenceNumber < lockSequenceNumber)
               {
-                previousLock = x;
-                highestPreviousLockIndex = otherLockSequenceNumber;
+                // We didn't get the lock.  But keep going because we're looking for the
node right before the
+                // one we just asserted.
+                gotLock = false;
+                if (otherLockSequenceNumber > highestPreviousLockIndex)
+                {
+                  previousLock = x;
+                  highestPreviousLockIndex = otherLockSequenceNumber;
+                }
               }
             }
+            
+            if (gotLock)
+              // We got it!
+              return;
+
+            // There SHOULD be a previous node immediately prior to the one we asserted.
 If we didn't find one, go back around;
+            // the previous lock was probably created and destroyed before we managed to
get the children.
+            if (previousLock != null)
+            {
+              // Create an exists() watch on the previous node, and wait until we are awakened
by that watch firing.
+              ExistsWatcher w = new ExistsWatcher();
+              Stat s = zookeeper.exists(lockPath+"/"+previousLock, w);
+              if (s != null)
+                w.waitForEvent();
+            }
           }
-          
-          if (gotLock)
-            // We got it!
-            return;
-
-          // There SHOULD be a previous node immediately prior to the one we asserted.  If
we didn't find one, go back around;
-          // the previous lock was probably created and destroyed before we managed to get
the children.
-          if (previousLock != null)
-          {
-            // Create an exists() watch on the previous node, and wait until we are awakened
by that watch firing.
-            ExistsWatcher w = new ExistsWatcher();
-            Stat s = zookeeper.exists(lockPath+"/"+previousLock, w);
-            if (s != null)
-              w.waitForEvent();
-          }
+        }
+        catch (KeeperException e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (InterruptedException e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (Error e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (RuntimeException e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
         }
       }
       catch (KeeperException e)
       {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
+        handleKeeperException(e);
       }
-      catch (InterruptedException e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-      catch (Error e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-      catch (RuntimeException e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-    }
-    catch (KeeperException e)
-    {
-      throw new ManifoldCFException(e.getMessage(),e);
     }
   }
 
@@ -587,63 +637,66 @@ public class ZooKeeperConnection
     if (lockNode != null)
       throw new IllegalStateException("Already have a lock in place: '"+lockNode+"'; can't
also read lock '"+lockPath+"'");
 
-    try
+    while (true)
     {
-      // Assert that we want a read lock
-      lockNode = createSequentialChild(lockPath,READ_PREFIX);
       try
       {
-        String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 + READ_PREFIX.length());
-        // See if we got it
-        List<String> children = zookeeper.getChildren(lockPath,false);
-        for (String x : children)
-        {
-          String otherLock;
-          if (x.startsWith(WRITE_PREFIX))
-            otherLock = x.substring(WRITE_PREFIX.length());
-          else if (x.startsWith(NONEXWRITE_PREFIX))
-            otherLock = x.substring(NONEXWRITE_PREFIX.length());
-          else
-            continue;
-          if (otherLock.compareTo(lockSequenceNumber) < 0)
-          {
-            // We didn't get the lock.  Clean up and exit
-            zookeeper.delete(lockNode,-1);
-            lockNode = null;
-            return false;
+        // Assert that we want a read lock
+        lockNode = createSequentialChild(lockPath,READ_PREFIX);
+        try
+        {
+          String lockSequenceNumber = lockNode.substring(lockPath.length() + 1 + READ_PREFIX.length());
+          // See if we got it
+          List<String> children = zookeeper.getChildren(lockPath,false);
+          for (String x : children)
+          {
+            String otherLock;
+            if (x.startsWith(WRITE_PREFIX))
+              otherLock = x.substring(WRITE_PREFIX.length());
+            else if (x.startsWith(NONEXWRITE_PREFIX))
+              otherLock = x.substring(NONEXWRITE_PREFIX.length());
+            else
+              continue;
+            if (otherLock.compareTo(lockSequenceNumber) < 0)
+            {
+              // We didn't get the lock.  Clean up and exit
+              zookeeper.delete(lockNode,-1);
+              lockNode = null;
+              return false;
+            }
           }
         }
+        catch (KeeperException e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (InterruptedException e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (Error e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (RuntimeException e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        // We got it!
+        return true;
       }
       catch (KeeperException e)
       {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-      catch (InterruptedException e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-      catch (Error e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
+        handleKeeperException(e);
       }
-      catch (RuntimeException e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-      // We got it!
-      return true;
-    }
-    catch (KeeperException e)
-    {
-      throw new ManifoldCFException(e.getMessage(),e);
     }
   }
   
@@ -656,97 +709,100 @@ public class ZooKeeperConnection
     if (lockNode != null)
       throw new IllegalStateException("Already have a lock in place: '"+lockNode+"'; can't
also read lock '"+lockPath+"'");
 
-    try
+    while (true)
     {
-      // Assert that we want a read lock
-      lockNode = createSequentialChild(lockPath,READ_PREFIX);
       try
       {
-        long lockSequenceNumber = new Long(lockNode.substring(lockPath.length() + 1 + READ_PREFIX.length())).longValue();
-        //System.out.println("Trying to get read lock for '"+lockSequenceNumber+"'");
-        while (true)
+        // Assert that we want a read lock
+        lockNode = createSequentialChild(lockPath,READ_PREFIX);
+        try
         {
-          // See if we got it
-          List<String> children = zookeeper.getChildren(lockPath,false);
-          String previousLock = null;
-          boolean gotLock = true;
-          long highestPreviousLockIndex = -1L;
-          for (String x : children)
+          long lockSequenceNumber = new Long(lockNode.substring(lockPath.length() + 1 + READ_PREFIX.length())).longValue();
+          //System.out.println("Trying to get read lock for '"+lockSequenceNumber+"'");
+          while (true)
           {
-            String otherLock;
-            if (x.startsWith(WRITE_PREFIX))
-              otherLock = x.substring(WRITE_PREFIX.length());
-            else if (x.startsWith(NONEXWRITE_PREFIX))
-              otherLock = x.substring(NONEXWRITE_PREFIX.length());
-            else
-              continue;
-            long otherLockSequenceNumber = new Long(otherLock).longValue();
-            //System.out.println("Saw other child sequence number "+otherLockSequenceNumber);
-            if (otherLockSequenceNumber < lockSequenceNumber)
-            {
-              // We didn't get the lock.  But keep going because we're looking for the node
right before the
-              // one we just asserted.
-              gotLock = false;
-              if (otherLockSequenceNumber > highestPreviousLockIndex)
+            // See if we got it
+            List<String> children = zookeeper.getChildren(lockPath,false);
+            String previousLock = null;
+            boolean gotLock = true;
+            long highestPreviousLockIndex = -1L;
+            for (String x : children)
+            {
+              String otherLock;
+              if (x.startsWith(WRITE_PREFIX))
+                otherLock = x.substring(WRITE_PREFIX.length());
+              else if (x.startsWith(NONEXWRITE_PREFIX))
+                otherLock = x.substring(NONEXWRITE_PREFIX.length());
+              else
+                continue;
+              long otherLockSequenceNumber = new Long(otherLock).longValue();
+              //System.out.println("Saw other child sequence number "+otherLockSequenceNumber);
+              if (otherLockSequenceNumber < lockSequenceNumber)
               {
-                previousLock = x;
-                highestPreviousLockIndex = otherLockSequenceNumber;
+                // We didn't get the lock.  But keep going because we're looking for the
node right before the
+                // one we just asserted.
+                gotLock = false;
+                if (otherLockSequenceNumber > highestPreviousLockIndex)
+                {
+                  previousLock = x;
+                  highestPreviousLockIndex = otherLockSequenceNumber;
+                }
               }
             }
-          }
-          
-          if (gotLock)
-          {
-            // We got it!
-            //System.out.println("Got read lock for '"+lockSequenceNumber+"'");
-            return;
-          }
+            
+            if (gotLock)
+            {
+              // We got it!
+              //System.out.println("Got read lock for '"+lockSequenceNumber+"'");
+              return;
+            }
 
-          // There SHOULD be a previous node immediately prior to the one we asserted.  If
we didn't find one, go back around;
-          // the previous lock was probably created and destroyed before we managed to get
the children.
-          if (previousLock != null)
-          {
-            //System.out.println(" Waiting on '"+previousLock+"' for read lock '"+lockSequenceNumber+"'");
-            // Create an exists() watch on the previous node, and wait until we are awakened
by that watch firing.
-            ExistsWatcher w = new ExistsWatcher();
-            Stat s = zookeeper.exists(lockPath+"/"+previousLock, w);
-            if (s != null)
-              w.waitForEvent();
-          }
-          //else
-          //  System.out.println(" Retrying for read lock '"+lockSequenceNumber+"'");
+            // There SHOULD be a previous node immediately prior to the one we asserted.
 If we didn't find one, go back around;
+            // the previous lock was probably created and destroyed before we managed to
get the children.
+            if (previousLock != null)
+            {
+              //System.out.println(" Waiting on '"+previousLock+"' for read lock '"+lockSequenceNumber+"'");
+              // Create an exists() watch on the previous node, and wait until we are awakened
by that watch firing.
+              ExistsWatcher w = new ExistsWatcher();
+              Stat s = zookeeper.exists(lockPath+"/"+previousLock, w);
+              if (s != null)
+                w.waitForEvent();
+            }
+            //else
+            //  System.out.println(" Retrying for read lock '"+lockSequenceNumber+"'");
 
+          }
+        }
+        catch (KeeperException e)
+        {
+          //System.out.println("Unexpected keeper exception: "+e.getMessage());
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (InterruptedException e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (Error e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
+        }
+        catch (RuntimeException e)
+        {
+          zookeeper.delete(lockNode,-1);
+          lockNode = null;
+          throw e;
         }
       }
       catch (KeeperException e)
       {
-        //System.out.println("Unexpected keeper exception: "+e.getMessage());
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
+        handleKeeperException(e);
       }
-      catch (InterruptedException e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-      catch (Error e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-      catch (RuntimeException e)
-      {
-        zookeeper.delete(lockNode,-1);
-        lockNode = null;
-        throw e;
-      }
-    }
-    catch (KeeperException e)
-    {
-      throw new ManifoldCFException(e.getMessage(),e);
     }
   }
   
@@ -758,128 +814,155 @@ public class ZooKeeperConnection
     if (lockNode == null)
       throw new IllegalStateException("Can't release lock we don't hold");
     //System.out.println("Releasing lock '"+lockNode+"'");
-    try
-    {
-      zookeeper.delete(lockNode,-1);
-      lockNode = null;
-    }
-    catch (KeeperException e)
+    
+    while (true)
     {
-      throw new ManifoldCFException(e.getMessage(),e);
+      try
+      {
+        zookeeper.delete(lockNode,-1);
+        lockNode = null;
+        break;
+      }
+      catch (KeeperException e)
+      {
+        handleKeeperException(e);
+      }
     }
   }
 
   public byte[] readData(String resourcePath)
     throws ManifoldCFException, InterruptedException
   {
-    try
-    {
-      return zookeeper.getData(resourcePath,false,null);
-    }
-    catch (KeeperException.NoNodeException e)
-    {
-      return null;
-    }
-    catch (KeeperException e)
+    while (true)
     {
-      throw new ManifoldCFException(e.getMessage(),e);
+      try
+      {
+        return zookeeper.getData(resourcePath,false,null);
+      }
+      catch (KeeperException.NoNodeException e)
+      {
+        return null;
+      }
+      catch (KeeperException e)
+      {
+        handleKeeperException(e);
+      }
     }
   }
   
   public void writeData(String resourcePath, byte[] data)
     throws ManifoldCFException, InterruptedException
   {
-    try
+    while (true)
     {
-      if (data == null)
-      {
-        try
-        {
-          zookeeper.delete(resourcePath, -1);
-        }
-        catch (KeeperException.NoNodeException e)
-        {
-        }
-      }
-      else
+      try
       {
-        while (true)
+        if (data == null)
         {
           try
           {
-            zookeeper.setData(resourcePath, data, -1);
+            zookeeper.delete(resourcePath, -1);
             break;
           }
           catch (KeeperException.NoNodeException e)
           {
+            break;
+          }
+        }
+        else
+        {
+          while (true)
+          {
             try
             {
-              zookeeper.create(resourcePath, data, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+              zookeeper.setData(resourcePath, data, -1);
               break;
             }
-            catch (KeeperException.NodeExistsException e2)
+            catch (KeeperException.NoNodeException e)
             {
-              continue;
+              try
+              {
+                zookeeper.create(resourcePath, data, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+                break;
+              }
+              catch (KeeperException.NodeExistsException e2)
+              {
+                continue;
+              }
             }
           }
+          break;
         }
       }
-    }
-    catch (KeeperException e)
-    {
-      throw new ManifoldCFException(e.getMessage(),e);
+      catch (KeeperException e)
+      {
+        handleKeeperException(e);
+      }
     }
   }
 
   public void setGlobalFlag(String flagPath)
     throws ManifoldCFException, InterruptedException
   {
-    try
+    while (true)
     {
       try
       {
-        zookeeper.create(flagPath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+        try
+        {
+          zookeeper.create(flagPath, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+          break;
+        }
+        catch (KeeperException.NodeExistsException e)
+        {
+          break;
+        }
       }
-      catch (KeeperException.NodeExistsException e)
+      catch (KeeperException e)
       {
+        handleKeeperException(e);
       }
     }
-    catch (KeeperException e)
-    {
-      throw new ManifoldCFException(e.getMessage(),e);
-    }
   }
 
   public void clearGlobalFlag(String flagPath)
     throws ManifoldCFException, InterruptedException
   {
-    try
+    while (true)
     {
       try
       {
-        zookeeper.delete(flagPath,-1);
+        try
+        {
+          zookeeper.delete(flagPath,-1);
+          break;
+        }
+        catch (KeeperException.NoNodeException e)
+        {
+          break;
+        }
       }
-      catch (KeeperException.NoNodeException e)
+      catch (KeeperException e)
       {
+        handleKeeperException(e);
       }
     }
-    catch (KeeperException e)
-    {
-      throw new ManifoldCFException(e.getMessage(),e);
-    }
   }
   
   public boolean checkGlobalFlag(String flagPath)
     throws ManifoldCFException, InterruptedException
   {
-    try
-    {
-      Stat s = zookeeper.exists(flagPath,false);
-      return s != null;
-    }
-    catch (KeeperException e)
+    while (true)
     {
-      throw new ManifoldCFException(e.getMessage(),e);
+      try
+      {
+        Stat s = zookeeper.exists(flagPath,false);
+        return s != null;
+      }
+      catch (KeeperException e)
+      {
+        handleKeeperException(e);
+      }
     }
   }
   
@@ -894,6 +977,27 @@ public class ZooKeeperConnection
     zookeeperWatcher = null;
   }
   
+  protected void handleKeeperException(KeeperException e)
+    throws ManifoldCFException, InterruptedException
+  {
+    if (e instanceof KeeperException.ConnectionLossException)
+    {
+      // Retry if connection loss
+      ManifoldCF.sleep(100L);
+    }
+    else if (e instanceof KeeperException.SessionExpiredException)
+    {
+      // Close the handle, open a new one
+      zookeeper.close();
+      createSession();
+    }
+    else
+    {
+      // If nothing we know how to deal with, throw.
+      throw new ManifoldCFException(e.getMessage(),e);
+    }
+  }
+  
   public static String zooKeeperSafeName(String input)
   {
     // Escape "/" characters



Mime
View raw message