accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject accumulo git commit: ACCUMULO-3634 Fix race condition where master might delete node before tserver sees deletion.
Date Tue, 03 Mar 2015 00:56:06 GMT
Repository: accumulo
Updated Branches:
  refs/heads/master 293340ac6 -> 9339ecf82


ACCUMULO-3634 Fix race condition where master might delete node before tserver sees deletion.

On startup, the master will purge expired keys. The tserver might get the list of children
and then try to fetch the data for each child, but the child might be deleted after
the original list was obtained. Need to catch NoNodeException and gracefully move on.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/9339ecf8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/9339ecf8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/9339ecf8

Branch: refs/heads/master
Commit: 9339ecf825a6b530d0f93f6f7e02be51716d4ddc
Parents: 293340a
Author: Josh Elser <elserj@apache.org>
Authored: Mon Mar 2 16:54:17 2015 -0800
Committer: Josh Elser <elserj@apache.org>
Committed: Mon Mar 2 16:54:17 2015 -0800

----------------------------------------------------------------------
 .../delegation/ZooAuthenticationKeyWatcher.java | 14 ++++++++----
 .../ZooAuthenticationKeyWatcherTest.java        | 24 ++++++++++++++++++++
 2 files changed, 34 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/9339ecf8/server/base/src/main/java/org/apache/accumulo/server/security/delegation/ZooAuthenticationKeyWatcher.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/security/delegation/ZooAuthenticationKeyWatcher.java
b/server/base/src/main/java/org/apache/accumulo/server/security/delegation/ZooAuthenticationKeyWatcher.java
index 2913343..fe4407e 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/security/delegation/ZooAuthenticationKeyWatcher.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/security/delegation/ZooAuthenticationKeyWatcher.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 
 import org.apache.accumulo.fate.zookeeper.ZooReader;
 import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.Watcher.Event.EventType;
@@ -131,10 +132,15 @@ public class ZooAuthenticationKeyWatcher implements Watcher {
     int keysAdded = 0;
     for (String child : zk.getChildren(path, this)) {
       String childPath = path + "/" + child;
-      // Get the node data and reset the watcher
-      AuthenticationKey key = deserializeKey(zk.getData(childPath, this, null));
-      secretManager.addKey(key);
-      keysAdded++;
+      try {
+        // Get the node data and reset the watcher
+        AuthenticationKey key = deserializeKey(zk.getData(childPath, this, null));
+        secretManager.addKey(key);
+        keysAdded++;
+      } catch (NoNodeException e) {
+        // The master expired(deleted) the key between when we saw it in getChildren() and
when we went to add it to our secret manager.
+        log.trace("{} was deleted when we tried to access it", childPath);
+      }
     }
     return keysAdded;
   }

http://git-wip-us.apache.org/repos/asf/accumulo/blob/9339ecf8/server/base/src/test/java/org/apache/accumulo/server/security/delegation/ZooAuthenticationKeyWatcherTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/security/delegation/ZooAuthenticationKeyWatcherTest.java
b/server/base/src/test/java/org/apache/accumulo/server/security/delegation/ZooAuthenticationKeyWatcherTest.java
index a60c9bc..4792812 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/security/delegation/ZooAuthenticationKeyWatcherTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/security/delegation/ZooAuthenticationKeyWatcherTest.java
@@ -39,6 +39,7 @@ import javax.crypto.KeyGenerator;
 import org.apache.accumulo.core.Constants;
 import org.apache.accumulo.core.client.Instance;
 import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.zookeeper.KeeperException.NoNodeException;
 import org.apache.zookeeper.WatchedEvent;
 import org.apache.zookeeper.Watcher.Event.EventType;
 import org.apache.zookeeper.Watcher.Event.KeeperState;
@@ -315,6 +316,29 @@ public class ZooAuthenticationKeyWatcherTest {
     assertEquals(key2, secretManager.getKeys().get(key2.getKeyId()));
   }
 
+  @Test
+  public void missingKeyAfterGetChildren() throws Exception {
+    List<String> children = Arrays.asList("1");
+    AuthenticationKey key1 = new AuthenticationKey(1, 0l, 10000l, keyGen.generateKey());
+
+    expect(zk.exists(baseNode, keyWatcher)).andReturn(true);
+    // We saw key1
+    expect(zk.getChildren(baseNode, keyWatcher)).andReturn(children);
+    // but it was gone when we tried to access it (master deleted it)
+    expect(zk.getData(baseNode + "/" + key1.getKeyId(), keyWatcher, null)).andThrow(new NoNodeException());
+
+    replay(zk, instance);
+
+    // Initialize
+    keyWatcher.updateAuthKeys();
+
+    verify(zk, instance);
+
+    // We should have no auth keys after initializing things
+    assertEquals("Secret manager should be empty after a disconnect", 0, secretManager.getKeys().size());
+    assertNull("Current key should be null", secretManager.getCurrentKey());
+  }
+
   private byte[] serialize(AuthenticationKey key) throws IOException {
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     key.write(new DataOutputStream(baos));


Mime
View raw message