Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 76DDC200C8E for ; Thu, 8 Jun 2017 17:59:19 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 75809160BD5; Thu, 8 Jun 2017 15:59:19 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 3D382160BCA for ; Thu, 8 Jun 2017 17:59:18 +0200 (CEST) Received: (qmail 44702 invoked by uid 500); 8 Jun 2017 15:59:17 -0000 Mailing-List: contact commits-help@zookeeper.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@zookeeper.apache.org Delivered-To: mailing list commits@zookeeper.apache.org Received: (qmail 44691 invoked by uid 99); 8 Jun 2017 15:59:17 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 08 Jun 2017 15:59:17 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 485ECDFBC6; Thu, 8 Jun 2017 15:59:17 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: hanm@apache.org To: commits@zookeeper.apache.org Message-Id: <7b7b29a6b0ee48b9858644af38298dea@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: zookeeper git commit: ZOOKEEPER-2775: ZK Client not able to connect with Xid out of order error Date: Thu, 8 Jun 2017 15:59:17 +0000 (UTC) archived-at: Thu, 08 Jun 2017 15:59:19 -0000 Repository: zookeeper Updated Branches: refs/heads/master 1038966e8 -> fa1dc109d ZOOKEEPER-2775: ZK Client not able to connect with Xid out of order error Once client enters into Xid out of order issue, It never comes to normal state. It keeps trying to connect and fail with the same error. Recreating/Restarting is the only solution as of now. This happens because of bug in the ZK client code. This MR provides the fix. Author: Mohammad Arshad Reviewers: Michael Han , Rakesh Radhakrishnan , Abe Fine Closes #254 from arshadmohammad/ZOOKEEPER-2775-XidOutOfOrder Project: http://git-wip-us.apache.org/repos/asf/zookeeper/repo Commit: http://git-wip-us.apache.org/repos/asf/zookeeper/commit/fa1dc109 Tree: http://git-wip-us.apache.org/repos/asf/zookeeper/tree/fa1dc109 Diff: http://git-wip-us.apache.org/repos/asf/zookeeper/diff/fa1dc109 Branch: refs/heads/master Commit: fa1dc109d4c1bb7913fee43170ed6131e3dc1b1f Parents: 1038966 Author: Mohammad Arshad Authored: Thu Jun 8 08:59:05 2017 -0700 Committer: Michael Han Committed: Thu Jun 8 08:59:05 2017 -0700 ---------------------------------------------------------------------- .../main/org/apache/zookeeper/ClientCnxn.java | 2 + .../test/org/apache/zookeeper/SaslAuthTest.java | 213 +++++++++++++++++++ .../org/apache/zookeeper/test/SaslAuthTest.java | 146 ------------- 3 files changed, 215 insertions(+), 146 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zookeeper/blob/fa1dc109/src/java/main/org/apache/zookeeper/ClientCnxn.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/ClientCnxn.java b/src/java/main/org/apache/zookeeper/ClientCnxn.java index 2a1da4c..a1984a3 100644 --- a/src/java/main/org/apache/zookeeper/ClientCnxn.java +++ b/src/java/main/org/apache/zookeeper/ClientCnxn.java @@ -1052,6 +1052,8 @@ public class ClientCnxn { private boolean saslLoginFailed = false; private void startConnect() throws IOException { + // initializing it for new connection + saslLoginFailed = false; if(!isFirstConnect){ try { Thread.sleep(r.nextInt(1000)); http://git-wip-us.apache.org/repos/asf/zookeeper/blob/fa1dc109/src/java/test/org/apache/zookeeper/SaslAuthTest.java ---------------------------------------------------------------------- diff --git a/src/java/test/org/apache/zookeeper/SaslAuthTest.java b/src/java/test/org/apache/zookeeper/SaslAuthTest.java new file mode 100644 index 0000000..eac0703 --- /dev/null +++ b/src/java/test/org/apache/zookeeper/SaslAuthTest.java @@ -0,0 +1,213 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper; + +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.zookeeper.ClientCnxn.SendThread; +import org.apache.zookeeper.Watcher.Event.KeeperState; +import org.apache.zookeeper.ZooDefs.Ids; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Id; +import org.apache.zookeeper.test.ClientBase; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +public class SaslAuthTest extends ClientBase { + @BeforeClass + public static void init() { + System.setProperty("zookeeper.authProvider.1", + "org.apache.zookeeper.server.auth.SASLAuthenticationProvider"); + try { + File tmpDir = createTmpDir(); + File saslConfFile = new File(tmpDir, "jaas.conf"); + String jaasContent = getJaasFileContent(); + FileWriter fwriter = new FileWriter(saslConfFile); + fwriter.write(jaasContent); + fwriter.close(); + System.setProperty("java.security.auth.login.config", saslConfFile.getAbsolutePath()); + } catch (IOException e) { + // could not create tmp directory to hold JAAS conf file : test will + // fail now. + } + } + + private static String getJaasFileContent() { + StringBuilder jaasContent=new StringBuilder(); + String newLine = System.getProperty("line.separator"); + jaasContent.append("Server {"); + jaasContent.append(newLine); + jaasContent.append("org.apache.zookeeper.server.auth.DigestLoginModule required"); + jaasContent.append(newLine); + jaasContent.append("user_super=\"test\";"); + jaasContent.append(newLine); + jaasContent.append("};"); + jaasContent.append(newLine); + jaasContent.append("Client {"); + jaasContent.append(newLine); + jaasContent.append("org.apache.zookeeper.server.auth.DigestLoginModule required"); + jaasContent.append(newLine); + jaasContent.append("username=\"super\""); + jaasContent.append(newLine); + jaasContent.append("password=\"test\";"); + jaasContent.append(newLine); + jaasContent.append("};"); + jaasContent.append(newLine); + return jaasContent.toString(); + } + + @AfterClass + public static void clean() { + System.clearProperty("zookeeper.authProvider.1"); + System.clearProperty("java.security.auth.login.config"); + } + + private AtomicInteger authFailed = new AtomicInteger(0); + + @Override + protected TestableZooKeeper createClient(String hp) + throws IOException, InterruptedException + { + MyWatcher watcher = new MyWatcher(); + return createClient(watcher, hp); + } + + private class MyWatcher extends CountdownWatcher { + @Override + public synchronized void process(WatchedEvent event) { + if (event.getState() == KeeperState.AuthFailed) { + authFailed.incrementAndGet(); + } + else { + super.process(event); + } + } + } + + @Test + public void testAuth() throws Exception { + ZooKeeper zk = createClient(); + try { + zk.create("/path1", null, Ids.CREATOR_ALL_ACL, CreateMode.PERSISTENT); + Thread.sleep(1000); + } finally { + zk.close(); + } + } + + @Test + public void testValidSaslIds() throws Exception { + ZooKeeper zk = createClient(); + + List validIds = new ArrayList(); + validIds.add("user"); + validIds.add("service/host.name.com"); + validIds.add("user@KERB.REALM"); + validIds.add("service/host.name.com@KERB.REALM"); + + int i = 0; + for(String validId: validIds) { + List aclList = new ArrayList(); + ACL acl = new ACL(0,new Id("sasl",validId)); + aclList.add(acl); + zk.create("/valid"+i,null,aclList,CreateMode.PERSISTENT); + i++; + } + } + + @Test + public void testInvalidSaslIds() throws Exception { + ZooKeeper zk = createClient(); + + List invalidIds = new ArrayList(); + invalidIds.add("user@KERB.REALM/server.com"); + invalidIds.add("user@KERB.REALM1@KERB.REALM2"); + + int i = 0; + for(String invalidId: invalidIds) { + List aclList = new ArrayList(); + try { + ACL acl = new ACL(0,new Id("sasl",invalidId)); + aclList.add(acl); + zk.create("/invalid"+i,null,aclList,CreateMode.PERSISTENT); + Assert.fail("SASLAuthenticationProvider.isValid() failed to catch invalid Id."); + } + catch (KeeperException.InvalidACLException e) { + // ok. + } + finally { + i++; + } + } + } + + @Test + public void testZKOperationsAfterClientSaslAuthFailure() throws Exception { + CountdownWatcher watcher = new CountdownWatcher(); + ZooKeeper zk = new ZooKeeper(hostPort, CONNECTION_TIMEOUT, watcher); + watcher.waitForConnected(CONNECTION_TIMEOUT); + try { + setSaslFailureFlag(zk); + + // try node creation for around 15 second, + int totalTry = 10; + int tryCount = 0; + + boolean success = false; + while (!success && tryCount++ <= totalTry) { + try { + zk.create("/saslAuthFail", "data".getBytes(), Ids.OPEN_ACL_UNSAFE, + CreateMode.PERSISTENT_SEQUENTIAL); + success = true; + } catch (KeeperException.ConnectionLossException e) { + Thread.sleep(1000); + // do nothing + } + } + assertTrue("ZNode creation is failing continuously after Sasl auth failure.", success); + + } finally { + zk.close(); + } + } + + // set saslLoginFailed to true to simulate the LoginException + private void setSaslFailureFlag(ZooKeeper zk) throws Exception { + Field cnxnField = zk.getClass().getDeclaredField("cnxn"); + cnxnField.setAccessible(true); + ClientCnxn clientCnxn = (ClientCnxn) cnxnField.get(zk); + Field sendThreadField = clientCnxn.getClass().getDeclaredField("sendThread"); + sendThreadField.setAccessible(true); + SendThread sendThread = (SendThread) sendThreadField.get(clientCnxn); + Field saslLoginFailedField = sendThread.getClass().getDeclaredField("saslLoginFailed"); + saslLoginFailedField.setAccessible(true); + saslLoginFailedField.setBoolean(sendThread, true); + } + +} http://git-wip-us.apache.org/repos/asf/zookeeper/blob/fa1dc109/src/java/test/org/apache/zookeeper/test/SaslAuthTest.java ---------------------------------------------------------------------- diff --git a/src/java/test/org/apache/zookeeper/test/SaslAuthTest.java b/src/java/test/org/apache/zookeeper/test/SaslAuthTest.java deleted file mode 100644 index 6e75998..0000000 --- a/src/java/test/org/apache/zookeeper/test/SaslAuthTest.java +++ /dev/null @@ -1,146 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zookeeper.test; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.TestableZooKeeper; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.ZooKeeper; -import org.apache.zookeeper.Watcher.Event.KeeperState; -import org.apache.zookeeper.ZooDefs.Ids; -import org.apache.zookeeper.data.ACL; -import org.apache.zookeeper.data.Id; -import org.junit.Assert; -import org.junit.Test; - -public class SaslAuthTest extends ClientBase { - static { - System.setProperty("zookeeper.authProvider.1","org.apache.zookeeper.server.auth.SASLAuthenticationProvider"); - - try { - File tmpDir = createTmpDir(); - File saslConfFile = new File(tmpDir, "jaas.conf"); - FileWriter fwriter = new FileWriter(saslConfFile); - - fwriter.write("" + - "Server {\n" + - " org.apache.zookeeper.server.auth.DigestLoginModule required\n" + - " user_super=\"test\";\n" + - "};\n" + - "Client {\n" + - " org.apache.zookeeper.server.auth.DigestLoginModule required\n" + - " username=\"super\"\n" + - " password=\"test\";\n" + - "};" + "\n"); - fwriter.close(); - System.setProperty("java.security.auth.login.config",saslConfFile.getAbsolutePath()); - } - catch (IOException e) { - // could not create tmp directory to hold JAAS conf file : test will fail now. - } - } - - private AtomicInteger authFailed = new AtomicInteger(0); - - @Override - protected TestableZooKeeper createClient(String hp) - throws IOException, InterruptedException - { - MyWatcher watcher = new MyWatcher(); - return createClient(watcher, hp); - } - - private class MyWatcher extends CountdownWatcher { - @Override - public synchronized void process(WatchedEvent event) { - if (event.getState() == KeeperState.AuthFailed) { - authFailed.incrementAndGet(); - } - else { - super.process(event); - } - } - } - - @Test - public void testAuth() throws Exception { - ZooKeeper zk = createClient(); - try { - zk.create("/path1", null, Ids.CREATOR_ALL_ACL, CreateMode.PERSISTENT); - Thread.sleep(1000); - } finally { - zk.close(); - } - } - - @Test - public void testValidSaslIds() throws Exception { - ZooKeeper zk = createClient(); - - List validIds = new ArrayList(); - validIds.add("user"); - validIds.add("service/host.name.com"); - validIds.add("user@KERB.REALM"); - validIds.add("service/host.name.com@KERB.REALM"); - - int i = 0; - for(String validId: validIds) { - List aclList = new ArrayList(); - ACL acl = new ACL(0,new Id("sasl",validId)); - aclList.add(acl); - zk.create("/valid"+i,null,aclList,CreateMode.PERSISTENT); - i++; - } - } - - @Test - public void testInvalidSaslIds() throws Exception { - ZooKeeper zk = createClient(); - - List invalidIds = new ArrayList(); - invalidIds.add("user@KERB.REALM/server.com"); - invalidIds.add("user@KERB.REALM1@KERB.REALM2"); - - int i = 0; - for(String invalidId: invalidIds) { - List aclList = new ArrayList(); - try { - ACL acl = new ACL(0,new Id("sasl",invalidId)); - aclList.add(acl); - zk.create("/invalid"+i,null,aclList,CreateMode.PERSISTENT); - Assert.fail("SASLAuthenticationProvider.isValid() failed to catch invalid Id."); - } - catch (KeeperException.InvalidACLException e) { - // ok. - } - finally { - i++; - } - } - } - -}