From commits-return-6996-archive-asf-public=cust-asf.ponee.io@zookeeper.apache.org Wed Sep 12 13:29:20 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 03CC5180630 for ; Wed, 12 Sep 2018 13:29:19 +0200 (CEST) Received: (qmail 67970 invoked by uid 500); 12 Sep 2018 11:29:19 -0000 Mailing-List: contact commits-help@zookeeper.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@zookeeper.apache.org Delivered-To: mailing list commits@zookeeper.apache.org Received: (qmail 67959 invoked by uid 99); 12 Sep 2018 11:29:19 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 12 Sep 2018 11:29:19 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D57AADFF2E; Wed, 12 Sep 2018 11:29:18 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: andor@apache.org To: commits@zookeeper.apache.org Message-Id: <8f7431e381234c8e95c99f0228f096d8@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: zookeeper git commit: ZOOKEEPER-3125: Fixing pzxid consistent issue when replaying a txn for a deleted node Date: Wed, 12 Sep 2018 11:29:18 +0000 (UTC) Repository: zookeeper Updated Branches: refs/heads/master ef8b5ab26 -> 6651a126c ZOOKEEPER-3125: Fixing pzxid consistent issue when replaying a txn for a deleted node Author: Fangmin Lyu Reviewers: breed@apache.org, andor@apache.org Closes #605 from lvfangmin/ZOOKEEPER-3125 Project: http://git-wip-us.apache.org/repos/asf/zookeeper/repo Commit: http://git-wip-us.apache.org/repos/asf/zookeeper/commit/6651a126 Tree: http://git-wip-us.apache.org/repos/asf/zookeeper/tree/6651a126 Diff: http://git-wip-us.apache.org/repos/asf/zookeeper/diff/6651a126 Branch: refs/heads/master Commit: 6651a126cd85ced8b26786875b81be140ad97c80 Parents: ef8b5ab Author: Fangmin Lyu Authored: Wed Sep 12 13:29:13 2018 +0200 Committer: Andor Molnar Committed: Wed Sep 12 13:29:13 2018 +0200 ---------------------------------------------------------------------- .../org/apache/zookeeper/server/DataTree.java | 33 ++++-- .../server/quorum/FuzzySnapshotRelatedTest.java | 117 +++++++++++++++++++ 2 files changed, 142 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zookeeper/blob/6651a126/src/java/main/org/apache/zookeeper/server/DataTree.java ---------------------------------------------------------------------- diff --git a/src/java/main/org/apache/zookeeper/server/DataTree.java b/src/java/main/org/apache/zookeeper/server/DataTree.java index e0e6661..ac91c10 100644 --- a/src/java/main/org/apache/zookeeper/server/DataTree.java +++ b/src/java/main/org/apache/zookeeper/server/DataTree.java @@ -545,6 +545,19 @@ public class DataTree { int lastSlash = path.lastIndexOf('/'); String parentName = path.substring(0, lastSlash); String childName = path.substring(lastSlash + 1); + + // The child might already be deleted during taking fuzzy snapshot, + // but we still need to update the pzxid here before throw exception + // for no such child + DataNode parent = nodes.get(parentName); + if (parent == null) { + throw new KeeperException.NoNodeException(); + } + synchronized (parent) { + parent.removeChild(childName); + parent.stat.setPzxid(zxid); + } + DataNode node = nodes.get(path); if (node == null) { throw new KeeperException.NoNodeException(); @@ -554,13 +567,11 @@ public class DataTree { aclCache.removeUsage(node.acl); nodeDataSize.addAndGet(-getNodeSize(path, node.data)); } - DataNode parent = nodes.get(parentName); - if (parent == null) { - throw new KeeperException.NoNodeException(); - } + + // Synchronized to sync the containers and ttls change, probably + // only need to sync on containers and ttls, will update it in a + // separate patch. synchronized (parent) { - parent.removeChild(childName); - parent.stat.setPzxid(zxid); long eowner = node.stat.getEphemeralOwner(); EphemeralType ephemeralType = EphemeralType.get(eowner); if (ephemeralType == EphemeralType.CONTAINER) { @@ -576,6 +587,7 @@ public class DataTree { } } } + if (parentName.startsWith(procZookeeper) && Quotas.limitNode.equals(childName)) { // delete the node in the trie. // we need to update the trie as well @@ -1198,8 +1210,7 @@ public class DataTree { Set childs = node.getChildren(); children = childs.toArray(new String[childs.size()]); } - oa.writeString(pathString, "path"); - oa.writeRecord(nodeCopy, "node"); + serializeNodeData(oa, pathString, nodeCopy); path.append('/'); int off = path.length(); for (String child : children) { @@ -1212,6 +1223,12 @@ public class DataTree { } } + // visiable for test + public void serializeNodeData(OutputArchive oa, String path, DataNode node) throws IOException { + oa.writeString(path, "path"); + oa.writeRecord(node, "node"); + } + public void serialize(OutputArchive oa, String tag) throws IOException { aclCache.serialize(oa); serializeNode(oa, new StringBuilder("")); http://git-wip-us.apache.org/repos/asf/zookeeper/blob/6651a126/src/java/test/org/apache/zookeeper/server/quorum/FuzzySnapshotRelatedTest.java ---------------------------------------------------------------------- diff --git a/src/java/test/org/apache/zookeeper/server/quorum/FuzzySnapshotRelatedTest.java b/src/java/test/org/apache/zookeeper/server/quorum/FuzzySnapshotRelatedTest.java index 0e3b230..8abab52 100644 --- a/src/java/test/org/apache/zookeeper/server/quorum/FuzzySnapshotRelatedTest.java +++ b/src/java/test/org/apache/zookeeper/server/quorum/FuzzySnapshotRelatedTest.java @@ -18,22 +18,27 @@ package org.apache.zookeeper.server.quorum; +import java.io.IOException; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import javax.security.sasl.SaslException; +import org.apache.jute.OutputArchive; + import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException.NoNodeException; import org.apache.zookeeper.KeeperException.NodeExistsException; import org.apache.zookeeper.Op; + import org.apache.zookeeper.PortAssignment; import org.apache.zookeeper.ZooDefs.Ids; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.ZooKeeper.States; import org.apache.zookeeper.data.ACL; import org.apache.zookeeper.data.Stat; +import org.apache.zookeeper.server.DataNode; import org.apache.zookeeper.server.DataTree; import org.apache.zookeeper.server.ZKDatabase; import org.apache.zookeeper.server.ZooKeeperServer; @@ -162,6 +167,98 @@ public class FuzzySnapshotRelatedTest extends QuorumPeerTestBase { new String(zk[followerA].getData(node2, null, null))); } + /** + * It's possibel during SNAP sync, the parent is serialized before the + * child get deleted during sending the snapshot over. + * + * In which case, we need to make sure the pzxid get correctly updated + * when applying the txns received. + */ + @Test + public void testPZxidUpdatedDuringSnapSyncing() throws Exception { + LOG.info("Enable force snapshot sync"); + System.setProperty(LearnerHandler.FORCE_SNAP_SYNC, "true"); + + final String parent = "/testPZxidUpdatedWhenDeletingNonExistNode"; + final String child = parent + "/child"; + createEmptyNode(zk[leaderId], parent); + createEmptyNode(zk[leaderId], child); + + LOG.info("shutdown follower {}", followerA); + mt[followerA].shutdown(); + QuorumPeerMainTest.waitForOne(zk[followerA], States.CONNECTING); + + LOG.info("Set up ZKDatabase to catch the node serializing in DataTree"); + addSerializeListener(leaderId, parent, child); + + LOG.info("Restart follower A to trigger a SNAP sync with leader"); + mt[followerA].start(); + QuorumPeerMainTest.waitForOne(zk[followerA], States.CONNECTED); + + LOG.info("Check and make sure the pzxid of the parent is the same " + + "on leader and follower A"); + compareStat(parent, leaderId, followerA); + } + + /** + * It's possible during taking fuzzy snapshot, the parent is serialized + * before the child get deleted in the fuzzy range. + * + * In which case, we need to make sure the pzxid get correctly updated + * when replaying the txns. + */ + @Test + public void testPZxidUpdatedWhenLoadingSnapshot() throws Exception { + + final String parent = "/testPZxidUpdatedDuringTakingSnapshot"; + final String child = parent + "/child"; + createEmptyNode(zk[followerA], parent); + createEmptyNode(zk[followerA], child); + + LOG.info("Set up ZKDatabase to catch the node serializing in DataTree"); + addSerializeListener(followerA, parent, child); + + LOG.info("Take snapshot on follower A"); + ZooKeeperServer zkServer = mt[followerA].main.quorumPeer.getActiveServer(); + zkServer.takeSnapshot(true); + + LOG.info("Restarting follower A to load snapshot"); + mt[followerA].shutdown(); + mt[followerA].start(); + QuorumPeerMainTest.waitForOne(zk[followerA], States.CONNECTED); + + LOG.info("Check and make sure the pzxid of the parent is the same " + + "on leader and follower A"); + compareStat(parent, leaderId, followerA); + } + + private void addSerializeListener(int sid, String parent, String child) { + final ZooKeeper zkClient = zk[followerA]; + CustomDataTree dt = + (CustomDataTree) mt[sid].main.quorumPeer.getZkDb().getDataTree(); + dt.addListener(parent, new NodeSerializeListener() { + @Override + public void nodeSerialized(String path) { + try { + zkClient.delete(child, -1); + LOG.info("Deleted the child node after the parent is serialized"); + } catch (Exception e) { + LOG.error("Error when deleting node {}", e); + } + } + }); + } + + private void compareStat(String path, int sid, int compareWithSid) throws Exception{ + Stat stat1 = new Stat(); + zk[sid].getData(path, null, stat1); + + Stat stat2 = new Stat(); + zk[compareWithSid].getData(path, null, stat2); + + Assert.assertEquals(stat1, stat2); + } + private void createEmptyNode(ZooKeeper zk, String path) throws Exception { zk.create(path, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } @@ -174,6 +271,22 @@ public class FuzzySnapshotRelatedTest extends QuorumPeerTestBase { static class CustomDataTree extends DataTree { Map nodeCreateListeners = new HashMap(); + Map listeners = + new HashMap(); + + @Override + public void serializeNodeData(OutputArchive oa, String path, + DataNode node) throws IOException { + super.serializeNodeData(oa, path, node); + NodeSerializeListener listener = listeners.get(path); + if (listener != null) { + listener.nodeSerialized(path); + } + } + + public void addListener(String path, NodeSerializeListener listener) { + listeners.put(path, listener); + } @Override public void createNode(final String path, byte data[], List acl, @@ -193,6 +306,10 @@ public class FuzzySnapshotRelatedTest extends QuorumPeerTestBase { } } + static interface NodeSerializeListener { + public void nodeSerialized(String path); + } + static class CustomizedQPMain extends TestQPMain { @Override protected QuorumPeer getQuorumPeer() throws SaslException {