hadoop-zookeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maha...@apache.org
Subject svn commit: r882742 - in /hadoop/zookeeper/branches/branch-3.2: ./ src/java/main/org/apache/zookeeper/server/quorum/ src/java/test/org/apache/zookeeper/test/
Date Fri, 20 Nov 2009 22:27:31 GMT
Author: mahadev
Date: Fri Nov 20 22:27:29 2009
New Revision: 882742

URL: http://svn.apache.org/viewvc?rev=882742&view=rev
Log:
ZOOKEEPER-582. ZooKeeper can revert to old data when a snapshot is created outside of normal
processing (ben reed and mahadev via mahadev)

Added:
    hadoop/zookeeper/branches/branch-3.2/src/java/test/org/apache/zookeeper/test/QuorumZxidSyncTest.java
Modified:
    hadoop/zookeeper/branches/branch-3.2/CHANGES.txt
    hadoop/zookeeper/branches/branch-3.2/src/java/main/org/apache/zookeeper/server/quorum/Follower.java
    hadoop/zookeeper/branches/branch-3.2/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java
    hadoop/zookeeper/branches/branch-3.2/src/java/test/org/apache/zookeeper/test/ClientBase.java
    hadoop/zookeeper/branches/branch-3.2/src/java/test/org/apache/zookeeper/test/QuorumBase.java

Modified: hadoop/zookeeper/branches/branch-3.2/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/branches/branch-3.2/CHANGES.txt?rev=882742&r1=882741&r2=882742&view=diff
==============================================================================
--- hadoop/zookeeper/branches/branch-3.2/CHANGES.txt (original)
+++ hadoop/zookeeper/branches/branch-3.2/CHANGES.txt Fri Nov 20 22:27:29 2009
@@ -28,10 +28,13 @@
   (mahadev via breed)
 
   ZOOKEEPER-585. Update README for zkpython in 3.2.2 (henry robinson via phunt)
-
+  
   ZOOKEEPER-576. docs need to be updated for session moved exception and how
   to handle it (breed via phunt)
 
+  ZOOKEEPER-582. ZooKeeper can revert to old data when a snapshot is created
+  outside of normal processing (ben reed and mahadev via mahadev)
+
 Release 3.2.1 - 2009-08-27
 
 Backward compatible changes:

Modified: hadoop/zookeeper/branches/branch-3.2/src/java/main/org/apache/zookeeper/server/quorum/Follower.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/branches/branch-3.2/src/java/main/org/apache/zookeeper/server/quorum/Follower.java?rev=882742&r1=882741&r2=882742&view=diff
==============================================================================
--- hadoop/zookeeper/branches/branch-3.2/src/java/main/org/apache/zookeeper/server/quorum/Follower.java
(original)
+++ hadoop/zookeeper/branches/branch-3.2/src/java/main/org/apache/zookeeper/server/quorum/Follower.java
Fri Nov 20 22:27:29 2009
@@ -192,7 +192,16 @@
                 writePacket(qp, true);
                 readPacket(qp);
                 long newLeaderZxid = qp.getZxid();
-    
+                
+                //check to see if the leader zxid is lower than ours
+                //this should never happen but is just a safety check
+                long lastLoggedZxid = sentLastZxid;
+                if ((newLeaderZxid >> 32L) < (lastLoggedZxid >> 32L)) {
+                    LOG.fatal("Leader epoch " + Long.toHexString(newLeaderZxid >> 32L)
+                            + " is less than our zxid " + Long.toHexString(lastLoggedZxid
>> 32L));
+                    throw new IOException("Error: Epoch of leader is lower");
+                }
+
                 if (qp.getType() != Leader.NEWLEADER) {
                     LOG.error("First packet should have been NEWLEADER");
                     throw new IOException("First packet should have been NEWLEADER");

Modified: hadoop/zookeeper/branches/branch-3.2/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/branches/branch-3.2/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java?rev=882742&r1=882741&r2=882742&view=diff
==============================================================================
--- hadoop/zookeeper/branches/branch-3.2/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java
(original)
+++ hadoop/zookeeper/branches/branch-3.2/src/java/main/org/apache/zookeeper/server/quorum/QuorumPeer.java
Fri Nov 20 22:27:29 2009
@@ -34,6 +34,7 @@
 import org.apache.zookeeper.server.NIOServerCnxn;
 import org.apache.zookeeper.server.ZooKeeperServer;
 import org.apache.zookeeper.server.persistence.FileTxnSnapLog;
+import org.apache.zookeeper.server.persistence.Util;
 import org.apache.zookeeper.server.quorum.flexible.QuorumVerifier;
 import org.apache.zookeeper.server.quorum.flexible.QuorumMaj;
 
@@ -375,9 +376,35 @@
                 new NIOServerCnxn.Factory(clientPort), quorumConfig);
     }
     
-    public long getLastLoggedZxid(){
-        return logFactory.getLastLoggedZxid();
+    /**
+     * returns the highest zxid that this host has seen
+     * 
+     * @return the highest zxid for this host
+     */
+    public long getLastLoggedZxid() {
+        /*
+         * it is possible to have the last zxid with just a snapshot and no log
+         * related to it. one example is during upgrade wherein the there is no
+         * corresponding log to the snapshot. in that case just use the snapshot
+         * zxid
+         */
+
+        File lastSnapshot = null;
+        long maxZxid = -1L;
+        long maxLogZxid = logFactory.getLastLoggedZxid();
+        try {
+            lastSnapshot = logFactory.findMostRecentSnapshot();
+            if (lastSnapshot != null) {
+                maxZxid = Math.max(Util.getZxidFromName(lastSnapshot.getName(),
+                        "snapshot"), maxLogZxid);
+            }
+        } catch (IOException ie) {
+            LOG.warn("Exception finding last snapshot ", ie);
+            maxZxid = maxLogZxid;
+        }
+        return maxZxid;
     }
+    
     public Follower follower;
     public Leader leader;
 

Modified: hadoop/zookeeper/branches/branch-3.2/src/java/test/org/apache/zookeeper/test/ClientBase.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/branches/branch-3.2/src/java/test/org/apache/zookeeper/test/ClientBase.java?rev=882742&r1=882741&r2=882742&view=diff
==============================================================================
--- hadoop/zookeeper/branches/branch-3.2/src/java/test/org/apache/zookeeper/test/ClientBase.java
(original)
+++ hadoop/zookeeper/branches/branch-3.2/src/java/test/org/apache/zookeeper/test/ClientBase.java
Fri Nov 20 22:27:29 2009
@@ -155,6 +155,8 @@
 
     public static boolean waitForServerUp(String hp, long timeout) {
         long start = System.currentTimeMillis();
+        // if there are multiple host ports just take the first one
+        hp = hp.split(",")[0];
         String split[] = hp.split(":");
         String host = split[0];
         int port = Integer.parseInt(split[1]);

Modified: hadoop/zookeeper/branches/branch-3.2/src/java/test/org/apache/zookeeper/test/QuorumBase.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/branches/branch-3.2/src/java/test/org/apache/zookeeper/test/QuorumBase.java?rev=882742&r1=882741&r2=882742&view=diff
==============================================================================
--- hadoop/zookeeper/branches/branch-3.2/src/java/test/org/apache/zookeeper/test/QuorumBase.java
(original)
+++ hadoop/zookeeper/branches/branch-3.2/src/java/test/org/apache/zookeeper/test/QuorumBase.java
Fri Nov 20 22:27:29 2009
@@ -36,13 +36,12 @@
 
     File s1dir, s2dir, s3dir, s4dir, s5dir;
     QuorumPeer s1, s2, s3, s4, s5;
-
+    
     protected void setUp() throws Exception {
         LOG.info("STARTING " + getName());
         setupTestEnv();
 
-        JMXEnv.setUp();
-
+        
         hostPort = "127.0.0.1:2181,127.0.0.1:2182,127.0.0.1:2183,127.0.0.1:2184,127.0.0.1:2185";
 
         s1dir = ClientBase.createTmpDir();
@@ -55,10 +54,13 @@
 
         LOG.info("Setup finished");
     }
+    
     void startServers() throws Exception {
         int tickTime = 2000;
         int initLimit = 3;
         int syncLimit = 3;
+        JMXEnv.setUp();
+                
         HashMap<Long,QuorumServer> peers = new HashMap<Long,QuorumServer>();
         peers.put(Long.valueOf(1), new QuorumServer(1, new InetSocketAddress("127.0.0.1",
3181)));
         peers.put(Long.valueOf(2), new QuorumServer(2, new InetSocketAddress("127.0.0.1",
3182)));
@@ -92,13 +94,13 @@
         LOG.info("start QuorumPeer 5");
         s5.start();
         LOG.info("started QuorumPeer 5");
-
         for (String hp : hostPort.split(",")) {
             assertTrue("waiting for server up",
                        ClientBase.waitForServerUp(hp,
-                                    CONNECTION_TIMEOUT));
+                                                  CONNECTION_TIMEOUT));
             LOG.info(hp + " is accepting client connections");
         }
+        
 
         // interesting to see what's there...
         JMXEnv.dump();
@@ -122,16 +124,50 @@
         }
         JMXEnv.ensureAll(ensureNames.toArray(new String[ensureNames.size()]));
     }
+    
+    public void setUpServers() throws Exception {
+        int tickTime = 2000;
+        int initLimit = 3;
+        int syncLimit = 3;
+        
+        HashMap<Long,QuorumServer> peers = new HashMap<Long,QuorumServer>();
+        peers.put(Long.valueOf(1), new QuorumServer(1, new InetSocketAddress("127.0.0.1",
3181)));
+        peers.put(Long.valueOf(2), new QuorumServer(2, new InetSocketAddress("127.0.0.1",
3182)));
+        peers.put(Long.valueOf(3), new QuorumServer(3, new InetSocketAddress("127.0.0.1",
3183)));
+        peers.put(Long.valueOf(4), new QuorumServer(4, new InetSocketAddress("127.0.0.1",
3184)));
+        peers.put(Long.valueOf(5), new QuorumServer(5, new InetSocketAddress("127.0.0.1",
3185)));
 
-    @After
-    @Override
-    protected void tearDown() throws Exception {
-        LOG.info("TearDown started");
+        LOG.info("creating QuorumPeer 1");
+        s1 = new QuorumPeer(peers, s1dir, s1dir, 2181, 0, 1, tickTime, initLimit, syncLimit);
+        assertEquals(2181, s1.getClientPort());
+        LOG.info("creating QuorumPeer 2");
+        s2 = new QuorumPeer(peers, s2dir, s2dir, 2182, 0, 2, tickTime, initLimit, syncLimit);
+        assertEquals(2182, s2.getClientPort());
+        LOG.info("creating QuorumPeer 3");
+        s3 = new QuorumPeer(peers, s3dir, s3dir, 2183, 0, 3, tickTime, initLimit, syncLimit);
+        assertEquals(2183, s3.getClientPort());
+        LOG.info("creating QuorumPeer 4");
+        s4 = new QuorumPeer(peers, s4dir, s4dir, 2184, 0, 4, tickTime, initLimit, syncLimit);
+        assertEquals(2184, s4.getClientPort());
+        LOG.info("creating QuorumPeer 5");
+        s5 = new QuorumPeer(peers, s5dir, s5dir, 2185, 0, 5, tickTime, initLimit, syncLimit);
+        assertEquals(2185, s5.getClientPort());
+    }
+
+    public void shutdownServers() throws Exception {
         shutdown(s1);
         shutdown(s2);
         shutdown(s3);
         shutdown(s4);
         shutdown(s5);
+    }
+
+
+    @After
+    @Override
+    protected void tearDown() throws Exception {
+        LOG.info("TearDown started");
+        shutdownServers();
 
         for (String hp : hostPort.split(",")) {
             assertTrue("waiting for server down",

Added: hadoop/zookeeper/branches/branch-3.2/src/java/test/org/apache/zookeeper/test/QuorumZxidSyncTest.java
URL: http://svn.apache.org/viewvc/hadoop/zookeeper/branches/branch-3.2/src/java/test/org/apache/zookeeper/test/QuorumZxidSyncTest.java?rev=882742&view=auto
==============================================================================
--- hadoop/zookeeper/branches/branch-3.2/src/java/test/org/apache/zookeeper/test/QuorumZxidSyncTest.java
(added)
+++ hadoop/zookeeper/branches/branch-3.2/src/java/test/org/apache/zookeeper/test/QuorumZxidSyncTest.java
Fri Nov 20 22:27:29 2009
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.test;
+
+import java.io.File;
+
+import junit.framework.TestCase;
+
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.server.quorum.QuorumPeer;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Test what happens when logs fall behind the snapshots or a follower has a
+ * higher epoch than a leader.
+ */
+public class QuorumZxidSyncTest extends TestCase {
+    QuorumBase qb = new QuorumBase();
+    
+    @Before
+    @Override
+    protected void setUp() throws Exception {
+        qb.setUp();
+    }
+    
+    @Test
+    /**
+     * find out what happens when a follower connects to leader that is behind
+     */
+    public void testBehindLeader() throws Exception {
+        // crank up the epoch numbers
+        ClientBase.waitForServerUp(qb.hostPort, 10000);
+        ClientBase.waitForServerUp(qb.hostPort, 10000);
+        ZooKeeper zk = new ZooKeeper(qb.hostPort, 10000, new Watcher() {
+            public void process(WatchedEvent event) {
+            }});
+        zk.create("/0", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+        zk.close();
+        qb.shutdownServers();
+        qb.startServers();
+        ClientBase.waitForServerUp(qb.hostPort, 10000);
+        qb.tearDown();
+        qb.startServers();
+        ClientBase.waitForServerUp(qb.hostPort, 10000);
+        zk = new ZooKeeper(qb.hostPort, 10000, new Watcher() {
+            public void process(WatchedEvent event) {
+            }});
+        zk.create("/1", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+        zk.close();
+        qb.shutdownServers();
+        qb.startServers();
+        ClientBase.waitForServerUp(qb.hostPort, 10000);
+        qb.shutdownServers();
+        qb.startServers();
+        ClientBase.waitForServerUp(qb.hostPort, 10000);
+        zk = new ZooKeeper(qb.hostPort, 10000, new Watcher() {
+            public void process(WatchedEvent event) {
+            }});
+        zk.create("/2", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+        zk.close();
+        qb.shutdownServers();
+        deleteFiles(qb.s1dir);
+        deleteFiles(qb.s2dir);
+        deleteFiles(qb.s3dir);
+        deleteFiles(qb.s4dir);
+        qb.setUpServers();
+        qb.s1.start();
+        qb.s2.start();
+        qb.s3.start();
+        qb.s4.start();
+        assertTrue("Servers didn't come up", ClientBase.waitForServerUp(qb.hostPort, 10000));
+        qb.s5.start();
+        String hostPort = "127.0.0.1:" + qb.s5.getClientPort();
+        assertFalse("Servers came up, but shouldn't have since it's ahead of leader",
+                ClientBase.waitForServerUp(hostPort, 10000));
+    }
+    
+
+    private void deleteFiles(File f) {
+        File v = new File(f, "version-2");
+        for(File c: v.listFiles()) {
+            c.delete();
+        }
+    }
+    
+    @Test
+    /**
+     * find out what happens when the latest state is in the snapshots not
+     * the logs.
+     */
+    public void testLateLogs() throws Exception {
+        // crank up the epoch numbers
+        ClientBase.waitForServerUp(qb.hostPort, 10000);
+        ClientBase.waitForServerUp(qb.hostPort, 10000);
+        ZooKeeper zk = new ZooKeeper(qb.hostPort, 10000, new Watcher() {
+            public void process(WatchedEvent event) {
+            }});
+        zk.create("/0", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+        zk.close();
+        qb.tearDown();
+        qb.startServers();
+        ClientBase.waitForServerUp(qb.hostPort, 10000);
+        qb.tearDown();
+        qb.startServers();
+        ClientBase.waitForServerUp(qb.hostPort, 10000);
+        zk = new ZooKeeper(qb.hostPort, 10000, new Watcher() {
+            public void process(WatchedEvent event) {
+            }});
+        zk.create("/1", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+        zk.close();
+        qb.tearDown();
+        qb.startServers();
+        ClientBase.waitForServerUp(qb.hostPort, 10000);
+        qb.tearDown();
+        deleteLogs(qb.s1dir);
+        deleteLogs(qb.s2dir);
+        deleteLogs(qb.s3dir);
+        deleteLogs(qb.s4dir);
+        deleteLogs(qb.s5dir);
+        qb.startServers();
+        ClientBase.waitForServerUp(qb.hostPort, 10000);
+        zk = new ZooKeeper(qb.hostPort, 10000, new Watcher() {
+            public void process(WatchedEvent event) {
+            }});
+        zk.create("/2", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+        zk.close();
+        qb.tearDown();
+        qb.startServers();
+        ClientBase.waitForServerUp(qb.hostPort, 10000);
+        zk = new ZooKeeper(qb.hostPort, 10000, new Watcher() {
+            public void process(WatchedEvent event) {
+            }});
+        boolean saw2 = false;
+        for(String child: zk.getChildren("/", false)) {
+            if (child.equals("2")) {
+                saw2 = true;
+            }
+        }
+        zk.close();
+        assertTrue("Didn't see /2 (went back in time)", saw2);
+    }
+    
+    private void deleteLogs(File f) {
+        File v = new File(f, "version-2");
+        for(File c: v.listFiles()) {
+            if (c.getName().startsWith("log")) {
+                c.delete();
+            }
+        }
+    }
+    
+    @After
+    @Override
+    public void tearDown() throws Exception {
+        qb.tearDown();
+    }
+}



Mime
View raw message