Author: fpj
Date: Sat Aug 13 12:57:54 2016
New Revision: 1756260
URL: http://svn.apache.org/viewvc?rev=1756260&view=rev
Log:
ZOOKEEPER-2247: Zookeeper service becomes unavailable when leader fails to write transaction
log (Rakesh via fpj)
Added:
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServerListenerImpl.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServerShutdownHandler.java
zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/test/NonRecoverableErrorTest.java
Modified:
zookeeper/branches/branch-3.4/CHANGES.txt
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ServerCnxnFactory.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/SyncRequestProcessor.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Follower.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Leader.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Learner.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Observer.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java
zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ReadOnlyZooKeeperServer.java
zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/ZooKeeperServerMainTest.java
zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/quorum/QuorumPeerTestBase.java
Modified: zookeeper/branches/branch-3.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/CHANGES.txt?rev=1756260&r1=1756259&r2=1756260&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/CHANGES.txt (original)
+++ zookeeper/branches/branch-3.4/CHANGES.txt Sat Aug 13 12:57:54 2016
@@ -50,6 +50,9 @@ BUGFIXES:
ZOOKEEPER-2452: Back-port ZOOKEEPER-1460 to 3.4 for IPv6 literal
address support. (Abraham Fine via cnauroth)
+ ZOOKEEPER-2247: Zookeeper service becomes unavailable when leader
+ fails to write transaction log (Rakesh via fpj)
+
IMPROVEMENTS:
ZOOKEEPER-2240 Make the three-node minimum more explicit in
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ServerCnxnFactory.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ServerCnxnFactory.java?rev=1756260&r1=1756259&r2=1756260&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ServerCnxnFactory.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ServerCnxnFactory.java
Sat Aug 13 12:57:54 2016
@@ -63,6 +63,10 @@ public abstract class ServerCnxnFactory
}
}
+ ZooKeeperServer getZooKeeperServer() {
+ return zkServer;
+ }
+
public abstract void closeSession(long sessionId);
public abstract void configure(InetSocketAddress addr,
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/SyncRequestProcessor.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/SyncRequestProcessor.java?rev=1756260&r1=1756259&r2=1756260&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/SyncRequestProcessor.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/SyncRequestProcessor.java
Sat Aug 13 12:57:54 2016
@@ -182,7 +182,6 @@ public class SyncRequestProcessor extend
} catch (Throwable t) {
handleException(this.getName(), t);
running = false;
- System.exit(11);
}
LOG.info("SyncRequestProcessor exited!");
}
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java?rev=1756260&r1=1756259&r2=1756260&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServer.java
Sat Aug 13 12:57:54 2016
@@ -114,8 +114,8 @@ public class ZooKeeperServer implements
protected RequestProcessor firstProcessor;
protected volatile State state = State.INITIAL;
- enum State {
- INITIAL, RUNNING, SHUTDOWN;
+ protected enum State {
+ INITIAL, RUNNING, SHUTDOWN, ERROR;
}
/**
@@ -133,8 +133,8 @@ public class ZooKeeperServer implements
private ServerCnxnFactory serverCnxnFactory;
private final ServerStats serverStats;
-
- private final ZooKeeperServerListener listener = new ZooKeeperServerListenerImpl();
+ private final ZooKeeperServerListener listener;
+ private ZooKeeperServerShutdownHandler zkShutdownHandler;
void removeCnxn(ServerCnxn cnxn) {
zkDb.removeCnxn(cnxn);
@@ -149,6 +149,7 @@ public class ZooKeeperServer implements
*/
public ZooKeeperServer() {
serverStats = new ServerStats(this);
+ listener = new ZooKeeperServerListenerImpl(this);
}
/**
@@ -166,7 +167,9 @@ public class ZooKeeperServer implements
this.tickTime = tickTime;
this.minSessionTimeout = minSessionTimeout;
this.maxSessionTimeout = maxSessionTimeout;
-
+
+ listener = new ZooKeeperServerListenerImpl(this);
+
LOG.info("Created server with tickTime " + tickTime
+ " minSessionTimeout " + getMinSessionTimeout()
+ " maxSessionTimeout " + getMaxSessionTimeout()
@@ -417,7 +420,7 @@ public class ZooKeeperServer implements
registerJMX();
- state = State.RUNNING;
+ setState(State.RUNNING);
notifyAll();
}
@@ -434,20 +437,6 @@ public class ZooKeeperServer implements
return listener;
}
- /**
- * Default listener implementation, which will do a graceful shutdown on
- * notification
- */
- private class ZooKeeperServerListenerImpl implements
- ZooKeeperServerListener {
-
- @Override
- public void notifyStopping(String threadName, int exitCode) {
- LOG.info("Thread {} exits, error code {}", threadName, exitCode);
- shutdown();
- }
- }
-
protected void createSessionTracker() {
sessionTracker = new SessionTrackerImpl(this, zkDb.getSessionWithTimeOuts(),
tickTime, 1, getZooKeeperServerListener());
@@ -457,19 +446,58 @@ public class ZooKeeperServer implements
((SessionTrackerImpl)sessionTracker).start();
}
+ /**
+ * Sets the state of ZooKeeper server. After changing the state, it notifies
+ * the server state change to a registered shutdown handler, if any.
+ * <p>
+ * The following are the server state transitions:
+ * <li>During startup the server will be in the INITIAL state.</li>
+ * <li>After successfully starting, the server sets the state to RUNNING.
+ * </li>
+ * <li>The server transitions to the ERROR state if it hits an internal
+ * error. {@link ZooKeeperServerListenerImpl} notifies any critical resource
+ * error events, e.g., SyncRequestProcessor not being able to write a txn to
+ * disk.</li>
+ * <li>During shutdown the server sets the state to SHUTDOWN, which
+ * corresponds to the server not running.</li>
+ *
+ * @param state new server state.
+ */
+ protected void setState(State state) {
+ this.state = state;
+ // Notify server state changes to the registered shutdown handler, if any.
+ if (zkShutdownHandler != null) {
+ zkShutdownHandler.handle(state);
+ } else {
+ LOG.error("ZKShutdownHandler is not registered, so ZooKeeper server "
+ + "won't take any action on ERROR or SHUTDOWN server state changes");
+ }
+ }
+
+ /**
+ * This can be used while shutting down the server to see whether the server
+ * is already shutdown or not.
+ *
+ * @return true if the server is running or server hits an error, false
+ * otherwise.
+ */
+ protected boolean canShutdown() {
+ return state == State.RUNNING || state == State.ERROR;
+ }
+
public boolean isRunning() {
return state == State.RUNNING;
}
public synchronized void shutdown() {
- if (!isRunning()) {
+ if (!canShutdown()) {
LOG.debug("ZooKeeper server is not running, so not proceeding to shutdown!");
return;
}
LOG.info("shutting down");
// new RuntimeException("Calling shutdown").printStackTrace();
- state = State.SHUTDOWN;
+ setState(State.SHUTDOWN);
// Since sessionTracker and syncThreads poll we just have to
// set running to false and they will detect it during the poll
// interval.
@@ -1040,5 +1068,15 @@ public class ZooKeeperServer implements
return rc;
}
-
+ /**
+ * This method is used to register the ZooKeeperServerShutdownHandler to get
+ * server's error or shutdown state change notifications.
+ * {@link ZooKeeperServerShutdownHandler#handle(State)} will be called for
+ * every server state changes {@link #setState(State)}.
+ *
+ * @param zkShutdownHandler shutdown handler
+ */
+ void registerServerShutdownHandler(ZooKeeperServerShutdownHandler zkShutdownHandler)
{
+ this.zkShutdownHandler = zkShutdownHandler;
+ }
}
Added: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServerListenerImpl.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServerListenerImpl.java?rev=1756260&view=auto
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServerListenerImpl.java
(added)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServerListenerImpl.java
Sat Aug 13 12:57:54 2016
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.server;
+
+import org.apache.zookeeper.server.ZooKeeperServer.State;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Default listener implementation, which will be used to notify internal
+ * errors. For example, if some critical thread has stopped due to fatal errors,
+ * then it will get notifications and will change the state of ZooKeeper server
+ * to ERROR representing an error status.
+ */
+class ZooKeeperServerListenerImpl implements ZooKeeperServerListener {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(ZooKeeperServerListenerImpl.class);
+
+ private final ZooKeeperServer zkServer;
+
+ ZooKeeperServerListenerImpl(ZooKeeperServer zkServer) {
+ this.zkServer = zkServer;
+ }
+
+ @Override
+ public void notifyStopping(String threadName, int exitCode) {
+ LOG.info("Thread {} exits, error code {}", threadName, exitCode);
+ zkServer.setState(State.ERROR);
+ }
+}
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java?rev=1756260&r1=1756259&r2=1756260&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServerMain.java
Sat Aug 13 12:57:54 2016
@@ -20,6 +20,7 @@ package org.apache.zookeeper.server;
import java.io.File;
import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
import javax.management.JMException;
@@ -99,7 +100,12 @@ public class ZooKeeperServerMain {
// so rather than spawning another thread, we will just call
// run() in this thread.
// create a file logger url from the command line args
- ZooKeeperServer zkServer = new ZooKeeperServer();
+ final ZooKeeperServer zkServer = new ZooKeeperServer();
+ // Registers shutdown handler which will be used to know the
+ // server error or shutdown state changes.
+ final CountDownLatch shutdownLatch = new CountDownLatch(1);
+ zkServer.registerServerShutdownHandler(
+ new ZooKeeperServerShutdownHandler(shutdownLatch));
txnLog = new FileTxnSnapLog(new File(config.dataLogDir), new File(
config.dataDir));
@@ -111,8 +117,13 @@ public class ZooKeeperServerMain {
cnxnFactory.configure(config.getClientPortAddress(),
config.getMaxClientCnxns());
cnxnFactory.startup(zkServer);
+ // Watch status of ZooKeeper server. It will do a graceful shutdown
+ // if the server is not running or hits an internal error.
+ shutdownLatch.await();
+ shutdown();
+
cnxnFactory.join();
- if (zkServer.isRunning()) {
+ if (zkServer.canShutdown()) {
zkServer.shutdown();
}
} catch (InterruptedException e) {
@@ -129,6 +140,13 @@ public class ZooKeeperServerMain {
* Shutdown the serving instance
*/
protected void shutdown() {
- cnxnFactory.shutdown();
+ if (cnxnFactory != null) {
+ cnxnFactory.shutdown();
+ }
+ }
+
+ // VisibleForTesting
+ ServerCnxnFactory getCnxnFactory() {
+ return cnxnFactory;
}
}
Added: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServerShutdownHandler.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServerShutdownHandler.java?rev=1756260&view=auto
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServerShutdownHandler.java
(added)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/ZooKeeperServerShutdownHandler.java
Sat Aug 13 12:57:54 2016
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.server;
+
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.zookeeper.server.ZooKeeperServer.State;
+
+/**
+ * ZooKeeper server shutdown handler which will be used to handle ERROR or
+ * SHUTDOWN server state transitions, which in turn releases the associated
+ * shutdown latch.
+ */
+class ZooKeeperServerShutdownHandler {
+ private final CountDownLatch shutdownLatch;
+
+ ZooKeeperServerShutdownHandler(CountDownLatch shutdownLatch) {
+ this.shutdownLatch = shutdownLatch;
+ }
+
+ /**
+ * This will be invoked when the server transition to a new server state.
+ *
+ * @param state new server state
+ */
+ void handle(State state) {
+ if (state == State.ERROR || state == State.SHUTDOWN) {
+ shutdownLatch.countDown();
+ }
+ }
+}
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Follower.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Follower.java?rev=1756260&r1=1756259&r2=1756260&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Follower.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Follower.java
Sat Aug 13 12:57:54 2016
@@ -18,11 +18,9 @@
package org.apache.zookeeper.server.quorum;
-import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
-import org.apache.jute.BinaryInputArchive;
import org.apache.jute.Record;
import org.apache.zookeeper.server.util.SerializeUtils;
import org.apache.zookeeper.server.util.ZxidUtils;
@@ -81,7 +79,7 @@ public class Follower extends Learner{
}
syncWithLeader(newEpochZxid);
QuorumPacket qp = new QuorumPacket();
- while (self.isRunning()) {
+ while (this.isRunning()) {
readPacket(qp);
processPacket(qp);
}
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Leader.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Leader.java?rev=1756260&r1=1756259&r2=1756260&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Leader.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Leader.java
Sat Aug 13 12:57:54 2016
@@ -471,6 +471,12 @@ public class Leader {
f.ping();
}
+ // check leader running status
+ if (!this.isRunning()) {
+ shutdown("Unexpected internal error");
+ return;
+ }
+
if (!tickSkip && !self.getQuorumVerifier().containsQuorum(syncedSet))
{
//if (!tickSkip && syncedCount < self.quorumPeers.size() / 2)
{
// Lost quorum, shutdown
@@ -1055,4 +1061,8 @@ public class Leader {
return "UNKNOWN";
}
}
+
+ private boolean isRunning() {
+ return self.isRunning() && zk.isRunning();
+ }
}
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Learner.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Learner.java?rev=1756260&r1=1756259&r2=1756260&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Learner.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Learner.java
Sat Aug 13 12:57:54 2016
@@ -549,4 +549,8 @@ public class Learner {
zk.shutdown();
}
}
+
+ boolean isRunning() {
+ return self.isRunning() && zk.isRunning();
+ }
}
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Observer.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Observer.java?rev=1756260&r1=1756259&r2=1756260&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Observer.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/Observer.java
Sat Aug 13 12:57:54 2016
@@ -18,11 +18,9 @@
package org.apache.zookeeper.server.quorum;
-import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
-import org.apache.jute.BinaryInputArchive;
import org.apache.jute.Record;
import org.apache.zookeeper.server.ObserverBean;
import org.apache.zookeeper.server.Request;
@@ -68,10 +66,10 @@ public class Observer extends Learner{
try {
connectToLeader(addr);
long newLeaderZxid = registerWithLeader(Leader.OBSERVERINFO);
-
+
syncWithLeader(newLeaderZxid);
QuorumPacket qp = new QuorumPacket();
- while (self.isRunning()) {
+ while (this.isRunning()) {
readPacket(qp);
processPacket(qp);
}
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java?rev=1756260&r1=1756259&r2=1756260&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ObserverZooKeeperServer.java
Sat Aug 13 12:57:54 2016
@@ -140,7 +140,7 @@ public class ObserverZooKeeperServer ext
@Override
public synchronized void shutdown() {
- if (!isRunning()) {
+ if (!canShutdown()) {
LOG.debug("ZooKeeper server is not running, so not proceeding to shutdown!");
return;
}
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java?rev=1756260&r1=1756259&r2=1756260&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/QuorumZooKeeperServer.java
Sat Aug 13 12:57:54 2016
@@ -57,4 +57,9 @@ public abstract class QuorumZooKeeperSer
pwriter.print("peerType=");
pwriter.println(self.getLearnerType().ordinal());
}
+
+ @Override
+ protected void setState(State state) {
+ this.state = state;
+ }
}
Modified: zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ReadOnlyZooKeeperServer.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ReadOnlyZooKeeperServer.java?rev=1756260&r1=1756259&r2=1756260&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ReadOnlyZooKeeperServer.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/main/org/apache/zookeeper/server/quorum/ReadOnlyZooKeeperServer.java
Sat Aug 13 12:57:54 2016
@@ -91,6 +91,11 @@ public class ReadOnlyZooKeeperServer ext
}
@Override
+ protected void setState(State state) {
+ this.state = state;
+ }
+
+ @Override
protected void unregisterJMX() {
// unregister from JMX
try {
@@ -131,7 +136,7 @@ public class ReadOnlyZooKeeperServer ext
@Override
public synchronized void shutdown() {
- if (!isRunning()) {
+ if (!canShutdown()) {
LOG.debug("ZooKeeper server is not running, so not proceeding to shutdown!");
return;
}
Modified: zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/ZooKeeperServerMainTest.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/ZooKeeperServerMainTest.java?rev=1756260&r1=1756259&r2=1756260&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/ZooKeeperServerMainTest.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/ZooKeeperServerMainTest.java
Sat Aug 13 12:57:54 2016
@@ -19,6 +19,7 @@
package org.apache.zookeeper.server;
import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT;
+import static org.junit.Assert.fail;
import java.io.File;
import java.io.FileWriter;
@@ -127,6 +128,10 @@ public class ZooKeeperServerMainTest ext
throw new IOException("Failed to delete file: " + f);
}
}
+
+ ServerCnxnFactory getCnxnFactory() {
+ return main.getCnxnFactory();
+ }
}
public static class TestZKSMain extends ZooKeeperServerMain {
@@ -136,6 +141,63 @@ public class ZooKeeperServerMainTest ext
}
/**
+ * Test case for https://issues.apache.org/jira/browse/ZOOKEEPER-2247.
+ * Test to verify that even after non recoverable error (error while
+ * writing transaction log) on ZooKeeper service will be available
+ */
+ @Test(timeout = 30000)
+ public void testNonRecoverableError() throws Exception {
+ ClientBase.setupTestEnv();
+
+ final int CLIENT_PORT = PortAssignment.unique();
+
+ MainThread main = new MainThread(CLIENT_PORT, true);
+ main.start();
+
+ Assert.assertTrue("waiting for server being up",
+ ClientBase.waitForServerUp("127.0.0.1:" + CLIENT_PORT,
+ CONNECTION_TIMEOUT));
+
+
+ ZooKeeper zk = new ZooKeeper("127.0.0.1:" + CLIENT_PORT,
+ ClientBase.CONNECTION_TIMEOUT, this);
+
+ zk.create("/foo1", "foobar".getBytes(), Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ Assert.assertEquals(new String(zk.getData("/foo1", null, null)), "foobar");
+
+ // inject problem in server
+ ZooKeeperServer zooKeeperServer = main.getCnxnFactory()
+ .getZooKeeperServer();
+ FileTxnSnapLog snapLog = zooKeeperServer.getTxnLogFactory();
+ FileTxnSnapLog fileTxnSnapLogWithError = new FileTxnSnapLog(
+ snapLog.getDataDir(), snapLog.getSnapDir()) {
+ @Override
+ public void commit() throws IOException {
+ throw new IOException("Input/output error");
+ }
+ };
+ ZKDatabase newDB = new ZKDatabase(fileTxnSnapLogWithError);
+ zooKeeperServer.setZKDatabase(newDB);
+
+ try {
+ // do create operation, so that injected IOException is thrown
+ zk.create("/foo2", "foobar".getBytes(), Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ fail("IOException is expected as error is injected in transaction log commit
funtionality");
+ } catch (Exception e) {
+ // do nothing
+ }
+ zk.close();
+ Assert.assertTrue("waiting for server down",
+ ClientBase.waitForServerDown("127.0.0.1:" + CLIENT_PORT,
+ ClientBase.CONNECTION_TIMEOUT));
+ fileTxnSnapLogWithError.close();
+ main.shutdown();
+ main.deleteDirs();
+ }
+
+ /**
* Verify the ability to start a standalone server instance.
*/
@Test
Modified: zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/quorum/QuorumPeerTestBase.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/quorum/QuorumPeerTestBase.java?rev=1756260&r1=1756259&r2=1756260&view=diff
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/quorum/QuorumPeerTestBase.java
(original)
+++ zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/server/quorum/QuorumPeerTestBase.java
Sat Aug 13 12:57:54 2016
@@ -146,5 +146,10 @@ public class QuorumPeerTestBase extends
ClientBase.recursiveDelete(main.quorumPeer.getTxnFactory()
.getDataDir());
}
+
+ public QuorumPeer getQuorumPeer() {
+ return main.quorumPeer;
+ }
+
}
}
Added: zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/test/NonRecoverableErrorTest.java
URL: http://svn.apache.org/viewvc/zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/test/NonRecoverableErrorTest.java?rev=1756260&view=auto
==============================================================================
--- zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/test/NonRecoverableErrorTest.java
(added)
+++ zookeeper/branches/branch-3.4/src/java/test/org/apache/zookeeper/test/NonRecoverableErrorTest.java
Sat Aug 13 12:57:54 2016
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.zookeeper.test;
+
+import static org.apache.zookeeper.test.ClientBase.CONNECTION_TIMEOUT;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.PortAssignment;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.server.ZKDatabase;
+import org.apache.zookeeper.server.persistence.FileTxnSnapLog;
+import org.apache.zookeeper.server.quorum.QuorumPeer;
+import org.apache.zookeeper.server.quorum.QuorumPeer.ServerState;
+import org.apache.zookeeper.server.quorum.QuorumPeerTestBase;
+import org.apache.zookeeper.test.ClientBase.CountdownWatcher;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * This class tests the non-recoverable error behavior of quorum server.
+ */
+public class NonRecoverableErrorTest extends QuorumPeerTestBase {
+ private static final String NODE_PATH = "/noLeaderIssue";
+
+ /**
+ * Test case for https://issues.apache.org/jira/browse/ZOOKEEPER-2247.
+ * Test to verify that even after non recoverable error (error while
+ * writing transaction log), ZooKeeper is still available.
+ */
+ @Test(timeout = 30000)
+ public void testZooKeeperServiceAvailableOnLeader() throws Exception {
+ int SERVER_COUNT = 3;
+ final int clientPorts[] = new int[SERVER_COUNT];
+ StringBuilder sb = new StringBuilder();
+ String server;
+
+ for (int i = 0; i < SERVER_COUNT; i++) {
+ clientPorts[i] = PortAssignment.unique();
+ server = "server." + i + "=127.0.0.1:" + PortAssignment.unique()
+ + ":" + PortAssignment.unique();
+ sb.append(server + "\n");
+ }
+ String currentQuorumCfgSection = sb.toString();
+ MainThread mt[] = new MainThread[SERVER_COUNT];
+
+ for (int i = 0; i < SERVER_COUNT; i++) {
+ mt[i] = new MainThread(i, clientPorts[i], currentQuorumCfgSection);
+ mt[i].start();
+ }
+
+ // ensure server started
+ for (int i = 0; i < SERVER_COUNT; i++) {
+ Assert.assertTrue("waiting for server " + i + " being up",
+ ClientBase.waitForServerUp("127.0.0.1:" + clientPorts[i],
+ CONNECTION_TIMEOUT));
+ }
+
+ CountdownWatcher watcher = new CountdownWatcher();
+ ZooKeeper zk = new ZooKeeper("127.0.0.1:" + clientPorts[0],
+ ClientBase.CONNECTION_TIMEOUT, watcher);
+ watcher.waitForConnected(ClientBase.CONNECTION_TIMEOUT);
+
+ String data = "originalData";
+ zk.create(NODE_PATH, data.getBytes(), Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+
+ // get information of current leader
+ QuorumPeer leader = getLeaderQuorumPeer(mt);
+ assertNotNull("Leader must have been elected by now", leader);
+
+ // inject problem in leader
+ FileTxnSnapLog snapLog = leader.getActiveServer().getTxnLogFactory();
+ FileTxnSnapLog fileTxnSnapLogWithError = new FileTxnSnapLog(
+ snapLog.getDataDir(), snapLog.getSnapDir()) {
+ @Override
+ public void commit() throws IOException {
+ throw new IOException("Input/output error");
+ }
+ };
+ ZKDatabase originalZKDatabase = leader.getActiveServer()
+ .getZKDatabase();
+ long leaderCurrentEpoch = leader.getCurrentEpoch();
+
+ ZKDatabase newDB = new ZKDatabase(fileTxnSnapLogWithError);
+ leader.getActiveServer().setZKDatabase(newDB);
+
+ try {
+ // do create operation, so that injected IOException is thrown
+ zk.create(uniqueZnode(), data.getBytes(), Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ fail("IOException is expected due to error injected to transaction log commit");
+ } catch (Exception e) {
+ // do nothing
+ }
+
+ // resetting watcher so that this watcher can be again used to ensure
+ // that the zkClient is able to re-establish connection with the
+ // newly elected zookeeper quorum.
+ watcher.reset();
+ waitForNewLeaderElection(leader, leaderCurrentEpoch);
+
+ // ensure server started, give enough time, so that new leader election
+ // takes place
+ for (int i = 0; i < SERVER_COUNT; i++) {
+ Assert.assertTrue("waiting for server " + i + " being up",
+ ClientBase.waitForServerUp("127.0.0.1:" + clientPorts[i],
+ CONNECTION_TIMEOUT));
+ }
+
+ // revert back the error
+ leader.getActiveServer().setZKDatabase(originalZKDatabase);
+
+ // verify that now ZooKeeper service is up and running
+ leader = getLeaderQuorumPeer(mt);
+ assertNotNull("New leader must have been elected by now", leader);
+
+ String uniqueNode = uniqueZnode();
+ watcher.waitForConnected(ClientBase.CONNECTION_TIMEOUT);
+ String createNode = zk.create(uniqueNode, data.getBytes(),
+ Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ // if node is created successfully then it means that ZooKeeper service
+ // is available
+ assertEquals("Failed to create znode", uniqueNode, createNode);
+ zk.close();
+ // stop all severs
+ for (int i = 0; i < SERVER_COUNT; i++) {
+ mt[i].shutdown();
+ }
+ }
+
+ private void waitForNewLeaderElection(QuorumPeer peer,
+ long leaderCurrentEpoch) throws IOException, InterruptedException {
+ LOG.info("Waiting for new LE cycle..");
+ int count = 100; // giving a grace period of 10seconds
+ while (count > 0) {
+ if (leaderCurrentEpoch == peer.getCurrentEpoch()) {
+ Thread.sleep(100);
+ }
+ count--;
+ }
+ Assert.assertTrue("New LE cycle must have triggered",
+ leaderCurrentEpoch != peer.getCurrentEpoch());
+ }
+
+ private QuorumPeer getLeaderQuorumPeer(MainThread[] mt) {
+ for (int i = mt.length - 1; i >= 0; i--) {
+ QuorumPeer quorumPeer = mt[i].getQuorumPeer();
+ if (null != quorumPeer
+ && ServerState.LEADING == quorumPeer.getPeerState()) {
+ return quorumPeer;
+ }
+ }
+ return null;
+ }
+
+ private String uniqueZnode() {
+ UUID randomUUID = UUID.randomUUID();
+ String node = NODE_PATH + "/" + randomUUID.toString();
+ return node;
+ }
+}
|