Author: ivank
Date: Tue Apr  2 15:38:49 2013
New Revision: 1463599

URL: http://svn.apache.org/r1463599
Log:
BOOKKEEPER-595: Crash of inprocess autorecovery daemon should not take down the bookie (ivank)

Added:
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/TestDeathwatcher.java
Modified:
    zookeeper/bookkeeper/trunk/CHANGES.txt
    zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java

Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1463599&r1=1463598&r2=1463599&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Tue Apr  2 15:38:49 2013
@@ -10,6 +10,8 @@ Trunk (unreleased changes)
 
     BUGFIXES:
 
+      BOOKKEEPER-595: Crash of inprocess autorecovery daemon should not take down the bookie (ivank)
+
       BOOKKEEPER-597: Add flag to output test logs to stdout (ivank)
 
       BOOKKEEPER-579: TestSubAfterCloseSub was put in a wrong package (sijie via ivank)

Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java?rev=1463599&r1=1463598&r2=1463599&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java Tue Apr  2 15:38:49 2013
@@ -65,7 +65,7 @@ public class BookieServer {
 
     // operation stats
     protected BookieServerBean jmxBkServerBean;
-    private AutoRecoveryMain autoRecoveryMain = null;
+    AutoRecoveryMain autoRecoveryMain = null;
     private boolean isAutoRecoveryDaemonEnabled;
 
     public BookieServer(ServerConfiguration conf) throws IOException,
@@ -219,10 +219,12 @@ public class BookieServer {
                 } catch (InterruptedException ie) {
                     // do nothing
                 }
-                if (!isBookieRunning()
-                        || (isAutoRecoveryDaemonEnabled && !isAutoRecoveryRunning())) {
+                if (!isBookieRunning()) {
                     shutdown();
-                    break;
+                }
+                if (isAutoRecoveryDaemonEnabled && !isAutoRecoveryRunning()) {
+                    LOG.error("Autorecovery daemon has stopped. Please check the logs");
+                    isAutoRecoveryDaemonEnabled = false; // to avoid spamming the logs
                 }
             }
         }

Added: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/TestDeathwatcher.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/TestDeathwatcher.java?rev=1463599&view=auto
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/TestDeathwatcher.java (added)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/proto/TestDeathwatcher.java Tue Apr  2 15:38:49 2013
@@ -0,0 +1,60 @@
+package org.apache.bookkeeper.proto;
+
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+import org.junit.*;
+
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Tests for the BookieServer death watcher
+ */
+public class TestDeathwatcher extends BookKeeperClusterTestCase {
+    static Logger LOG = LoggerFactory.getLogger(TestDeathwatcher.class);
+
+    public TestDeathwatcher() {
+        super(1);
+    }
+
+    /**
+     * Ensure that if the autorecovery daemon is running inside the bookie
+     * then a failure/crash in the autorecovery daemon will not take down the
+     * bookie also.
+     */
+    @Test(timeout=30000)
+    public void testAutorecoveryFailureDoesntKillBookie() throws Exception {
+        ServerConfiguration conf = newServerConfiguration().setAutoRecoveryDaemonEnabled(true);
+        BookieServer bs = startBookie(conf);
+
+        assertNotNull("Autorecovery daemon should exist", bs.autoRecoveryMain);
+        assertTrue("Bookie should be running", bs.isBookieRunning());
+        bs.autoRecoveryMain.shutdown();
+        Thread.sleep(conf.getDeathWatchInterval()*2); // give deathwatcher time to run
+        assertTrue("Bookie should be running", bs.isBookieRunning());
+        bs.shutdown();
+    }
+}
+