Mailing-List: contact commits-help@zookeeper.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@<zookeeper.apache.org>
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1592739 - in /zookeeper/bookkeeper/branches/branch-4.2:
 ./
 bookkeeper-server/conf/
 bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/
 bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/
 bookkeeper-server/src/test/ja...
Date: Tue, 06 May 2014 13:13:16 -0000
To: commits@zookeeper.apache.org
From: ivank@apache.org
Message-Id: <20140506131316.D4960238889B@eris.apache.org>

Author: ivank
Date: Tue May  6 13:13:16 2014
New Revision: 1592739

URL: http://svn.apache.org/r1592739
Log:
BOOKKEEPER-744: Run the auditor bookie check periodically (ivank)

Added:
    zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java
    zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/TestCallbacks.java
Modified:
    zookeeper/bookkeeper/branches/branch-4.2/CHANGES.txt
    zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/conf/bk_server.conf
    zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
    zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java

Modified: zookeeper/bookkeeper/branches/branch-4.2/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/branches/branch-4.2/CHANGES.txt?rev=1592739&r1=1592738&r2=1592739&view=diff
==============================================================================
--- zookeeper/bookkeeper/branches/branch-4.2/CHANGES.txt (original)
+++ zookeeper/bookkeeper/branches/branch-4.2/CHANGES.txt Tue May  6 13:13:16 2014
@@ -24,6 +24,8 @@ Release 4.2.3 - 2013-12-04
 
         BOOKKEEPER-743: Periodic ledger check running too often as doc doesn't match implementation. (ivank)
 
+        BOOKKEEPER-744: Run the auditor bookie check periodically (ivank)
+
 Release 4.2.2 - 2013-10-02
 
   Backward compatible changes:

Modified: zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/conf/bk_server.conf
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/conf/bk_server.conf?rev=1592739&r1=1592738&r2=1592739&view=diff
==============================================================================
--- zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/conf/bk_server.conf (original)
+++ zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/conf/bk_server.conf Tue May  6 13:13:16 2014
@@ -213,3 +213,12 @@ zkTimeout=10000
 # Note that periodic checking will put extra load on the cluster, so it should
 # not be run more frequently than once a day.
 #auditorPeriodicCheckInterval=604800
+
+# The interval between auditor bookie checks.
+# The auditor bookie check, checks ledger metadata to see which bookies should
+# contain entries for each ledger. If a bookie which should contain entries is
+# unavailable, then the ledger containing that entry is marked for recovery.
+# Setting this to 0 disabled the periodic check. Bookie checks will still
+# run when a bookie fails.
+# The interval is specified in seconds.
+#auditorPeriodicBookieCheckInterval=84600

Modified: zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java?rev=1592739&r1=1592738&r2=1592739&view=diff
==============================================================================
--- zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java (original)
+++ zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/main/java/org/apache/bookkeeper/conf/ServerConfiguration.java Tue May  6 13:13:16 2014
@@ -70,6 +70,7 @@ public class ServerConfiguration extends
     protected final static String DISK_USAGE_THRESHOLD = "diskUsageThreshold";
     protected final static String DISK_CHECK_INTERVAL = "diskCheckInterval";
     protected final static String AUDITOR_PERIODIC_CHECK_INTERVAL = "auditorPeriodicCheckInterval";
+    protected final static String AUDITOR_PERIODIC_BOOKIE_CHECK_INTERVAL = "auditorPeriodicBookieCheckInterval";
     protected final static String AUTO_RECOVERY_DAEMON_ENABLED = "autoRecoveryDaemonEnabled";
 
     /**
@@ -736,6 +737,29 @@ public class ServerConfiguration extends
     }
 
     /**
+     * Set the interval between auditor bookie checks.
+     * The auditor bookie check, checks ledger metadata to see which bookies
+     * contain entries for each ledger. If a bookie which should contain entries
+     * is unavailable, then the ledger containing that entry is marked for recovery.
+     * Setting this to 0 disabled the periodic check. Bookie checks will still
+     * run when a bookie fails.
+     *
+     * @param interval The period in seconds.
+     */
+    public void setAuditorPeriodicBookieCheckInterval(long interval) {
+        setProperty(AUDITOR_PERIODIC_BOOKIE_CHECK_INTERVAL, interval);
+    }
+
+    /**
+     * Get the interval between auditor bookie check runs.
+     * @see #setAuditorPeriodicBookieCheckInterval(long)
+     * @return the interval between bookie check runs, in seconds. Default is 84600 (= 1 day)
+     */
+    public long getAuditorPeriodicBookieCheckInterval() {
+        return getLong(AUDITOR_PERIODIC_BOOKIE_CHECK_INTERVAL, 84600);
+    }
+
+    /**
      * Sets that whether the auto-recovery service can start along with Bookie
      * server itself or not
      *

Modified: zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java?rev=1592739&r1=1592738&r2=1592739&view=diff
==============================================================================
--- zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java (original)
+++ zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/main/java/org/apache/bookkeeper/replication/Auditor.java Tue May  6 13:13:16 2014
@@ -203,8 +203,10 @@ public class Auditor implements BookiesL
             }
 
             long interval = conf.getAuditorPeriodicCheckInterval();
+
             if (interval > 0) {
-                LOG.info("Periodic checking enabled");
+                LOG.info("Auditor periodic ledger checking enabled"
+                         + " 'auditorPeriodicCheckInterval' {} seconds", interval);
                 executor.scheduleAtFixedRate(new Runnable() {
                         public void run() {
                             LOG.info("Running periodic check");
@@ -237,7 +239,7 @@ public class Auditor implements BookiesL
                 LOG.info("Periodic checking disabled");
             }
 
-            executor.submit(new Runnable() {
+            Runnable bookieCheck = new Runnable() {
                     public void run() {
                         try {
                             knownBookies = getAvailableBookies();
@@ -257,7 +259,17 @@ public class Auditor implements BookiesL
                             submitShutdownTask();
                         }
                     }
-                });
+                };
+
+            long bookieCheckInterval = conf.getAuditorPeriodicBookieCheckInterval();
+            if (bookieCheckInterval == 0) {
+                LOG.info("Auditor periodic bookie checking disabled, running once check now anyhow");
+                executor.submit(bookieCheck);
+            } else {
+                LOG.info("Auditor periodic bookie checking enabled"
+                         + " 'auditorPeriodicBookieCheckInterval' {} seconds", bookieCheckInterval);
+                executor.scheduleAtFixedRate(bookieCheck, 0, bookieCheckInterval, TimeUnit.SECONDS);
+            }
         }
     }
 

Added: zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java?rev=1592739&view=auto
==============================================================================
--- zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java (added)
+++ zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/test/java/org/apache/bookkeeper/replication/AuditorPeriodicBookieCheckTest.java Tue May  6 13:13:16 2014
@@ -0,0 +1,127 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.bookkeeper.replication;
+
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.bookkeeper.test.TestCallbacks;
+
+import java.util.List;
+import java.net.InetSocketAddress;
+
+import org.apache.bookkeeper.util.StringUtils;
+import org.apache.bookkeeper.zookeeper.ZooKeeperWatcherBase;
+import org.apache.bookkeeper.client.BookKeeper.DigestType;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.client.LedgerHandleAdapter;
+import org.apache.bookkeeper.client.LedgerMetadata;
+import org.apache.bookkeeper.conf.ServerConfiguration;
+import org.apache.bookkeeper.meta.LedgerManager;
+import org.apache.bookkeeper.meta.LedgerManagerFactory;
+import org.apache.bookkeeper.meta.LedgerUnderreplicationManager;
+
+import org.apache.bookkeeper.util.ZkUtils;
+
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.Before;
+import org.junit.After;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This test verifies that the period check on the auditor
+ * will pick up on missing data in the client
+ */
+public class AuditorPeriodicBookieCheckTest extends BookKeeperClusterTestCase {
+    private final static Logger LOG = LoggerFactory
+            .getLogger(AuditorPeriodicBookieCheckTest.class);
+
+    private AuditorElector auditorElector = null;
+    private ZooKeeper auditorZookeeper = null;
+
+    private final static int CHECK_INTERVAL = 1; // run every second
+
+    public AuditorPeriodicBookieCheckTest() {
+        super(3);
+        baseConf.setPageLimit(1); // to make it easy to push ledger out of cache
+        baseConf.setAllowLoopback(true);
+    }
+
+    @Before
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+
+        ServerConfiguration conf = new ServerConfiguration(bsConfs.get(0));
+        conf.setAllowLoopback(true);
+        conf.setAuditorPeriodicBookieCheckInterval(CHECK_INTERVAL);
+        String addr = StringUtils.addrToString(bs.get(0).getLocalAddress());
+
+        ZooKeeperWatcherBase w = new ZooKeeperWatcherBase(10000);
+        auditorZookeeper = ZkUtils.createConnectedZookeeperClient(
+                zkUtil.getZooKeeperConnectString(), w);
+
+        auditorElector = new AuditorElector(addr, conf, auditorZookeeper);
+        auditorElector.start();
+    }
+
+    @After
+    @Override
+    public void tearDown() throws Exception {
+        auditorElector.shutdown();
+        auditorZookeeper.close();
+
+        super.tearDown();
+    }
+
+    /**
+     * Test that the periodic bookie checker works
+     */
+    @Test(timeout=30000)
+    public void testPeriodicBookieCheckInterval() throws Exception {
+        LedgerManagerFactory mFactory = LedgerManagerFactory.newLedgerManagerFactory(bsConfs.get(0), zkc);
+        LedgerManager ledgerManager = mFactory.newLedgerManager();
+        final LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
+        final int numLedgers = 1;
+
+        LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
+        LedgerMetadata md = LedgerHandleAdapter.getLedgerMetadata(lh);
+
+        List<InetSocketAddress> ensemble = md.getEnsembles().get(0L);
+        ensemble.set(0, new InetSocketAddress("1.1.1.1", 1000));
+
+        TestCallbacks.GenericCallbackFuture<Void> cb = new TestCallbacks.GenericCallbackFuture<Void>();
+        ledgerManager.writeLedgerMetadata(lh.getId(), md, cb);
+        cb.get();
+
+        long underReplicatedLedger = -1;
+        for (int i = 0; i < 10; i++) {
+            underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
+            if (underReplicatedLedger != -1) {
+                break;
+            }
+            Thread.sleep(CHECK_INTERVAL*1000);
+        }
+        assertEquals("Ledger should be under replicated", lh.getId(), underReplicatedLedger);
+    }
+}

Added: zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/TestCallbacks.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/TestCallbacks.java?rev=1592739&view=auto
==============================================================================
--- zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/TestCallbacks.java (added)
+++ zookeeper/bookkeeper/branches/branch-4.2/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/TestCallbacks.java Tue May  6 13:13:16 2014
@@ -0,0 +1,43 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.bookkeeper.test;
+
+import org.apache.bookkeeper.client.BKException;
+import org.apache.bookkeeper.proto.BookkeeperInternalCallbacks.GenericCallback;
+import com.google.common.util.concurrent.AbstractFuture;
+
+/**
+ * Callbacks implemented with SettableFuture, to be used in tests
+ */
+public class TestCallbacks {
+    public static class GenericCallbackFuture<T>
+        extends AbstractFuture<T> implements GenericCallback<T> {
+        @Override
+        public void operationComplete(int rc, T value) {
+            if (rc != BKException.Code.OK) {
+                setException(BKException.create(rc));
+            } else {
+                set(value);
+            }
+        }
+    }
+}
+