From commits-return-2451-archive-asf-public=cust-asf.ponee.io@bookkeeper.apache.org Wed Jan 24 19:56:51 2018 Return-Path: X-Original-To: archive-asf-public@eu.ponee.io Delivered-To: archive-asf-public@eu.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by mx-eu-01.ponee.io (Postfix) with ESMTP id 2A94E180630 for ; Wed, 24 Jan 2018 19:56:51 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 1B079160C3C; Wed, 24 Jan 2018 18:56:51 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id DF764160C39 for ; Wed, 24 Jan 2018 19:56:49 +0100 (CET) Received: (qmail 14692 invoked by uid 500); 24 Jan 2018 18:56:49 -0000 Mailing-List: contact commits-help@bookkeeper.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: bookkeeper-dev@bookkeeper.apache.org Delivered-To: mailing list commits@bookkeeper.apache.org Received: (qmail 14682 invoked by uid 99); 24 Jan 2018 18:56:49 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 24 Jan 2018 18:56:49 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 82A8C81652; Wed, 24 Jan 2018 18:56:48 +0000 (UTC) Date: Wed, 24 Jan 2018 18:56:48 +0000 To: "commits@bookkeeper.apache.org" Subject: [bookkeeper] branch master updated: Test compat with useHostNameAsBookieID MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <151682020843.7386.14033241791763030421@gitbox.apache.org> From: sijie@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: bookkeeper X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: c0cd03031c7decf29276e286904c431b2be75fab X-Git-Newrev: 305efca865f1cc8d6c89f6e6a53669e0b1265948 X-Git-Rev: 305efca865f1cc8d6c89f6e6a53669e0b1265948 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. sijie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/bookkeeper.git The following commit(s) were added to refs/heads/master by this push: new 305efca Test compat with useHostNameAsBookieID 305efca is described below commit 305efca865f1cc8d6c89f6e6a53669e0b1265948 Author: Ivan Kelly AuthorDate: Wed Jan 24 10:56:42 2018 -0800 Test compat with useHostNameAsBookieID Test that old client can continue to work with servers which are using their hostname as their bookie id. This replaces TestBackwardCompat#testCompatWrites and TestBackwardCompat#testCompatReads from the old BC tests. Master Issue: #903 Author: Ivan Kelly Reviewers: Sijie Guo This closes #1033 from ivankelly/bc-hostname --- tests/backward-compat-hostname-bookieid/pom.xml | 36 ++++++ .../TestCompatUpgradeWithHostnameBookieId.groovy | 131 +++++++++++++++++++++ .../src/test/resources/arquillian.xml | 28 +++++ .../tests/TestCompatUpgradeDirect.groovy | 3 +- .../tests/backward/TestBackwardCompat.java | 102 ---------------- tests/pom.xml | 1 + 6 files changed, 197 insertions(+), 104 deletions(-) diff --git a/tests/backward-compat-hostname-bookieid/pom.xml b/tests/backward-compat-hostname-bookieid/pom.xml new file mode 100644 index 0000000..3b09d7a --- /dev/null +++ b/tests/backward-compat-hostname-bookieid/pom.xml @@ -0,0 +1,36 @@ + + + + 4.0.0 + + org.apache.bookkeeper.tests + integration-tests-base-groovy + 4.7.0-SNAPSHOT + ../integration-tests-base-groovy + + + org.apache.bookkeeper.tests + backward-compat-hostname-bookieid + jar + Apache BookKeeper :: Tests :: Test upgrade between 4.1.0 and current version (with hostname bookie ID) + + diff --git a/tests/backward-compat-hostname-bookieid/src/test/groovy/org/apache/bookkeeper/tests/TestCompatUpgradeWithHostnameBookieId.groovy b/tests/backward-compat-hostname-bookieid/src/test/groovy/org/apache/bookkeeper/tests/TestCompatUpgradeWithHostnameBookieId.groovy new file mode 100644 index 0000000..67100de --- /dev/null +++ b/tests/backward-compat-hostname-bookieid/src/test/groovy/org/apache/bookkeeper/tests/TestCompatUpgradeWithHostnameBookieId.groovy @@ -0,0 +1,131 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.bookkeeper.tests + +import com.github.dockerjava.api.DockerClient + +import org.jboss.arquillian.junit.Arquillian +import org.jboss.arquillian.test.api.ArquillianResource + +import org.junit.Assert +import org.junit.Test +import org.junit.runner.RunWith + +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +@RunWith(Arquillian.class) +class TestCompatUpgradeWithHostnameBookieId { + private static final Logger LOG = LoggerFactory.getLogger(TestCompatUpgradeWithHostnameBookieId.class) + private static byte[] PASSWD = "foobar".getBytes() + + private def oldClientVersions = ["4.1.0", "4.2.0", "4.2.1", "4.2.2", "4.2.3", "4.2.4", + "4.3.0", "4.3.1", "4.3.2", "4.4.0", "4.5.0", "4.5.1", + "4.6.0"] + + @ArquillianResource + DockerClient docker + + + private void writeEntries(def ledger, int numEntries) throws Exception { + for (int i = 0; i < numEntries; i++) { + ledger.addEntry(("foobar" + i).getBytes()) + } + } + + private void assertHasEntries(def ledger, int numEntries) throws Exception { + Assert.assertEquals(numEntries, ledger.getLastAddConfirmed() + 1 /* counts from 0 */) + def entries = ledger.readEntries(0, ledger.getLastAddConfirmed()) + int j = 0 + while (entries.hasMoreElements()) { + def e = entries.nextElement() + Assert.assertEquals(new String(e.getEntry()), "foobar"+ j) + j++ + } + } + + /** + * Test compatability between version old version and the current version. + * - 4.1.0 server restarts with useHostNameAsBookieID=true. + * - Write ledgers with old and new clients + * - Read ledgers written by old clients. + */ + @Test + public void testOldClientsWorkWithNewServerUsingHostnameAsBookieID() throws Exception { + int numEntries = 10 + BookKeeperClusterUtils.legacyMetadataFormat(docker) + String zookeeper = BookKeeperClusterUtils.zookeeperConnectString(docker) + + String currentVersion = System.getProperty("currentVersion") + + Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, "4.1.0")) + + // Write a ledger with v4.1.0 client + def v410CL = MavenClassLoader.forBookKeeperVersion("4.1.0") + def v410BK = v410CL.newBookKeeper(zookeeper) + + def ledger410 = v410BK.createLedger(3, 2, v410CL.digestType("CRC32"), PASSWD) + writeEntries(ledger410, numEntries) + ledger410.close() + + // Write a ledger with v4.2.0 client + def v420CL = MavenClassLoader.forBookKeeperVersion("4.2.0") + def v420BK = v420CL.newBookKeeper(zookeeper) + + def ledger420 = v420BK.createLedger(3, 2, v420CL.digestType("CRC32"), PASSWD) + writeEntries(ledger420, numEntries) + ledger420.close() + + // Stop bookies, change config to use hostname as id, restart with latest version + Assert.assertTrue(BookKeeperClusterUtils.stopAllBookies(docker)) + BookKeeperClusterUtils.updateAllBookieConf(docker, currentVersion, "useHostNameAsBookieID", "true") + Assert.assertTrue(BookKeeperClusterUtils.startAllBookiesWithVersion(docker, currentVersion)) + + // Ensure we can read ledger with v4.1.0 client + def ledger410r = v410BK.openLedger(ledger410.getId(), v410CL.digestType("CRC32"), PASSWD) + assertHasEntries(ledger410r, numEntries) + ledger410r.close() + + // Ensure we can read ledger with v4.2.0 client + def ledger420r = v420BK.openLedger(ledger420.getId(), v420CL.digestType("CRC32"), PASSWD) + assertHasEntries(ledger420r, numEntries) + ledger420r.close() + + // Ensure we can write and read new ledgers with all client versions + def currentCL = MavenClassLoader.forBookKeeperVersion(currentVersion) + def currentBK = currentCL.newBookKeeper(zookeeper) + oldClientVersions.each{ + LOG.info("Testing ledger creation for version {}", it) + def oldCL = MavenClassLoader.forBookKeeperVersion(it) + def oldBK = oldCL.newBookKeeper(zookeeper) + + def ledger0 = oldBK.createLedger(3, 2, oldCL.digestType("CRC32"), PASSWD) + writeEntries(ledger0, numEntries) + ledger0.close() + + def ledger1 = currentBK.openLedger(ledger0.getId(), currentCL.digestType("CRC32"), PASSWD) + assertHasEntries(ledger1, numEntries) + ledger1.close() + + oldBK.close() + } + currentBK.close() + } + + +} diff --git a/tests/backward-compat-hostname-bookieid/src/test/resources/arquillian.xml b/tests/backward-compat-hostname-bookieid/src/test/resources/arquillian.xml new file mode 100644 index 0000000..f914ff2 --- /dev/null +++ b/tests/backward-compat-hostname-bookieid/src/test/resources/arquillian.xml @@ -0,0 +1,28 @@ + + + + + + CUBE + cube-definitions/3-node-all-version-unstarted.yaml + + + diff --git a/tests/backward-compat-upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/TestCompatUpgradeDirect.groovy b/tests/backward-compat-upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/TestCompatUpgradeDirect.groovy index cc32991..2f7fb67 100644 --- a/tests/backward-compat-upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/TestCompatUpgradeDirect.groovy +++ b/tests/backward-compat-upgrade-direct/src/test/groovy/org/apache/bookkeeper/tests/TestCompatUpgradeDirect.groovy @@ -32,7 +32,6 @@ import org.slf4j.Logger import org.slf4j.LoggerFactory @RunWith(Arquillian.class) -@FixMethodOrder(MethodSorters.NAME_ASCENDING) class TestCompatUpgradeDirect { private static final Logger LOG = LoggerFactory.getLogger(TestCompatUpgradeDirect.class) private static byte[] PASSWD = "foobar".getBytes() @@ -91,7 +90,7 @@ class TestCompatUpgradeDirect { } @Test - public void test1_v410ClientCantFenceLedgerFromCurrent() throws Exception { + public void test9_v410ClientCantFenceLedgerFromCurrent() throws Exception { String currentVersion = System.getProperty("currentVersion") String zookeeper = BookKeeperClusterUtils.zookeeperConnectString(docker) diff --git a/tests/backward/src/test/java/org/apache/bookkeeper/tests/backward/TestBackwardCompat.java b/tests/backward/src/test/java/org/apache/bookkeeper/tests/backward/TestBackwardCompat.java index 7a42fc2..7d86eb7 100644 --- a/tests/backward/src/test/java/org/apache/bookkeeper/tests/backward/TestBackwardCompat.java +++ b/tests/backward/src/test/java/org/apache/bookkeeper/tests/backward/TestBackwardCompat.java @@ -20,7 +20,6 @@ */ package org.apache.bookkeeper.tests.backward; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -438,105 +437,4 @@ public class TestBackwardCompat { } } - /** - * Test compatability between old versions and the current version. - * - old server restarts with useHostNameAsBookieID=true. - * - Read ledgers with old and new clients - */ - @Test - public void testCompatReads() throws Exception { - File journalDir = createTempDir("bookie", "journal"); - File ledgerDir = createTempDir("bookie", "ledger"); - - int port = PortManager.nextFreePort(); - // start server, upgrade - Server410 s410 = new Server410(journalDir, ledgerDir, port); - s410.start(); - - Ledger410 l410 = Ledger410.newLedger(); - l410.write100(); - long oldLedgerId = l410.getId(); - l410.close(); - - // Check that 420 client can to write to 410 server - Ledger420 l420 = Ledger420.newLedger(); - l420.write100(); - long lid420 = l420.getId(); - l420.close(); - - s410.stop(); - - // Start the current server, will not require a filesystem upgrade - ServerCurrent scur = new ServerCurrent(journalDir, ledgerDir, port, - false); - scur.start(); - - // check that old client can read its old ledgers on new server - l410 = Ledger410.openLedger(oldLedgerId); - assertEquals(100, l410.readAll()); - l410.close(); - - // Check that 420 client can read old ledgers on new server - l420 = Ledger420.openLedger(lid420); - assertEquals("Failed to read entries!", 100, l420.readAll()); - l420.close(); - - // Check that current client can read old ledgers on new server - final LedgerCurrent curledger = LedgerCurrent.openLedger(lid420); - assertEquals("Failed to read entries!", 100, curledger.readAll()); - curledger.close(); - } - - /** - * Test compatability between version old version and the current version. - * - 4.1.0 server restarts with useHostNameAsBookieID=true. - * - Write ledgers with old and new clients - * - Read ledgers written by old clients. - */ - @Test - public void testCompatWrites() throws Exception { - File journalDir = createTempDir("bookie", "journal"); - File ledgerDir = createTempDir("bookie", "ledger"); - - int port = PortManager.nextFreePort(); - // start server, upgrade - Server410 s410 = new Server410(journalDir, ledgerDir, port); - s410.start(); - s410.stop(); - - // Start the current server, will not require a filesystem upgrade - ServerCurrent scur = new ServerCurrent(journalDir, ledgerDir, port, - false); - scur.start(); - - // Check that current client can to write to server - LedgerCurrent lcur = LedgerCurrent.newLedger(); - lcur.write100(); - lcur.close(); - final LedgerCurrent curledger = LedgerCurrent.openLedger(lcur.getId()); - assertEquals("Failed to read entries!", 100, curledger.readAll()); - - // Check that 410 client can write to server - Ledger410 l410 = Ledger410.newLedger(); - l410.write100(); - long oldLedgerId = l410.getId(); - l410.close(); - - // Check that 420 client can write to server - Ledger410 l420 = Ledger410.newLedger(); - l420.write100(); - long lid420 = l420.getId(); - l420.close(); - - // check that new client can read old ledgers on new server - LedgerCurrent oldledger = LedgerCurrent.openLedger(oldLedgerId); - assertEquals("Failed to read entries!", 100, oldledger.readAll()); - oldledger.close(); - - // check that new client can read old ledgers on new server - oldledger = LedgerCurrent.openLedger(lid420); - assertEquals("Failed to read entries!", 100, oldledger.readAll()); - oldledger.close(); - scur.stop(); - } } diff --git a/tests/pom.xml b/tests/pom.xml index 8382eb2..f3df8d4 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -43,6 +43,7 @@ backward-compat-upgrade backward-compat-upgrade-direct backward-compat-hierarchical-ledger-manager + backward-compat-hostname-bookieid backward-compat-recovery-no-password -- To stop receiving notification emails like this one, please contact sijie@apache.org.