Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 489BE200BEE for ; Sat, 17 Dec 2016 02:42:24 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 47404160B35; Sat, 17 Dec 2016 01:42:24 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 433D8160B24 for ; Sat, 17 Dec 2016 02:42:23 +0100 (CET) Received: (qmail 65932 invoked by uid 500); 17 Dec 2016 01:42:22 -0000 Mailing-List: contact commits-help@bookkeeper.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: bookkeeper-dev@bookkeeper.apache.org Delivered-To: mailing list commits@bookkeeper.apache.org Received: (qmail 65921 invoked by uid 99); 17 Dec 2016 01:42:22 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 17 Dec 2016 01:42:22 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 65F53DF9A3; Sat, 17 Dec 2016 01:42:22 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: sijie@apache.org To: commits@bookkeeper.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: bookkeeper git commit: BOOKKEEPER-967: New testsuite for RackPlacement Date: Sat, 17 Dec 2016 01:42:22 +0000 (UTC) archived-at: Sat, 17 Dec 2016 01:42:24 -0000 Repository: bookkeeper Updated Branches: refs/heads/master d2e82d494 -> 4bb57ef0b BOOKKEEPER-967: New testsuite for RackPlacement Added New Testsuite for testing RackAwareEnsemblePlacementPolicy using ScriptBasedMapping. It works only on Unix based OS, because of Shell script. This testsuite has similar testscenarios as in TestRackawareEnsemblePlacementPolicy.java. Also here it is assumed that the script file has execution permission. Author: Charan Reddy Guttapalem Reviewers: Sijie Guo Closes #86 from reddycharan/testrackawareusingscript Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/4bb57ef0 Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/4bb57ef0 Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/4bb57ef0 Branch: refs/heads/master Commit: 4bb57ef0b1b72eb1118963efc3ab9f0e9bec0e1c Parents: d2e82d4 Author: Charan Reddy Guttapalem Authored: Fri Dec 16 17:42:18 2016 -0800 Committer: Sijie Guo Committed: Fri Dec 16 17:42:18 2016 -0800 ---------------------------------------------------------------------- ...awareEnsemblePlacementPolicyUsingScript.java | 271 +++++++++++++++++++ .../src/test/resources/networkmappingscript.sh | 14 + 2 files changed, 285 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/4bb57ef0/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicyUsingScript.java ---------------------------------------------------------------------- diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicyUsingScript.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicyUsingScript.java new file mode 100644 index 0000000..ff53d7f --- /dev/null +++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/TestRackawareEnsemblePlacementPolicyUsingScript.java @@ -0,0 +1,271 @@ +package org.apache.bookkeeper.client; + +import static org.apache.bookkeeper.client.RackawareEnsemblePlacementPolicy.REPP_DNS_RESOLVER_CLASS; +import static org.apache.bookkeeper.feature.SettableFeatureProvider.DISABLE_ALL; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import org.apache.bookkeeper.client.BKException.BKNotEnoughBookiesException; +import org.apache.bookkeeper.conf.ClientConfiguration; +import org.apache.bookkeeper.net.BookieSocketAddress; +import org.apache.bookkeeper.net.CommonConfigurationKeys; +import org.apache.bookkeeper.net.DNSToSwitchMapping; +import org.apache.bookkeeper.net.ScriptBasedMapping; +import org.apache.bookkeeper.util.Shell; +import org.jboss.netty.util.HashedWheelTimer; +import org.junit.After; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Optional; +import com.google.common.util.concurrent.ThreadFactoryBuilder; + +/** + * In this testsuite, ScriptBasedMapping is used as DNS_RESOLVER_CLASS for + * mapping nodes to racks. Shell Script - + * src/test/resources/networkmappingscript.sh is used in ScriptBasedMapping for + * resolving racks. This script maps HostAddress to rack depending on the last + * character of the HostAddress string. for eg. 127.0.0.1 :- /1, 127.0.0.2 :- + * /2, 99.12.34.21 :- /1 + * + * This testsuite has same testscenarios as in + * TestRackawareEnsemblePlacementPolicy.java. + * + * For now this Testsuite works only on Unix based OS. + */ +public class TestRackawareEnsemblePlacementPolicyUsingScript { + + static final Logger LOG = LoggerFactory.getLogger(TestRackawareEnsemblePlacementPolicyUsingScript.class); + + HashedWheelTimer timer; + RackawareEnsemblePlacementPolicy repp; + ClientConfiguration conf = new ClientConfiguration(); + + @Before + public void setUp() throws Exception { + conf.setProperty(REPP_DNS_RESOLVER_CLASS, ScriptBasedMapping.class.getName()); + conf.setProperty(CommonConfigurationKeys.NET_TOPOLOGY_SCRIPT_FILE_NAME_KEY, + "src/test/resources/networkmappingscript.sh"); + timer = new HashedWheelTimer( + new ThreadFactoryBuilder().setNameFormat("TestTimer-%d").build(), + conf.getTimeoutTimerTickDurationMs(), TimeUnit.MILLISECONDS, + conf.getTimeoutTimerNumTicks()); + + repp = new RackawareEnsemblePlacementPolicy(); + repp.initialize(conf, Optional.absent(), timer, DISABLE_ALL, null); + } + + @After + public void tearDown() throws Exception { + repp.uninitalize(); + } + + private void ignoreTestIfItIsWindowsOS() { + Assume.assumeTrue(!Shell.WINDOWS); + } + + @Test(timeout = 60000) + public void testReplaceBookieWithEnoughBookiesInSameRack() throws Exception { + ignoreTestIfItIsWindowsOS(); + BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181); // /1 rack + BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.2", 3181); // /2 rack + BookieSocketAddress addr3 = new BookieSocketAddress("127.0.1.2", 3181); // /2 rack + BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181); // /4 rack + + // Update cluster + Set addrs = new HashSet(); + addrs.add(addr1); + addrs.add(addr2); + addrs.add(addr3); + addrs.add(addr4); + repp.onClusterChanged(addrs, new HashSet()); + // replace node under r2 + BookieSocketAddress replacedBookie = repp.replaceBookie(1, 1, 1, null, new HashSet(), addr2, new HashSet()); + assertEquals(addr3, replacedBookie); + } + + @Test(timeout = 60000) + public void testReplaceBookieWithEnoughBookiesInDifferentRack() throws Exception { + ignoreTestIfItIsWindowsOS(); + BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181); // /1 rack + BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.2", 3181); // /2 rack + BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.3", 3181); // /3 rack + BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181); // /4 rack + + // Update cluster + Set addrs = new HashSet(); + addrs.add(addr1); + addrs.add(addr2); + addrs.add(addr3); + addrs.add(addr4); + repp.onClusterChanged(addrs, new HashSet()); + // replace node under r2 + Set excludedAddrs = new HashSet(); + excludedAddrs.add(addr1); + BookieSocketAddress replacedBookie = repp.replaceBookie(1, 1, 1, null, new HashSet(), addr2, excludedAddrs); + + assertFalse(addr1.equals(replacedBookie)); + assertTrue(addr3.equals(replacedBookie) || addr4.equals(replacedBookie)); + } + + @Test(timeout = 60000) + public void testReplaceBookieWithNotEnoughBookies() throws Exception { + ignoreTestIfItIsWindowsOS(); + BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181); // /1 rack + BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.2", 3181); // /2 rack + BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.3", 3181); // /3 rack + BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181); // /4 rack + + // Update cluster + Set addrs = new HashSet(); + addrs.add(addr1); + addrs.add(addr2); + addrs.add(addr3); + addrs.add(addr4); + repp.onClusterChanged(addrs, new HashSet()); + // replace node under r2 + Set excludedAddrs = new HashSet(); + excludedAddrs.add(addr1); + excludedAddrs.add(addr3); + excludedAddrs.add(addr4); + try { + repp.replaceBookie(1, 1, 1, null, new HashSet(), addr2, excludedAddrs); + fail("Should throw BKNotEnoughBookiesException when there is not enough bookies"); + } catch (BKNotEnoughBookiesException bnebe) { + // should throw not BKNotEnoughBookiesException + } + } + + @Test(timeout = 60000) + public void testNewEnsembleWithSingleRack() throws Exception { + ignoreTestIfItIsWindowsOS(); + BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181); // /1 rack + BookieSocketAddress addr2 = new BookieSocketAddress("127.0.1.1", 3181); // /1 rack + BookieSocketAddress addr3 = new BookieSocketAddress("127.0.2.1", 3181); // /1 rack + BookieSocketAddress addr4 = new BookieSocketAddress("127.0.3.1", 3181); // /1 rack + // Update cluster + Set addrs = new HashSet(); + addrs.add(addr1); + addrs.add(addr2); + addrs.add(addr3); + addrs.add(addr4); + repp.onClusterChanged(addrs, new HashSet()); + try { + ArrayList ensemble = repp.newEnsemble(3, 2, 2, null, new HashSet()); + assertEquals(0, getNumCoveredWriteQuorums(ensemble, 2)); + ArrayList ensemble2 = repp.newEnsemble(4, 2, 2, null, new HashSet()); + assertEquals(0, getNumCoveredWriteQuorums(ensemble2, 2)); + } catch (BKNotEnoughBookiesException bnebe) { + fail("Should not get not enough bookies exception even there is only one rack."); + } + } + + @Test(timeout = 60000) + public void testNewEnsembleWithMultipleRacks() throws Exception { + ignoreTestIfItIsWindowsOS(); + BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181); // /1 rack + BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.2", 3181); // /2 rack + BookieSocketAddress addr3 = new BookieSocketAddress("127.0.1.2", 3181); // /2 rack + BookieSocketAddress addr4 = new BookieSocketAddress("127.0.2.2", 3181); // /2 rack + // Update cluster + Set addrs = new HashSet(); + addrs.add(addr1); + addrs.add(addr2); + addrs.add(addr3); + addrs.add(addr4); + repp.onClusterChanged(addrs, new HashSet()); + try { + ArrayList ensemble = repp.newEnsemble(3, 2, 2, null, new HashSet()); + int numCovered = getNumCoveredWriteQuorums(ensemble, 2); + assertTrue(numCovered == 2); + ArrayList ensemble2 = repp.newEnsemble(4, 2, 2, null, new HashSet()); + numCovered = getNumCoveredWriteQuorums(ensemble2, 2); + assertTrue(numCovered == 2); + } catch (BKNotEnoughBookiesException bnebe) { + fail("Should not get not enough bookies exception"); + } + } + + @Test(timeout = 90000) + public void testNewEnsembleWithEnoughRacks() throws Exception { + ignoreTestIfItIsWindowsOS(); + BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181); // /1 rack + BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.2", 3181); // /2 rack + BookieSocketAddress addr3 = new BookieSocketAddress("127.0.0.3", 3181); // /3 rack + BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181); // /4 rack + BookieSocketAddress addr5 = new BookieSocketAddress("127.0.1.1", 3181); // /1 rack + BookieSocketAddress addr6 = new BookieSocketAddress("127.0.1.2", 3181); // /2 rack + BookieSocketAddress addr7 = new BookieSocketAddress("127.0.1.3", 3181); // /3 rack + BookieSocketAddress addr8 = new BookieSocketAddress("127.0.1.4", 3181); // /4 rack + // Update cluster + Set addrs = new HashSet(); + addrs.add(addr1); + addrs.add(addr2); + addrs.add(addr3); + addrs.add(addr4); + addrs.add(addr5); + addrs.add(addr6); + addrs.add(addr7); + addrs.add(addr8); + repp.onClusterChanged(addrs, new HashSet()); + try { + ArrayList ensemble1 = repp.newEnsemble(3, 2, 2, null, new HashSet()); + assertEquals(3, getNumCoveredWriteQuorums(ensemble1, 2)); + ArrayList ensemble2 = repp.newEnsemble(4, 2, 2, null, new HashSet()); + assertEquals(4, getNumCoveredWriteQuorums(ensemble2, 2)); + } catch (BKNotEnoughBookiesException bnebe) { + fail("Should not get not enough bookies exception."); + } + } + + /** + * Test for BOOKKEEPER-633 + */ + + @Test(timeout = 60000) + public void testRemoveBookieFromCluster() { + ignoreTestIfItIsWindowsOS(); + BookieSocketAddress addr1 = new BookieSocketAddress("127.0.0.1", 3181); // /1 rack + BookieSocketAddress addr2 = new BookieSocketAddress("127.0.0.2", 3181); // /2 rack + BookieSocketAddress addr3 = new BookieSocketAddress("127.0.1.2", 3181); // /2 rack + BookieSocketAddress addr4 = new BookieSocketAddress("127.0.0.4", 3181); // /4 rack + // Update cluster + Set addrs = new HashSet(); + addrs.add(addr1); + addrs.add(addr2); + addrs.add(addr3); + addrs.add(addr4); + repp.onClusterChanged(addrs, new HashSet()); + addrs.remove(addr1); + repp.onClusterChanged(addrs, new HashSet()); + } + + private int getNumCoveredWriteQuorums(ArrayList ensemble, int writeQuorumSize) + throws Exception { + int ensembleSize = ensemble.size(); + int numCoveredWriteQuorums = 0; + for (int i = 0; i < ensembleSize; i++) { + Set racks = new HashSet(); + for (int j = 0; j < writeQuorumSize; j++) { + int bookieIdx = (i + j) % ensembleSize; + BookieSocketAddress addr = ensemble.get(bookieIdx); + String hostAddress = addr.getSocketAddress().getAddress().getHostAddress(); + String rack = "/" + hostAddress.charAt(hostAddress.length() - 1); + racks.add(rack); + } + numCoveredWriteQuorums += (racks.size() > 1 ? 1 : 0); + } + return numCoveredWriteQuorums; + } + +} http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/4bb57ef0/bookkeeper-server/src/test/resources/networkmappingscript.sh ---------------------------------------------------------------------- diff --git a/bookkeeper-server/src/test/resources/networkmappingscript.sh b/bookkeeper-server/src/test/resources/networkmappingscript.sh new file mode 100755 index 0000000..136ee0b --- /dev/null +++ b/bookkeeper-server/src/test/resources/networkmappingscript.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# This script is used as NetworkTopology Mapping Script in TestRackawareEnsemblePlacementPolicyUsingScript.java TestSuite +# It just maps HostAddress to rack depending on the last character of the HostAddress string +# for eg. +# 127.0.0.1 - /1 +# 127.0.0.2 - /2 +# 199.12.34.21 - /1 +# This script file is used just for testing purpose + +for var in "$@" +do + i=$((${#var}-1)) + echo /${var:$i:1} +done \ No newline at end of file