Return-Path: Delivered-To: apmail-incubator-cassandra-commits-archive@minotaur.apache.org Received: (qmail 97140 invoked from network); 21 Dec 2009 17:21:17 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 21 Dec 2009 17:21:17 -0000 Received: (qmail 63787 invoked by uid 500); 21 Dec 2009 17:21:17 -0000 Delivered-To: apmail-incubator-cassandra-commits-archive@incubator.apache.org Received: (qmail 63773 invoked by uid 500); 21 Dec 2009 17:21:17 -0000 Mailing-List: contact cassandra-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: cassandra-dev@incubator.apache.org Delivered-To: mailing list cassandra-commits@incubator.apache.org Received: (qmail 63763 invoked by uid 99); 21 Dec 2009 17:21:17 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 21 Dec 2009 17:21:17 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=10.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 21 Dec 2009 17:21:13 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 31D1C23889B9; Mon, 21 Dec 2009 17:20:51 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r892889 - in /incubator/cassandra/branches/cassandra-0.5: CHANGES.txt src/java/org/apache/cassandra/service/StorageService.java test/unit/org/apache/cassandra/service/MoveTest.java Date: Mon, 21 Dec 2009 17:20:51 -0000 To: cassandra-commits@incubator.apache.org From: jbellis@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20091221172051.31D1C23889B9@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: jbellis Date: Mon Dec 21 17:20:50 2009 New Revision: 892889 URL: http://svn.apache.org/viewvc?rev=892889&view=rev Log: add unit tests for node movement. patch by Jaakko Laine; reviewed by jbellis for CASSANDRA-572 Added: incubator/cassandra/branches/cassandra-0.5/test/unit/org/apache/cassandra/service/MoveTest.java (with props) Modified: incubator/cassandra/branches/cassandra-0.5/CHANGES.txt incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/service/StorageService.java Modified: incubator/cassandra/branches/cassandra-0.5/CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.5/CHANGES.txt?rev=892889&r1=892888&r2=892889&view=diff ============================================================================== --- incubator/cassandra/branches/cassandra-0.5/CHANGES.txt (original) +++ incubator/cassandra/branches/cassandra-0.5/CHANGES.txt Mon Dec 21 17:20:50 2009 @@ -7,6 +7,8 @@ * Fix anti-entropy assertion error (CASSANDRA-639) * Fix pending range conflicts when bootstapping or moving multiple nodes at once (CASSANDRA-603) + * Handle obsolete gossip related to node movement in the case where + one or more nodes is down when the movement occurs (CASSANDRA-572) * Include dead nodes in gossip to avoid a variety of problems (CASSANDRA-634) Modified: incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/service/StorageService.java URL: http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/service/StorageService.java?rev=892889&r1=892888&r2=892889&view=diff ============================================================================== --- incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/service/StorageService.java (original) +++ incubator/cassandra/branches/cassandra-0.5/src/java/org/apache/cassandra/service/StorageService.java Mon Dec 21 17:20:50 2009 @@ -1448,4 +1448,21 @@ { return isClientMode; } + + // Never ever do this at home. Used by tests. + AbstractReplicationStrategy setReplicationStrategyUnsafe(AbstractReplicationStrategy newStrategy) + { + AbstractReplicationStrategy oldStrategy = replicationStrategy_; + replicationStrategy_ = newStrategy; + return oldStrategy; + } + + // Never ever do this at home. Used by tests. + IPartitioner setPartitionerUnsafe(IPartitioner newPartitioner) + { + IPartitioner oldPartitioner = partitioner_; + partitioner_ = newPartitioner; + return oldPartitioner; + } + } Added: incubator/cassandra/branches/cassandra-0.5/test/unit/org/apache/cassandra/service/MoveTest.java URL: http://svn.apache.org/viewvc/incubator/cassandra/branches/cassandra-0.5/test/unit/org/apache/cassandra/service/MoveTest.java?rev=892889&view=auto ============================================================================== --- incubator/cassandra/branches/cassandra-0.5/test/unit/org/apache/cassandra/service/MoveTest.java (added) +++ incubator/cassandra/branches/cassandra-0.5/test/unit/org/apache/cassandra/service/MoveTest.java Mon Dec 21 17:20:50 2009 @@ -0,0 +1,516 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* KIND, either express or implied. See the License for the +* specific language governing permissions and limitations +* under the License. +*/ + +package org.apache.cassandra.service; + +import java.util.*; + +import java.net.InetAddress; +import java.net.UnknownHostException; + +import org.junit.Test; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.RandomPartitioner; +import org.apache.cassandra.dht.Token; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.dht.BigIntegerToken; +import org.apache.cassandra.locator.AbstractReplicationStrategy; +import org.apache.cassandra.locator.RackUnawareStrategy; +import org.apache.cassandra.locator.TokenMetadata; +import org.apache.cassandra.gms.ApplicationState; + +public class MoveTest +{ + /** + * Test whether write endpoints is correct when the node is leaving. Uses + * StorageService.onChange and does not manipulate token metadata directly. + */ + @Test + public void testWriteEndPointsDuringLeave() throws UnknownHostException + { + StorageService ss = StorageService.instance(); + + TokenMetadata tmd = ss.getTokenMetadata(); + tmd.clearUnsafe(); + IPartitioner partitioner = new RandomPartitioner(); + AbstractReplicationStrategy testStrategy = new RackUnawareStrategy(tmd, partitioner, 3); + + IPartitioner oldPartitioner = ss.setPartitionerUnsafe(partitioner); + AbstractReplicationStrategy oldStrategy = ss.setReplicationStrategyUnsafe(testStrategy); + + ArrayList endPointTokens = new ArrayList(); + ArrayList keyTokens = new ArrayList(); + List hosts = new ArrayList(); + + createInitialRing(ss, partitioner, endPointTokens, keyTokens, hosts, 5); + + // Third node leaves + ss.onChange(hosts.get(2), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_LEAVING + StorageService.Delimiter + partitioner.getTokenFactory().toString(endPointTokens.get(2)))); + + // check that it is correctly marked as leaving in tmd + assertTrue(tmd.isLeaving(hosts.get(2))); + + // check that pending ranges are correct (primary range should go to 1st node, first + // replica range to 4th node and 2nd replica range to 5th node) + assertTrue(tmd.getPendingRanges(hosts.get(0)).get(0).equals(new Range(endPointTokens.get(1), + endPointTokens.get(2)))); + assertTrue(tmd.getPendingRanges(hosts.get(3)).get(0).equals(new Range(endPointTokens.get(4), + endPointTokens.get(0)))); + assertTrue(tmd.getPendingRanges(hosts.get(4)).get(0).equals(new Range(endPointTokens.get(0), + endPointTokens.get(1)))); + + for (int i=0; i endPoints = testStrategy.getWriteEndpoints(keyTokens.get(i), testStrategy.getNaturalEndpoints(keyTokens.get(i))); + + // Original third node does not store replicas for 4th and 5th node (ranges 20-30 + // and 30-40 respectively), so their write endpoints count should be still 3. The + // third node stores data for ranges 40-0, 0-10 and 10-20, so writes falling to + // these ranges should have four endpoints now. keyTokens[2] is 25 and keyTokens[3] + // is 35, so these are the ones that should have 3 endpoints. + if (i==2 || i==3) + assertTrue(endPoints.size() == 3); + else + assertTrue(endPoints.size() == 4); + } + + ss.setPartitionerUnsafe(oldPartitioner); + ss.setReplicationStrategyUnsafe(oldStrategy); + } + + /** + * Test pending ranges and write endpoints when multiple nodes are on the move + * simultaneously + */ + @Test + public void testSimultaneousMove() throws UnknownHostException + { + StorageService ss = StorageService.instance(); + TokenMetadata tmd = ss.getTokenMetadata(); + tmd.clearUnsafe(); + IPartitioner partitioner = new RandomPartitioner(); + AbstractReplicationStrategy testStrategy = new RackUnawareStrategy(tmd, partitioner, 3); + + IPartitioner oldPartitioner = ss.setPartitionerUnsafe(partitioner); + AbstractReplicationStrategy oldStrategy = ss.setReplicationStrategyUnsafe(testStrategy); + + ArrayList endPointTokens = new ArrayList(); + ArrayList keyTokens = new ArrayList(); + List hosts = new ArrayList(); + + // create a ring or 10 nodes + createInitialRing(ss, partitioner, endPointTokens, keyTokens, hosts, 10); + + // nodes 6, 8 and 9 leave + ss.onChange(hosts.get(6), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_LEAVING + StorageService.Delimiter + partitioner.getTokenFactory().toString(endPointTokens.get(6)))); + ss.onChange(hosts.get(8), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_LEAVING + StorageService.Delimiter + partitioner.getTokenFactory().toString(endPointTokens.get(8)))); + ss.onChange(hosts.get(9), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_LEAVING + StorageService.Delimiter + partitioner.getTokenFactory().toString(endPointTokens.get(9)))); + + // boot two new nodes with keyTokens.get(5) and keyTokens.get(7) + InetAddress boot1 = InetAddress.getByName("127.0.1.1"); + ss.onChange(boot1, StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_BOOTSTRAPPING + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(5)))); + InetAddress boot2 = InetAddress.getByName("127.0.1.2"); + ss.onChange(boot2, StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_BOOTSTRAPPING + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(7)))); + + Collection endPoints = null; + + // tokens 5, 15 and 25 should go three nodes + for (int i=0; i<3; ++i) + { + endPoints = testStrategy.getWriteEndpoints(keyTokens.get(i), testStrategy.getNaturalEndpoints(keyTokens.get(i))); + assertTrue(endPoints.size() == 3); + assertTrue(endPoints.contains(hosts.get(i+1))); + assertTrue(endPoints.contains(hosts.get(i+2))); + assertTrue(endPoints.contains(hosts.get(i+3))); + } + + // token 35 should go to nodes 4, 5, 6, 7 and boot1 + endPoints = testStrategy.getWriteEndpoints(keyTokens.get(3), testStrategy.getNaturalEndpoints(keyTokens.get(3))); + assertTrue(endPoints.size() == 5); + assertTrue(endPoints.contains(hosts.get(4))); + assertTrue(endPoints.contains(hosts.get(5))); + assertTrue(endPoints.contains(hosts.get(6))); + assertTrue(endPoints.contains(hosts.get(7))); + assertTrue(endPoints.contains(boot1)); + + // token 45 should go to nodes 5, 6, 7, 0, boot1 and boot2 + endPoints = testStrategy.getWriteEndpoints(keyTokens.get(4), testStrategy.getNaturalEndpoints(keyTokens.get(4))); + assertTrue(endPoints.size() == 6); + assertTrue(endPoints.contains(hosts.get(5))); + assertTrue(endPoints.contains(hosts.get(6))); + assertTrue(endPoints.contains(hosts.get(7))); + assertTrue(endPoints.contains(hosts.get(0))); + assertTrue(endPoints.contains(boot1)); + assertTrue(endPoints.contains(boot2)); + + // token 55 should go to nodes 6, 7, 8, 0, 1, boot1 and boot2 + endPoints = testStrategy.getWriteEndpoints(keyTokens.get(5), testStrategy.getNaturalEndpoints(keyTokens.get(5))); + assertTrue(endPoints.size() == 7); + assertTrue(endPoints.contains(hosts.get(6))); + assertTrue(endPoints.contains(hosts.get(7))); + assertTrue(endPoints.contains(hosts.get(8))); + assertTrue(endPoints.contains(hosts.get(0))); + assertTrue(endPoints.contains(hosts.get(1))); + assertTrue(endPoints.contains(boot1)); + assertTrue(endPoints.contains(boot2)); + + // token 65 should go to nodes 7, 8, 9, 0, 1 and boot2 + endPoints = testStrategy.getWriteEndpoints(keyTokens.get(6), testStrategy.getNaturalEndpoints(keyTokens.get(6))); + assertTrue(endPoints.size() == 6); + assertTrue(endPoints.contains(hosts.get(7))); + assertTrue(endPoints.contains(hosts.get(8))); + assertTrue(endPoints.contains(hosts.get(9))); + assertTrue(endPoints.contains(hosts.get(0))); + assertTrue(endPoints.contains(hosts.get(1))); + assertTrue(endPoints.contains(boot2)); + + // token 75 should to go nodes 8, 9, 0, 1, 2 and boot2 + endPoints = testStrategy.getWriteEndpoints(keyTokens.get(7), testStrategy.getNaturalEndpoints(keyTokens.get(7))); + assertTrue(endPoints.size() == 6); + assertTrue(endPoints.contains(hosts.get(8))); + assertTrue(endPoints.contains(hosts.get(9))); + assertTrue(endPoints.contains(hosts.get(0))); + assertTrue(endPoints.contains(hosts.get(1))); + assertTrue(endPoints.contains(hosts.get(2))); + assertTrue(endPoints.contains(boot2)); + + // token 85 should go to nodes 9, 0, 1 and 2 + endPoints = testStrategy.getWriteEndpoints(keyTokens.get(8), testStrategy.getNaturalEndpoints(keyTokens.get(8))); + assertTrue(endPoints.size() == 4); + assertTrue(endPoints.contains(hosts.get(9))); + assertTrue(endPoints.contains(hosts.get(0))); + assertTrue(endPoints.contains(hosts.get(1))); + assertTrue(endPoints.contains(hosts.get(2))); + + // token 95 should go to nodes 0, 1 and 2 + endPoints = testStrategy.getWriteEndpoints(keyTokens.get(9), testStrategy.getNaturalEndpoints(keyTokens.get(9))); + assertTrue(endPoints.size() == 3); + assertTrue(endPoints.contains(hosts.get(0))); + assertTrue(endPoints.contains(hosts.get(1))); + assertTrue(endPoints.contains(hosts.get(2))); + + // Now finish node 6 and node 9 leaving, as well as boot1 (after this node 8 is still + // leaving and boot2 in progress + ss.onChange(hosts.get(6), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_LEFT + StorageService.Delimiter + StorageService.LEFT_NORMALLY + StorageService.Delimiter + partitioner.getTokenFactory().toString(endPointTokens.get(6)))); + ss.onChange(hosts.get(9), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_LEFT + StorageService.Delimiter + StorageService.LEFT_NORMALLY + StorageService.Delimiter + partitioner.getTokenFactory().toString(endPointTokens.get(9)))); + ss.onChange(boot1, StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_NORMAL + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(5)))); + + // tokens 5, 15 and 25 should go three nodes + for (int i=0; i<3; ++i) + { + endPoints = testStrategy.getWriteEndpoints(keyTokens.get(i), testStrategy.getNaturalEndpoints(keyTokens.get(i))); + assertTrue(endPoints.size() == 3); + assertTrue(endPoints.contains(hosts.get(i+1))); + assertTrue(endPoints.contains(hosts.get(i+2))); + assertTrue(endPoints.contains(hosts.get(i+3))); + } + + // token 35 goes to nodes 4, 5 and boot1 + endPoints = testStrategy.getWriteEndpoints(keyTokens.get(3), testStrategy.getNaturalEndpoints(keyTokens.get(3))); + assertTrue(endPoints.size() == 3); + assertTrue(endPoints.contains(hosts.get(4))); + assertTrue(endPoints.contains(hosts.get(5))); + assertTrue(endPoints.contains(boot1)); + + // token 45 goes to nodes 5, boot1 and node7 + endPoints = testStrategy.getWriteEndpoints(keyTokens.get(4), testStrategy.getNaturalEndpoints(keyTokens.get(4))); + assertTrue(endPoints.size() == 3); + assertTrue(endPoints.contains(hosts.get(5))); + assertTrue(endPoints.contains(boot1)); + assertTrue(endPoints.contains(hosts.get(7))); + + // token 55 goes to boot1, 7, boot2, 8 and 0 + endPoints = testStrategy.getWriteEndpoints(keyTokens.get(5), testStrategy.getNaturalEndpoints(keyTokens.get(5))); + assertTrue(endPoints.size() == 5); + assertTrue(endPoints.contains(boot1)); + assertTrue(endPoints.contains(hosts.get(7))); + assertTrue(endPoints.contains(boot2)); + assertTrue(endPoints.contains(hosts.get(8))); + assertTrue(endPoints.contains(hosts.get(0))); + + // token 65 goes to nodes 7, boot2, 8, 0 and 1 + endPoints = testStrategy.getWriteEndpoints(keyTokens.get(6), testStrategy.getNaturalEndpoints(keyTokens.get(6))); + assertTrue(endPoints.size() == 5); + assertTrue(endPoints.contains(hosts.get(7))); + assertTrue(endPoints.contains(boot2)); + assertTrue(endPoints.contains(hosts.get(8))); + assertTrue(endPoints.contains(hosts.get(0))); + assertTrue(endPoints.contains(hosts.get(1))); + + // token 75 goes to nodes boot2, 8, 0, 1 and 2 + endPoints = testStrategy.getWriteEndpoints(keyTokens.get(7), testStrategy.getNaturalEndpoints(keyTokens.get(7))); + assertTrue(endPoints.size() == 5); + assertTrue(endPoints.contains(boot2)); + assertTrue(endPoints.contains(hosts.get(8))); + assertTrue(endPoints.contains(hosts.get(0))); + assertTrue(endPoints.contains(hosts.get(1))); + assertTrue(endPoints.contains(hosts.get(2))); + + // token 85 goes to nodes 0, 1 and 2 + endPoints = testStrategy.getWriteEndpoints(keyTokens.get(8), testStrategy.getNaturalEndpoints(keyTokens.get(8))); + assertTrue(endPoints.size() == 3); + assertTrue(endPoints.contains(hosts.get(0))); + assertTrue(endPoints.contains(hosts.get(1))); + assertTrue(endPoints.contains(hosts.get(2))); + + // token 95 goes to nodes 0, 1 and 2 + endPoints = testStrategy.getWriteEndpoints(keyTokens.get(9), testStrategy.getNaturalEndpoints(keyTokens.get(9))); + assertTrue(endPoints.size() == 3); + assertTrue(endPoints.contains(hosts.get(0))); + assertTrue(endPoints.contains(hosts.get(1))); + assertTrue(endPoints.contains(hosts.get(2))); + + ss.setPartitionerUnsafe(oldPartitioner); + ss.setReplicationStrategyUnsafe(oldStrategy); + } + + @Test + public void testStateJumpToBootstrap() throws UnknownHostException + { + StorageService ss = StorageService.instance(); + TokenMetadata tmd = ss.getTokenMetadata(); + tmd.clearUnsafe(); + IPartitioner partitioner = new RandomPartitioner(); + AbstractReplicationStrategy testStrategy = new RackUnawareStrategy(tmd, partitioner, 3); + + IPartitioner oldPartitioner = ss.setPartitionerUnsafe(partitioner); + AbstractReplicationStrategy oldStrategy = ss.setReplicationStrategyUnsafe(testStrategy); + + ArrayList endPointTokens = new ArrayList(); + ArrayList keyTokens = new ArrayList(); + List hosts = new ArrayList(); + + // create a ring or 5 nodes + createInitialRing(ss, partitioner, endPointTokens, keyTokens, hosts, 5); + + // node 2 leaves + ss.onChange(hosts.get(2), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_LEAVING + StorageService.Delimiter + partitioner.getTokenFactory().toString(endPointTokens.get(2)))); + + // don't bother to test pending ranges here, that is extensively tested by other + // tests. Just check that the node is in appropriate lists. + assertTrue(tmd.isMember(hosts.get(2))); + assertTrue(tmd.isLeaving(hosts.get(2))); + assertTrue(tmd.getBootstrapTokens().isEmpty()); + + // Bootstrap the node immedidiately to keyTokens.get(4) without going through STATE_LEFT + ss.onChange(hosts.get(2), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_BOOTSTRAPPING + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(4)))); + + assertFalse(tmd.isMember(hosts.get(2))); + assertFalse(tmd.isLeaving(hosts.get(2))); + assertTrue(tmd.getBootstrapTokens().get(keyTokens.get(4)).equals(hosts.get(2))); + + // Bootstrap node hosts.get(3) to keyTokens.get(1) + ss.onChange(hosts.get(3), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_BOOTSTRAPPING + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(1)))); + + assertFalse(tmd.isMember(hosts.get(3))); + assertFalse(tmd.isLeaving(hosts.get(3))); + assertTrue(tmd.getBootstrapTokens().get(keyTokens.get(4)).equals(hosts.get(2))); + assertTrue(tmd.getBootstrapTokens().get(keyTokens.get(1)).equals(hosts.get(3))); + + // Bootstrap node hosts.get(2) further to keyTokens.get(3) + ss.onChange(hosts.get(2), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_BOOTSTRAPPING + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(3)))); + + assertFalse(tmd.isMember(hosts.get(2))); + assertFalse(tmd.isLeaving(hosts.get(2))); + assertTrue(tmd.getBootstrapTokens().get(keyTokens.get(3)).equals(hosts.get(2))); + assertTrue(tmd.getBootstrapTokens().get(keyTokens.get(4)) == null); + assertTrue(tmd.getBootstrapTokens().get(keyTokens.get(1)).equals(hosts.get(3))); + + // Go to normal again for both nodes + ss.onChange(hosts.get(2), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_NORMAL + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(3)))); + ss.onChange(hosts.get(3), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_NORMAL + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(2)))); + + assertTrue(tmd.isMember(hosts.get(2))); + assertFalse(tmd.isLeaving(hosts.get(2))); + assertTrue(tmd.getToken(hosts.get(2)).equals(keyTokens.get(3))); + assertTrue(tmd.isMember(hosts.get(3))); + assertFalse(tmd.isLeaving(hosts.get(3))); + assertTrue(tmd.getToken(hosts.get(3)).equals(keyTokens.get(2))); + + assertTrue(tmd.getBootstrapTokens().isEmpty()); + + ss.setPartitionerUnsafe(oldPartitioner); + ss.setReplicationStrategyUnsafe(oldStrategy); + } + + @Test + public void testStateJumpToNormal() throws UnknownHostException + { + StorageService ss = StorageService.instance(); + TokenMetadata tmd = ss.getTokenMetadata(); + tmd.clearUnsafe(); + IPartitioner partitioner = new RandomPartitioner(); + AbstractReplicationStrategy testStrategy = new RackUnawareStrategy(tmd, partitioner, 3); + + IPartitioner oldPartitioner = ss.setPartitionerUnsafe(partitioner); + AbstractReplicationStrategy oldStrategy = ss.setReplicationStrategyUnsafe(testStrategy); + + ArrayList endPointTokens = new ArrayList(); + ArrayList keyTokens = new ArrayList(); + List hosts = new ArrayList(); + + // create a ring or 5 nodes + createInitialRing(ss, partitioner, endPointTokens, keyTokens, hosts, 5); + + // node 2 leaves + ss.onChange(hosts.get(2), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_LEAVING + StorageService.Delimiter + partitioner.getTokenFactory().toString(endPointTokens.get(2)))); + + assertTrue(tmd.isLeaving(hosts.get(2))); + assertTrue(tmd.getToken(hosts.get(2)).equals(endPointTokens.get(2))); + + // back to normal + ss.onChange(hosts.get(2), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_NORMAL + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(2)))); + + assertTrue(tmd.getLeavingEndPoints().isEmpty()); + assertTrue(tmd.getToken(hosts.get(2)).equals(keyTokens.get(2))); + + // node 3 goes through leave and left and then jumps to normal + ss.onChange(hosts.get(2), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_LEAVING + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(2)))); + ss.onChange(hosts.get(2), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_LEFT + StorageService.Delimiter + StorageService.LEFT_NORMALLY + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(2)))); + ss.onChange(hosts.get(2), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_NORMAL + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(4)))); + + assertTrue(tmd.getBootstrapTokens().isEmpty()); + assertTrue(tmd.getLeavingEndPoints().isEmpty()); + assertTrue(tmd.getToken(hosts.get(2)).equals(keyTokens.get(4))); + + ss.setPartitionerUnsafe(oldPartitioner); + ss.setReplicationStrategyUnsafe(oldStrategy); + } + + @Test + public void testStateJumpToLeaving() throws UnknownHostException + { + StorageService ss = StorageService.instance(); + TokenMetadata tmd = ss.getTokenMetadata(); + tmd.clearUnsafe(); + IPartitioner partitioner = new RandomPartitioner(); + AbstractReplicationStrategy testStrategy = new RackUnawareStrategy(tmd, partitioner, 3); + + IPartitioner oldPartitioner = ss.setPartitionerUnsafe(partitioner); + AbstractReplicationStrategy oldStrategy = ss.setReplicationStrategyUnsafe(testStrategy); + + ArrayList endPointTokens = new ArrayList(); + ArrayList keyTokens = new ArrayList(); + List hosts = new ArrayList(); + + // create a ring or 5 nodes + createInitialRing(ss, partitioner, endPointTokens, keyTokens, hosts, 5); + + // node 2 leaves with _different_ token + ss.onChange(hosts.get(2), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_LEAVING + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(0)))); + + assertTrue(tmd.getToken(hosts.get(2)).equals(keyTokens.get(0))); + assertTrue(tmd.isLeaving(hosts.get(2))); + assertTrue(tmd.getEndPoint(endPointTokens.get(2)) == null); + + // go to boostrap + ss.onChange(hosts.get(2), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_BOOTSTRAPPING + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(1)))); + + assertFalse(tmd.isLeaving(hosts.get(2))); + assertTrue(tmd.getBootstrapTokens().size() == 1); + assertTrue(tmd.getBootstrapTokens().get(keyTokens.get(1)).equals(hosts.get(2))); + + // jump to leaving again + ss.onChange(hosts.get(2), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_LEAVING + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(1)))); + + assertTrue(tmd.getEndPoint(keyTokens.get(1)).equals(hosts.get(2))); + assertTrue(tmd.isLeaving(hosts.get(2))); + assertTrue(tmd.getBootstrapTokens().isEmpty()); + + // go to state left + ss.onChange(hosts.get(2), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_LEFT + StorageService.Delimiter + StorageService.LEFT_NORMALLY + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(1)))); + + assertFalse(tmd.isMember(hosts.get(2))); + assertFalse(tmd.isLeaving(hosts.get(2))); + + ss.setPartitionerUnsafe(oldPartitioner); + ss.setReplicationStrategyUnsafe(oldStrategy); + } + + @Test + public void testStateJumpToLeft() throws UnknownHostException + { + StorageService ss = StorageService.instance(); + TokenMetadata tmd = ss.getTokenMetadata(); + tmd.clearUnsafe(); + IPartitioner partitioner = new RandomPartitioner(); + AbstractReplicationStrategy testStrategy = new RackUnawareStrategy(tmd, partitioner, 3); + + IPartitioner oldPartitioner = ss.setPartitionerUnsafe(partitioner); + AbstractReplicationStrategy oldStrategy = ss.setReplicationStrategyUnsafe(testStrategy); + + ArrayList endPointTokens = new ArrayList(); + ArrayList keyTokens = new ArrayList(); + List hosts = new ArrayList(); + + // create a ring or 5 nodes + createInitialRing(ss, partitioner, endPointTokens, keyTokens, hosts, 5); + + // node hosts.get(2) goes jumps to left + ss.onChange(hosts.get(2), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_LEFT + StorageService.Delimiter + StorageService.LEFT_NORMALLY + StorageService.Delimiter + partitioner.getTokenFactory().toString(endPointTokens.get(2)))); + + assertFalse(tmd.isMember(hosts.get(2))); + + // node hosts.get(4) goes to bootstrap + ss.onChange(hosts.get(3), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_BOOTSTRAPPING + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(1)))); + + assertFalse(tmd.isMember(hosts.get(3))); + assertTrue(tmd.getBootstrapTokens().size() == 1); + assertTrue(tmd.getBootstrapTokens().get(keyTokens.get(1)).equals(hosts.get(3))); + + // and then directly to 'left' + ss.onChange(hosts.get(2), StorageService.MOVE_STATE, new ApplicationState(StorageService.STATE_LEFT + StorageService.Delimiter + StorageService.LEFT_NORMALLY + StorageService.Delimiter + partitioner.getTokenFactory().toString(keyTokens.get(1)))); + + assertTrue(tmd.getBootstrapTokens().size() == 0); + assertFalse(tmd.isMember(hosts.get(2))); + assertFalse(tmd.isLeaving(hosts.get(2))); + + ss.setPartitionerUnsafe(oldPartitioner); + ss.setReplicationStrategyUnsafe(oldStrategy); + } + + /** + * Creates initial set of nodes and tokens. Nodes are added to StorageService as 'normal' + */ + private void createInitialRing(StorageService ss, IPartitioner partitioner, List endPointTokens, + List keyTokens, List hosts, int howMany) + throws UnknownHostException + { + for (int i=0; i