cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jasobr...@apache.org
Subject cassandra git commit: Add nodetool getseeds and reloadseeds commands
Date Wed, 31 Jan 2018 00:33:07 GMT
Repository: cassandra
Updated Branches:
  refs/heads/trunk 69db2359e -> bfecdf520


Add nodetool getseeds and reloadseeds commands

patch by Samuel Fink; reviewed by jasobrown for CASSANDRA-14190


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/bfecdf52
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/bfecdf52
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/bfecdf52

Branch: refs/heads/trunk
Commit: bfecdf52054a4da472af22b0c35c5db5f1132bbc
Parents: 69db235
Author: Samuel Fink <fsamuel@netapp.com>
Authored: Wed Jan 24 13:28:54 2018 -0500
Committer: Jason Brown <jasedbrown@gmail.com>
Committed: Tue Jan 30 16:32:09 2018 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../cassandra/config/DatabaseDescriptor.java    |  10 ++
 src/java/org/apache/cassandra/gms/Gossiper.java |  83 +++++++++--
 .../org/apache/cassandra/gms/GossiperMBean.java |   5 +
 .../org/apache/cassandra/tools/NodeProbe.java   |  10 ++
 .../org/apache/cassandra/tools/NodeTool.java    |   2 +
 .../cassandra/tools/nodetool/GetSeeds.java      |  44 ++++++
 .../cassandra/tools/nodetool/ReloadSeeds.java   |  47 +++++++
 .../org/apache/cassandra/gms/GossiperTest.java  | 140 ++++++++++++++++++-
 9 files changed, 332 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index e28ffd9..a2e3654 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 4.0
+ * Non-disruptive seed node list reload (CASSANDRA-14190)
  * Nodetool tablehistograms to print statics for all the tables (CASSANDRA-14185)
  * Migrate dtests to use pytest and python3 (CASSANDRA-14134)
  * Allow storage port to be configurable per node (CASSANDRA-7544)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
index 9012e3a..8e831cf 100644
--- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
+++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
@@ -1661,6 +1661,16 @@ public class DatabaseDescriptor
         return ImmutableSet.<InetAddressAndPort>builder().addAll(seedProvider.getSeeds()).build();
     }
 
+    public static SeedProvider getSeedProvider()
+    {
+        return seedProvider;
+    }
+
+    public static void setSeedProvider(SeedProvider newSeedProvider)
+    {
+        seedProvider = newSeedProvider;
+    }
+
     public static InetAddress getListenAddress()
     {
         return listenAddress;

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/gms/Gossiper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/gms/Gossiper.java b/src/java/org/apache/cassandra/gms/Gossiper.java
index eb6c500..a4e46f2 100644
--- a/src/java/org/apache/cassandra/gms/Gossiper.java
+++ b/src/java/org/apache/cassandra/gms/Gossiper.java
@@ -35,6 +35,7 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.util.concurrent.Uninterruptibles;
 
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.SeedProvider;
 import org.apache.cassandra.utils.CassandraVersion;
 import org.apache.cassandra.utils.Pair;
 import org.slf4j.Logger;
@@ -89,7 +90,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
     public final static int intervalInMillis = 1000;
     public final static int QUARANTINE_DELAY = StorageService.RING_DELAY * 2;
     private static final Logger logger = LoggerFactory.getLogger(Gossiper.class);
-    public static final Gossiper instance = new Gossiper();
+    public static final Gossiper instance = new Gossiper(true);
 
     // Timestamp to prevent processing any in-flight messages for we've not send any SYN
yet, see CASSANDRA-12653.
     volatile long firstSynSendAt = 0L;
@@ -199,7 +200,7 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         }
     }
 
-    private Gossiper()
+    Gossiper(boolean registerJmx)
     {
         // half of QUARATINE_DELAY, to ensure justRemovedEndpoints has enough leeway to prevent
re-gossip
         fatClientTimeout = (QUARANTINE_DELAY / 2);
@@ -207,14 +208,17 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         FailureDetector.instance.registerFailureDetectionEventListener(this);
 
         // Register this instance with JMX
-        try
-        {
-            MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
-            mbs.registerMBean(this, new ObjectName(MBEAN_NAME));
-        }
-        catch (Exception e)
+        if (registerJmx)
         {
-            throw new RuntimeException(e);
+            try
+            {
+                MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
+                mbs.registerMBean(this, new ObjectName(MBEAN_NAME));
+            }
+            catch (Exception e)
+            {
+                throw new RuntimeException(e);
+            }
         }
     }
 
@@ -1468,6 +1472,67 @@ public class Gossiper implements IFailureDetectionEventListener, GossiperMBean
         }
     }
 
+    /**
+     * JMX interface for triggering an update of the seed node list.
+     */
+    public List<String> reloadSeeds()
+    {
+        logger.trace("Triggering reload of seed node list");
+
+        // Get the new set in the same that buildSeedsList does
+        Set<InetAddressAndPort> tmp = new HashSet<>();
+        try
+        {
+            for (InetAddressAndPort seed : DatabaseDescriptor.getSeeds())
+            {
+                if (seed.equals(FBUtilities.getBroadcastAddressAndPort()))
+                    continue;
+                tmp.add(seed);
+            }
+        }
+        // If using the SimpleSeedProvider invalid yaml added to the config since startup
could
+        // cause this to throw. Additionally, third party seed providers may throw exceptions.
+        // Handle the error and return a null to indicate that there was a problem.
+        catch (Throwable e)
+        {
+            JVMStabilityInspector.inspectThrowable(e);
+            logger.warn("Error while getting seed node list: {}", e.getLocalizedMessage());
+            return null;
+        }
+
+        if (tmp.size() == 0)
+        {
+            logger.trace("New seed node list is empty. Not updating seed list.");
+            return getSeeds();
+        }
+
+        if (tmp.equals(seeds))
+        {
+            logger.trace("New seed node list matches the existing list.");
+            return getSeeds();
+        }
+
+        // Add the new entries
+        seeds.addAll(tmp);
+        // Remove the old entries
+        seeds.retainAll(tmp);
+        logger.trace("New seed node list after reload {}", seeds);
+        return getSeeds();
+    }
+
+    /**
+     * JMX endpoint for getting the list of seeds from the node
+     */
+    public List<String> getSeeds()
+    {
+        List<String> seedList = new ArrayList<String>();
+        for (InetAddressAndPort seed : seeds)
+        {
+            seedList.add(seed.toString());
+        }
+        return seedList;
+    }
+
     // initialize local HB state if needed, i.e., if gossiper has never been started before.
     public void maybeInitializeLocalState(int generationNbr)
     {

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/gms/GossiperMBean.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/gms/GossiperMBean.java b/src/java/org/apache/cassandra/gms/GossiperMBean.java
index c4b244c..1b116e1 100644
--- a/src/java/org/apache/cassandra/gms/GossiperMBean.java
+++ b/src/java/org/apache/cassandra/gms/GossiperMBean.java
@@ -18,6 +18,7 @@
 package org.apache.cassandra.gms;
 
 import java.net.UnknownHostException;
+import java.util.List;
 
 public interface GossiperMBean
 {
@@ -29,4 +30,8 @@ public interface GossiperMBean
 
     public void assassinateEndpoint(String address) throws UnknownHostException;
 
+    public List<String> reloadSeeds();
+
+    public List<String> getSeeds();
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/tools/NodeProbe.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java
index d330ed4..ec8f7ba 100644
--- a/src/java/org/apache/cassandra/tools/NodeProbe.java
+++ b/src/java/org/apache/cassandra/tools/NodeProbe.java
@@ -711,6 +711,16 @@ public class NodeProbe implements AutoCloseable
         gossProxy.assassinateEndpoint(address);
     }
 
+    public List<String> reloadSeeds()
+    {
+        return gossProxy.reloadSeeds();
+    }
+
+    public List<String> getSeeds()
+    {
+        return gossProxy.getSeeds();
+    }
+
     /**
      * Set the compaction threshold
      *

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/tools/NodeTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/NodeTool.java b/src/java/org/apache/cassandra/tools/NodeTool.java
index d707499..f7b7f76 100644
--- a/src/java/org/apache/cassandra/tools/NodeTool.java
+++ b/src/java/org/apache/cassandra/tools/NodeTool.java
@@ -87,6 +87,7 @@ public class NodeTool
                 GetTraceProbability.class,
                 GetInterDCStreamThroughput.class,
                 GetEndpoints.class,
+                GetSeeds.class,
                 GetSSTables.class,
                 GetMaxHintWindow.class,
                 GossipInfo.class,
@@ -102,6 +103,7 @@ public class NodeTool
                 Refresh.class,
                 RemoveNode.class,
                 Assassinate.class,
+                ReloadSeeds.class,
                 ResetFullQueryLog.class,
                 Repair.class,
                 RepairAdmin.class,

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java b/src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java
new file mode 100644
index 0000000..207363c
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/GetSeeds.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import java.util.List;
+
+import io.airlift.airline.Command;
+
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "getseeds", description = "Get the currently in use seed node IP list excluding
the node IP")
+public class GetSeeds extends NodeToolCmd
+{
+    @Override
+    public void execute(NodeProbe probe)
+    {
+        List<String> seedList = probe.getSeeds();
+        if (seedList.isEmpty())
+        {
+            System.out.println("Seed node list does not contain any remote node IPs");
+        }
+        else
+        {
+            System.out.println("Current list of seed node IPs, excluding the current node's
IP: " + String.join(" ", seedList));
+        }
+
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java b/src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java
new file mode 100644
index 0000000..b9682cf
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/nodetool/ReloadSeeds.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools.nodetool;
+
+import java.util.List;
+
+import io.airlift.airline.Command;
+
+import org.apache.cassandra.tools.NodeProbe;
+import org.apache.cassandra.tools.NodeTool.NodeToolCmd;
+
+@Command(name = "reloadseeds", description = "Reload the seed node list from the seed node
provider")
+public class ReloadSeeds extends NodeToolCmd
+{
+    @Override
+    public void execute(NodeProbe probe)
+    {
+        List<String> seedList = probe.reloadSeeds();
+        if (seedList == null)
+        {
+            System.out.println("Failed to reload the seed node list.");
+        }
+        else if (seedList.isEmpty())
+        {
+            System.out.println("Seed node list does not contain any remote node IPs");
+        }
+        else
+        {
+            System.out.println("Updated seed node IP list, excluding the current node's IP:
" + String.join(" ", seedList));
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/bfecdf52/test/unit/org/apache/cassandra/gms/GossiperTest.java
----------------------------------------------------------------------
diff --git a/test/unit/org/apache/cassandra/gms/GossiperTest.java b/test/unit/org/apache/cassandra/gms/GossiperTest.java
index 8c65cb4..b856983 100644
--- a/test/unit/org/apache/cassandra/gms/GossiperTest.java
+++ b/test/unit/org/apache/cassandra/gms/GossiperTest.java
@@ -18,12 +18,16 @@
 
 package org.apache.cassandra.gms;
 
+import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.UUID;
 
 import com.google.common.collect.ImmutableMap;
+import com.google.common.net.InetAddresses;
+import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -33,6 +37,7 @@ import org.apache.cassandra.dht.IPartitioner;
 import org.apache.cassandra.dht.RandomPartitioner;
 import org.apache.cassandra.dht.Token;
 import org.apache.cassandra.locator.InetAddressAndPort;
+import org.apache.cassandra.locator.SeedProvider;
 import org.apache.cassandra.locator.TokenMetadata;
 import org.apache.cassandra.service.StorageService;
 
@@ -44,6 +49,7 @@ public class GossiperTest
     {
         DatabaseDescriptor.daemonInitialization();
     }
+
     static final IPartitioner partitioner = new RandomPartitioner();
     StorageService ss = StorageService.instance;
     TokenMetadata tmd = StorageService.instance.getTokenMetadata();
@@ -52,11 +58,20 @@ public class GossiperTest
     List<InetAddressAndPort> hosts = new ArrayList<>();
     List<UUID> hostIds = new ArrayList<>();
 
+    private SeedProvider originalSeedProvider;
+
     @Before
     public void setup()
     {
         tmd.clearUnsafe();
-    };
+        originalSeedProvider = DatabaseDescriptor.getSeedProvider();
+    }
+
+    @After
+    public void tearDown()
+    {
+        DatabaseDescriptor.setSeedProvider(originalSeedProvider);
+    }
 
     @Test
     public void testLargeGenerationJump() throws UnknownHostException, InterruptedException
@@ -90,4 +105,127 @@ public class GossiperTest
         //The generation should not have been updated because it is over Gossiper.MAX_GENERATION_DIFFERENCE
in the future
         assertEquals(proposedRemoteHeartBeat.getGeneration(), actualRemoteHeartBeat.getGeneration());
     }
+
+    // Note: This test might fail if for some reason the node broadcast address is in 127.99.0.0/16
+    @Test
+    public void testReloadSeeds() throws UnknownHostException
+    {
+        Gossiper gossiper = new Gossiper(false);
+        List<String> loadedList;
+
+        // Initialize the seed list directly to a known set to start with
+        gossiper.seeds.clear();
+        InetAddressAndPort addr = InetAddressAndPort.getByAddress(InetAddress.getByName("127.99.1.1"));
+        int nextSize = 4;
+        List<InetAddressAndPort> nextSeeds = new ArrayList<>(nextSize);
+        for (int i = 0; i < nextSize; i ++)
+        {
+            gossiper.seeds.add(addr);
+            nextSeeds.add(addr);
+            addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address));
+        }
+        Assert.assertEquals(nextSize, gossiper.seeds.size());
+
+        // Add another unique address to the list
+        addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address));
+        nextSeeds.add(addr);
+        nextSize++;
+        DatabaseDescriptor.setSeedProvider(new TestSeedProvider(nextSeeds));
+        loadedList = gossiper.reloadSeeds();
+
+        // Check that the new entry was added
+        Assert.assertEquals(nextSize, loadedList.size());
+        for (InetAddressAndPort a : nextSeeds)
+            Assert.assertTrue(loadedList.contains(a.toString()));
+
+        // Check that the return value of the reloadSeeds matches the content of the getSeeds
call
+        // and that they both match the internal contents of the Gossiper seeds list
+        Assert.assertEquals(loadedList.size(), gossiper.getSeeds().size());
+        for (InetAddressAndPort a : gossiper.seeds)
+        {
+            Assert.assertTrue(loadedList.contains(a.toString()));
+            Assert.assertTrue(gossiper.getSeeds().contains(a.toString()));
+        }
+
+        // Add a duplicate of the last address to the seed provider list
+        int uniqueSize = nextSize;
+        nextSeeds.add(addr);
+        nextSize++;
+        DatabaseDescriptor.setSeedProvider(new TestSeedProvider(nextSeeds));
+        loadedList = gossiper.reloadSeeds();
+
+        // Check that the number of seed nodes reported hasn't increased
+        Assert.assertEquals(uniqueSize, loadedList.size());
+        for (InetAddressAndPort a : nextSeeds)
+            Assert.assertTrue(loadedList.contains(a.toString()));
+
+        // Create a new list that has no overlaps with the previous list
+        addr = InetAddressAndPort.getByAddress(InetAddress.getByName("127.99.2.1"));
+        int disjointSize = 3;
+        List<InetAddressAndPort> disjointSeeds = new ArrayList<>(disjointSize);
+        for (int i = 0; i < disjointSize; i ++)
+        {
+            disjointSeeds.add(addr);
+            addr = InetAddressAndPort.getByAddress(InetAddresses.increment(addr.address));
+        }
+        DatabaseDescriptor.setSeedProvider(new TestSeedProvider(disjointSeeds));
+        loadedList = gossiper.reloadSeeds();
+
+        // Check that the list now contains exactly the new other list.
+        Assert.assertEquals(disjointSize, gossiper.getSeeds().size());
+        Assert.assertEquals(disjointSize, loadedList.size());
+        for (InetAddressAndPort a : disjointSeeds)
+        {
+            Assert.assertTrue(gossiper.getSeeds().contains(a.toString()));
+            Assert.assertTrue(loadedList.contains(a.toString()));
+        }
+
+        // Set the seed node provider to return an empty list
+        DatabaseDescriptor.setSeedProvider(new TestSeedProvider(new ArrayList<InetAddressAndPort>()));
+        loadedList = gossiper.reloadSeeds();
+
+        // Check that the in memory seed node list was not modified
+        Assert.assertEquals(disjointSize, loadedList.size());
+        for (InetAddressAndPort a : disjointSeeds)
+            Assert.assertTrue(loadedList.contains(a.toString()));
+
+        // Change the seed provider to one that throws an unchecked exception
+        DatabaseDescriptor.setSeedProvider(new ErrorSeedProvider());
+        loadedList = gossiper.reloadSeeds();
+
+        // Check for the expected null response from a reload error
+        Assert.assertNull(loadedList);
+
+        // Check that the in memory seed node list was not modified and the exception was
caught
+        Assert.assertEquals(disjointSize, gossiper.getSeeds().size());
+        for (InetAddressAndPort a : disjointSeeds)
+            Assert.assertTrue(gossiper.getSeeds().contains(a.toString()));
+    }
+
+    static class TestSeedProvider implements SeedProvider
+    {
+        private List<InetAddressAndPort> seeds;
+
+        TestSeedProvider(List<InetAddressAndPort> seeds)
+        {
+            this.seeds = seeds;
+        }
+
+        @Override
+        public List<InetAddressAndPort> getSeeds()
+        {
+            return seeds;
+        }
+    }
+
+    // A seed provider for testing which throws assertion errors when queried
+    static class ErrorSeedProvider implements SeedProvider
+    {
+        @Override
+        public List<InetAddressAndPort> getSeeds()
+        {
+            assert(false);
+            return new ArrayList<>();
+        }
+    }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@cassandra.apache.org
For additional commands, e-mail: commits-help@cassandra.apache.org


Mime
View raw message