cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alek...@apache.org
Subject git commit: Fix merging schemas with re-dropped keyspaces
Date Fri, 10 Oct 2014 14:48:50 GMT
Repository: cassandra
Updated Branches:
  refs/heads/cassandra-2.0 b7adf9810 -> 0393c3063


Fix merging schemas with re-dropped keyspaces

patch by Aleksey Yeschenko; reviewed by Pavel Yaskevich for
CASSANDRA-7256


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0393c306
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0393c306
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0393c306

Branch: refs/heads/cassandra-2.0
Commit: 0393c306331e6b46182b0693ff192be13d20698e
Parents: b7adf98
Author: Aleksey Yeschenko <aleksey@apache.org>
Authored: Fri Oct 10 17:32:58 2014 +0300
Committer: Aleksey Yeschenko <aleksey@apache.org>
Committed: Fri Oct 10 17:32:58 2014 +0300

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../org/apache/cassandra/db/DefsTables.java     | 186 ++++++++-----------
 2 files changed, 78 insertions(+), 109 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/0393c306/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index bc12402..b633e48 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.0.11:
+ * Fix merging schemas with re-dropped keyspaces (CASSANDRA-7256)
  * Fix counters in supercolumns during live upgrades from 1.2 (CASSANDRA-7188)
  * Notify DT subscribers when a column family is truncated (CASSANDRA-8088)
  * Add sanity check of $JAVA on startup (CASSANDRA-7676)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/0393c306/src/java/org/apache/cassandra/db/DefsTables.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/db/DefsTables.java b/src/java/org/apache/cassandra/db/DefsTables.java
index bc2b36d..35eecc0 100644
--- a/src/java/org/apache/cassandra/db/DefsTables.java
+++ b/src/java/org/apache/cassandra/db/DefsTables.java
@@ -21,6 +21,7 @@ import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.*;
 
+import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
 import com.google.common.collect.MapDifference;
 import com.google.common.collect.Maps;
@@ -189,135 +190,102 @@ public class DefsTables
         Schema.instance.updateVersionAndAnnounce();
     }
 
-    private static Set<String> mergeKeyspaces(Map<DecoratedKey, ColumnFamily>
old, Map<DecoratedKey, ColumnFamily> updated)
+    private static Set<String> mergeKeyspaces(Map<DecoratedKey, ColumnFamily>
before, Map<DecoratedKey, ColumnFamily> after)
     {
-        // calculate the difference between old and new states (note that entriesOnlyLeft()
will be always empty)
-        MapDifference<DecoratedKey, ColumnFamily> diff = Maps.difference(old, updated);
-
-        /**
-         * At first step we check if any new keyspaces were added.
-         */
-        for (Map.Entry<DecoratedKey, ColumnFamily> entry : diff.entriesOnlyOnRight().entrySet())
-        {
-            ColumnFamily ksAttrs = entry.getValue();
-
-            // we don't care about nested ColumnFamilies here because those are going to
be processed separately
-            if (!(ksAttrs.getColumnCount() == 0))
-                addKeyspace(KSMetaData.fromSchema(new Row(entry.getKey(), entry.getValue()),
Collections.<CFMetaData>emptyList()));
-        }
-
-        /**
-         * At second step we check if there were any keyspaces re-created, in this context
-         * re-created means that they were previously deleted but still exist in the low-level
schema as empty keys
+        List<Row> created = new ArrayList<>();
+        List<String> altered = new ArrayList<>();
+        Set<String> dropped = new HashSet<>();
+
+        /*
+         * - we don't care about entriesOnlyOnLeft() or entriesInCommon(), because only the
changes are of interest to us
+         * - of all entriesOnlyOnRight(), we only care about ones that have live columns;
it's possible to have a ColumnFamily
+         *   there that only has the top-level deletion, if:
+         *      a) a pushed DROP KEYSPACE change for a keyspace hadn't ever made it to this
node in the first place
+         *      b) a pulled dropped keyspace that got dropped before it could find a way
to this node
+         * - of entriesDiffering(), we don't care about the scenario where both pre and post-values
have zero live columns:
+         *   that means that a keyspace had been recreated and dropped, and the recreated
keyspace had never found a way
+         *   to this node
          */
+        MapDifference<DecoratedKey, ColumnFamily> diff = Maps.difference(before, after);
 
-        Map<DecoratedKey, MapDifference.ValueDifference<ColumnFamily>> modifiedEntries
= diff.entriesDiffering();
-
-        // instead of looping over all modified entries and skipping processed keys all the
time
-        // we would rather store "left to process" items and iterate over them removing already
met keys
-        List<DecoratedKey> leftToProcess = new ArrayList<DecoratedKey>(modifiedEntries.size());
-
-        for (Map.Entry<DecoratedKey, MapDifference.ValueDifference<ColumnFamily>>
entry : modifiedEntries.entrySet())
-        {
-            ColumnFamily prevValue = entry.getValue().leftValue();
-            ColumnFamily newValue = entry.getValue().rightValue();
-
-            if (prevValue.getColumnCount() == 0)
-            {
-                addKeyspace(KSMetaData.fromSchema(new Row(entry.getKey(), newValue), Collections.<CFMetaData>emptyList()));
-                continue;
-            }
-
-            leftToProcess.add(entry.getKey());
-        }
-
-        if (leftToProcess.size() == 0)
-            return Collections.emptySet();
-
-        /**
-         * At final step we updating modified keyspaces and saving keyspaces drop them later
-         */
-
-        Set<String> keyspacesToDrop = new HashSet<String>();
+        for (Map.Entry<DecoratedKey, ColumnFamily> entry : diff.entriesOnlyOnRight().entrySet())
+            if (entry.getValue().getColumnCount() > 0)
+                created.add(new Row(entry.getKey(), entry.getValue()));
 
-        for (DecoratedKey key : leftToProcess)
+        for (Map.Entry<DecoratedKey, MapDifference.ValueDifference<ColumnFamily>>
entry : diff.entriesDiffering().entrySet())
         {
-            MapDifference.ValueDifference<ColumnFamily> valueDiff = modifiedEntries.get(key);
+            String keyspaceName = AsciiType.instance.compose(entry.getKey().key);
 
-            ColumnFamily newState = valueDiff.rightValue();
+            ColumnFamily pre  = entry.getValue().leftValue();
+            ColumnFamily post = entry.getValue().rightValue();
 
-            if (newState.getColumnCount() == 0)
-                keyspacesToDrop.add(AsciiType.instance.getString(key.key));
-            else
-                updateKeyspace(KSMetaData.fromSchema(new Row(key, newState), Collections.<CFMetaData>emptyList()));
+            if (pre.getColumnCount() > 0 && post.getColumnCount() > 0)
+                altered.add(keyspaceName);
+            else if (pre.getColumnCount() > 0)
+                dropped.add(keyspaceName);
+            else if (post.getColumnCount() > 0) // a (re)created keyspace
+                created.add(new Row(entry.getKey(), post));
         }
 
-        return keyspacesToDrop;
+        for (Row row : created)
+            addKeyspace(KSMetaData.fromSchema(row, Collections.<CFMetaData>emptyList()));
+        for (String name : altered)
+            updateKeyspace(name);
+        return dropped;
     }
 
-    private static void mergeColumnFamilies(Map<DecoratedKey, ColumnFamily> old, Map<DecoratedKey,
ColumnFamily> updated)
+    // see the comments for mergeKeyspaces()
+    private static void mergeColumnFamilies(Map<DecoratedKey, ColumnFamily> before,
Map<DecoratedKey, ColumnFamily> after)
     {
-        // calculate the difference between old and new states (note that entriesOnlyLeft()
will be always empty)
-        MapDifference<DecoratedKey, ColumnFamily> diff = Maps.difference(old, updated);
-
-        // check if any new Keyspaces with ColumnFamilies were added.
-        for (Map.Entry<DecoratedKey, ColumnFamily> entry : diff.entriesOnlyOnRight().entrySet())
-        {
-            ColumnFamily cfAttrs = entry.getValue();
+        List<CFMetaData> created = new ArrayList<>();
+        List<CFMetaData> altered = new ArrayList<>();
+        List<CFMetaData> dropped = new ArrayList<>();
 
-            if (!(cfAttrs.getColumnCount() == 0))
-            {
-               Map<String, CFMetaData> cfDefs = KSMetaData.deserializeColumnFamilies(new
Row(entry.getKey(), cfAttrs));
-
-                for (CFMetaData cfDef : cfDefs.values())
-                    addColumnFamily(cfDef);
-            }
-        }
+        MapDifference<DecoratedKey, ColumnFamily> diff = Maps.difference(before, after);
 
-        // deal with modified ColumnFamilies (remember that all of the keyspace nested ColumnFamilies
are put to the single row)
-        Map<DecoratedKey, MapDifference.ValueDifference<ColumnFamily>> modifiedEntries
= diff.entriesDiffering();
+        for (Map.Entry<DecoratedKey, ColumnFamily> entry : diff.entriesOnlyOnRight().entrySet())
+            if (entry.getValue().getColumnCount() > 0)
+                created.addAll(KSMetaData.deserializeColumnFamilies(new Row(entry.getKey(),
entry.getValue())).values());
 
-        for (DecoratedKey keyspace : modifiedEntries.keySet())
+        for (Map.Entry<DecoratedKey, MapDifference.ValueDifference<ColumnFamily>>
entry : diff.entriesDiffering().entrySet())
         {
-            MapDifference.ValueDifference<ColumnFamily> valueDiff = modifiedEntries.get(keyspace);
-
-            ColumnFamily prevValue = valueDiff.leftValue(); // state before external modification
-            ColumnFamily newValue = valueDiff.rightValue(); // updated state
+            String keyspaceName = AsciiType.instance.compose(entry.getKey().key);
 
-            Row newRow = new Row(keyspace, newValue);
+            ColumnFamily pre  = entry.getValue().leftValue();
+            ColumnFamily post = entry.getValue().rightValue();
 
-            if (prevValue.getColumnCount() == 0) // whole keyspace was deleted and now it's
re-created
+            if (pre.getColumnCount() > 0 && post.getColumnCount() > 0)
             {
-                for (CFMetaData cfm : KSMetaData.deserializeColumnFamilies(newRow).values())
-                    addColumnFamily(cfm);
+                MapDifference<String, CFMetaData> delta =
+                    Maps.difference(Schema.instance.getKSMetaData(keyspaceName).cfMetaData(),
+                                    KSMetaData.deserializeColumnFamilies(new Row(entry.getKey(),
post)));
+
+                dropped.addAll(delta.entriesOnlyOnLeft().values());
+                created.addAll(delta.entriesOnlyOnRight().values());
+                Iterables.addAll(altered, Iterables.transform(delta.entriesDiffering().values(),
new Function<MapDifference.ValueDifference<CFMetaData>, CFMetaData>()
+                {
+                    public CFMetaData apply(MapDifference.ValueDifference<CFMetaData>
pair)
+                    {
+                        return pair.rightValue();
+                    }
+                }));
             }
-            else if (newValue.getColumnCount() == 0) // whole keyspace is deleted
+            else if (pre.getColumnCount() > 0)
             {
-                for (CFMetaData cfm : KSMetaData.deserializeColumnFamilies(new Row(keyspace,
prevValue)).values())
-                    dropColumnFamily(cfm.ksName, cfm.cfName);
+                dropped.addAll(Schema.instance.getKSMetaData(keyspaceName).cfMetaData().values());
             }
-            else // has modifications in the nested ColumnFamilies, need to perform nested
diff to determine what was really changed
+            else if (post.getColumnCount() > 0)
             {
-                String ksName = AsciiType.instance.getString(keyspace.key);
-
-                Map<String, CFMetaData> oldCfDefs = new HashMap<String, CFMetaData>();
-                for (CFMetaData cfm : Schema.instance.getKSMetaData(ksName).cfMetaData().values())
-                    oldCfDefs.put(cfm.cfName, cfm);
-
-                Map<String, CFMetaData> newCfDefs = KSMetaData.deserializeColumnFamilies(newRow);
-
-                MapDifference<String, CFMetaData> cfDefDiff = Maps.difference(oldCfDefs,
newCfDefs);
-
-                for (CFMetaData cfDef : cfDefDiff.entriesOnlyOnRight().values())
-                    addColumnFamily(cfDef);
-
-                for (CFMetaData cfDef : cfDefDiff.entriesOnlyOnLeft().values())
-                    dropColumnFamily(cfDef.ksName, cfDef.cfName);
-
-                for (MapDifference.ValueDifference<CFMetaData> cfDef : cfDefDiff.entriesDiffering().values())
-                    updateColumnFamily(cfDef.rightValue());
+                created.addAll(KSMetaData.deserializeColumnFamilies(new Row(entry.getKey(),
post)).values());
             }
         }
+
+        for (CFMetaData cfm : created)
+            addColumnFamily(cfm);
+        for (CFMetaData cfm : altered)
+            updateColumnFamily(cfm.ksName, cfm.cfName);
+        for (CFMetaData cfm : dropped)
+            dropColumnFamily(cfm.ksName, cfm.cfName);
     }
 
     private static void addKeyspace(KSMetaData ksm)
@@ -355,9 +323,9 @@ public class DefsTables
         }
     }
 
-    private static void updateKeyspace(KSMetaData newState)
+    private static void updateKeyspace(String ksName)
     {
-        KSMetaData oldKsm = Schema.instance.getKSMetaData(newState.name);
+        KSMetaData oldKsm = Schema.instance.getKSMetaData(ksName);
         assert oldKsm != null;
         KSMetaData newKsm = KSMetaData.cloneWith(oldKsm.reloadAttributes(), oldKsm.cfMetaData().values());
 
@@ -365,14 +333,14 @@ public class DefsTables
 
         if (!StorageService.instance.isClientMode())
         {
-            Keyspace.open(newState.name).createReplicationStrategy(newKsm);
+            Keyspace.open(ksName).createReplicationStrategy(newKsm);
             MigrationManager.instance.notifyUpdateKeyspace(newKsm);
         }
     }
 
-    private static void updateColumnFamily(CFMetaData newState)
+    private static void updateColumnFamily(String ksName, String cfName)
     {
-        CFMetaData cfm = Schema.instance.getCFMetaData(newState.ksName, newState.cfName);
+        CFMetaData cfm = Schema.instance.getCFMetaData(ksName, cfName);
         assert cfm != null;
         cfm.reload();
 


Mime
View raw message