cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject cassandra git commit: Tool to create a leveling based on an existing set of sstables.
Date Mon, 02 Mar 2015 11:51:03 GMT
Repository: cassandra
Updated Branches:
  refs/heads/cassandra-2.0 5654e7368 -> 33a3a09cb


Tool to create a leveling based on an existing set of sstables.

Patch by marcuse; reviewed by carlyeks for CASSANDRA-8301


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/33a3a09c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/33a3a09c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/33a3a09c

Branch: refs/heads/cassandra-2.0
Commit: 33a3a09cbf2f0a31a823b6ef8f5e9fcc1546c9f5
Parents: 5654e73
Author: Marcus Eriksson <marcuse@apache.org>
Authored: Wed Dec 17 21:50:38 2014 +0100
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Mon Mar 2 11:41:25 2015 +0100

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../cassandra/tools/SSTableOfflineRelevel.java  | 191 +++++++++++++++++++
 tools/bin/sstableofflinerelevel                 |  49 +++++
 3 files changed, 241 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/33a3a09c/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 3a8d824..f98bb3f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.0.13:
+ * Add offline tool to relevel sstables (CASSANDRA-8301)
  * Preserve stream ID for more protocol errors (CASSANDRA-8848)
  * Fix combining token() function with multi-column relations on
    clustering columns (CASSANDRA-8797)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/33a3a09c/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java b/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
new file mode 100644
index 0000000..3fb2f7a
--- /dev/null
+++ b/src/java/org/apache/cassandra/tools/SSTableOfflineRelevel.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.cassandra.tools;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.cassandra.config.DatabaseDescriptor;
+import org.apache.cassandra.config.Schema;
+import org.apache.cassandra.db.DecoratedKey;
+import org.apache.cassandra.db.Directories;
+import org.apache.cassandra.db.Keyspace;
+import org.apache.cassandra.db.compaction.LeveledManifest;
+import org.apache.cassandra.io.sstable.Component;
+import org.apache.cassandra.io.sstable.Descriptor;
+import org.apache.cassandra.io.sstable.SSTableReader;
+import org.apache.cassandra.utils.Pair;
+
+/**
+ * Create a decent leveling for the given keyspace/column family
+ *
+ * Approach is to sort the sstables by their last token
+ *
+ * given an original leveling like this (note that [ ] indicates token boundaries, not sstable
size on disk, all sstables are the same size)
+ *
+ * L3 [][][][][][][][][][][]
+ * L2 [    ][    ][    ][  ]
+ * L1 [          ][        ]
+ * L0 [                    ]
+ *
+ * Will look like this after being dropped to L0 and sorted by last token (and, to illustrate
overlap, the overlapping ones are put on a new line):
+ *
+ * [][][]
+ * [    ][][][]
+ *       [    ]
+ * [          ]
+ *...
+ *
+ * Then, we start iterating from the smallest last-token and adding all sstables that do
not cause an overlap to
+ * a level, we will reconstruct the original leveling top-down. Whenever we add an sstable
to the level, we remove it from
+ * the sorted list. Once we reach the end of the sorted list, we have a full level, and can
start over with the level below.
+ *
+ * If we end up with more levels than expected, we put all levels exceeding the expected
in L0, for example, original L0
+ * files will most likely be put in a level of its own since they most often overlap many
other sstables
+ */
+public class SSTableOfflineRelevel
+{
+    /**
+     * @param args a list of sstables whose metadata we are changing
+     */
+    public static void main(String[] args) throws IOException
+    {
+        PrintStream out = System.out;
+        if (args.length < 2)
+        {
+            out.println("This command should be run with Cassandra stopped!");
+            out.println("Usage: sstableofflinerelevel [--dry-run] <keyspace> <columnfamily>");
+            System.exit(1);
+        }
+        boolean dryRun = args[0].equals("--dry-run");
+        String keyspace = args[args.length - 2];
+        String columnfamily = args[args.length - 1];
+        DatabaseDescriptor.loadSchemas();
+
+        if (Schema.instance.getCFMetaData(keyspace, columnfamily) == null)
+            throw new IllegalArgumentException(String.format("Unknown keyspace/columnFamily
%s.%s",
+                    keyspace,
+                    columnfamily));
+
+        Keyspace.openWithoutSSTables(keyspace);
+        Directories directories = Directories.create(keyspace, columnfamily);
+        Set<SSTableReader> sstables = new HashSet<>();
+        for (Map.Entry<Descriptor, Set<Component>> sstable : directories.sstableLister().list().entrySet())
+        {
+            if (sstable.getKey() != null)
+            {
+                SSTableReader reader = SSTableReader.open(sstable.getKey());
+                sstables.add(reader);
+            }
+        }
+        if (sstables.isEmpty())
+        {
+            out.println("No sstables to relevel for "+keyspace+"."+columnfamily);
+            System.exit(1);
+        }
+        Relevel rl = new Relevel(sstables);
+        rl.relevel(dryRun);
+        System.exit(0);
+
+    }
+
+    private static class Relevel
+    {
+        private final Set<SSTableReader> sstables;
+        private final int approxExpectedLevels;
+        public Relevel(Set<SSTableReader> sstables)
+        {
+            this.sstables = sstables;
+            approxExpectedLevels = (int) Math.ceil(Math.log10(sstables.size()));
+        }
+
+        public void relevel(boolean dryRun) throws IOException
+        {
+            List<SSTableReader> sortedSSTables = new ArrayList<>(sstables);
+            Collections.sort(sortedSSTables, new Comparator<SSTableReader>()
+            {
+                @Override
+                public int compare(SSTableReader o1, SSTableReader o2)
+                {
+                    return o1.last.compareTo(o2.last);
+                }
+            });
+
+            List<List<SSTableReader>> levels = new ArrayList<>();
+
+            while (!sortedSSTables.isEmpty())
+            {
+                Iterator<SSTableReader> it = sortedSSTables.iterator();
+                List<SSTableReader> level = new ArrayList<>();
+                DecoratedKey lastLast = null;
+                while (it.hasNext())
+                {
+                    SSTableReader sstable = it.next();
+                    if (lastLast == null || lastLast.compareTo(sstable.first) < 0)
+                    {
+                        level.add(sstable);
+                        lastLast = sstable.last;
+                        it.remove();
+                    }
+                }
+                levels.add(level);
+            }
+            List<SSTableReader> l0 = new ArrayList<>();
+            if (approxExpectedLevels < levels.size())
+            {
+                for (int i = approxExpectedLevels; i < levels.size(); i++)
+                    l0.addAll(levels.get(i));
+                levels = levels.subList(0, approxExpectedLevels);
+            }
+            if (dryRun)
+                System.out.println("Potential leveling: ");
+            else
+                System.out.println("New leveling: ");
+
+            System.out.println("L0="+l0.size());
+            for (int i = levels.size() - 1; i >= 0; i--)
+                System.out.println(String.format("L%d %d", levels.size() - i, levels.get(i).size()));
+
+            if (!dryRun)
+            {
+                for (SSTableReader sstable : l0)
+                {
+                    if (sstable.getSSTableLevel() != 0)
+                        LeveledManifest.mutateLevel(Pair.create(sstable.getSSTableMetadata(),
sstable.getAncestors()), sstable.descriptor, sstable.descriptor.filenameFor(Component.STATS),
0);
+                }
+                for (int i = levels.size() - 1; i >= 0; i--)
+                {
+                    for (SSTableReader sstable : levels.get(i))
+                    {
+                        int newLevel = levels.size() - i;
+                        if (newLevel != sstable.getSSTableLevel())
+                            LeveledManifest.mutateLevel(Pair.create(sstable.getSSTableMetadata(),
sstable.getAncestors()), sstable.descriptor, sstable.descriptor.filenameFor(Component.STATS),
newLevel);
+                    }
+                }
+            }
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/cassandra/blob/33a3a09c/tools/bin/sstableofflinerelevel
----------------------------------------------------------------------
diff --git a/tools/bin/sstableofflinerelevel b/tools/bin/sstableofflinerelevel
new file mode 100755
index 0000000..19cb4c1
--- /dev/null
+++ b/tools/bin/sstableofflinerelevel
@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
+    for include in "`dirname $0`/cassandra.in.sh" \
+                   "$HOME/.cassandra.in.sh" \
+                   /usr/share/cassandra/cassandra.in.sh \
+                   /usr/local/share/cassandra/cassandra.in.sh \
+                   /opt/cassandra/cassandra.in.sh; do
+        if [ -r $include ]; then
+            . $include
+            break
+        fi
+    done
+elif [ -r $CASSANDRA_INCLUDE ]; then
+    . $CASSANDRA_INCLUDE
+fi
+
+
+# Use JAVA_HOME if set, otherwise look for java in PATH
+if [ -x $JAVA_HOME/bin/java ]; then
+    JAVA=$JAVA_HOME/bin/java
+else
+    JAVA=`which java`
+fi
+
+if [ -z $CLASSPATH ]; then
+    echo "You must set the CLASSPATH var" >&2
+    exit 1
+fi
+
+$JAVA -cp $CLASSPATH  -Dstorage-config=$CASSANDRA_CONF \
+        -Dlog4j.configuration=log4j-tools.properties \
+        org.apache.cassandra.tools.SSTableOfflineRelevel "$@"


Mime
View raw message