Return-Path: X-Original-To: apmail-cassandra-commits-archive@www.apache.org Delivered-To: apmail-cassandra-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DFE65186EA for ; Tue, 11 Aug 2015 08:26:53 +0000 (UTC) Received: (qmail 14631 invoked by uid 500); 11 Aug 2015 08:26:53 -0000 Delivered-To: apmail-cassandra-commits-archive@cassandra.apache.org Received: (qmail 14587 invoked by uid 500); 11 Aug 2015 08:26:53 -0000 Mailing-List: contact commits-help@cassandra.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@cassandra.apache.org Delivered-To: mailing list commits@cassandra.apache.org Received: (qmail 14545 invoked by uid 99); 11 Aug 2015 08:26:53 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 11 Aug 2015 08:26:53 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 83EE6DFC1C; Tue, 11 Aug 2015 08:26:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: marcuse@apache.org To: commits@cassandra.apache.org Date: Tue, 11 Aug 2015 08:26:53 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/5] cassandra git commit: Add tool to find why expired sstables are not getting dropped Repository: cassandra Updated Branches: refs/heads/trunk b1c739874 -> 9e8741f28 Add tool to find why expired sstables are not getting dropped Patch by marcuse; reviewed by stefania for CASSANDRA-10015 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/028e7cb5 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/028e7cb5 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/028e7cb5 Branch: refs/heads/trunk Commit: 028e7cb5afb633cfb5197b7d29224b67b083b670 Parents: de84a5c Author: Marcus Eriksson Authored: Fri Aug 7 16:09:18 2015 +0200 Committer: Marcus Eriksson Committed: Tue Aug 11 09:31:30 2015 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../cassandra/tools/SSTableExpiredBlockers.java | 135 +++++++++++++++++++ .../cassandra/db/compaction/TTLExpiryTest.java | 29 ++++ tools/bin/sstableexpiredblockers | 49 +++++++ 4 files changed, 214 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/028e7cb5/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index fe060af..7d84538 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 2.0.17 + * Add tool to find why expired sstables are not getting dropped (CASSANDRA-10015) * Remove erroneous pending HH tasks from tpstats/jmx (CASSANDRA-9129) * Don't cast expected bf size to an int (CASSANDRA-9959) * Log when messages are dropped due to cross_node_timeout (CASSANDRA-9793) http://git-wip-us.apache.org/repos/asf/cassandra/blob/028e7cb5/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java new file mode 100644 index 0000000..b5fa779 --- /dev/null +++ b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.tools; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import com.google.common.base.Throwables; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.db.Directories; +import org.apache.cassandra.db.Keyspace; +import org.apache.cassandra.io.sstable.Component; +import org.apache.cassandra.io.sstable.Descriptor; +import org.apache.cassandra.io.sstable.SSTableReader; + +/** + * During compaction we can drop entire sstables if they only contain expired tombstones and if it is guaranteed + * to not cover anything in other sstables. An expired sstable can be blocked from getting dropped if its newest + * timestamp is newer than the oldest data in another sstable. + * + * This class outputs all sstables that are blocking other sstables from getting dropped so that a user can + * figure out why certain sstables are still on disk. + */ +public class SSTableExpiredBlockers +{ + public static void main(String[] args) throws IOException + { + PrintStream out = System.out; + if (args.length < 2) + { + out.println("Usage: sstableexpiredblockers "); + System.exit(1); + } + String keyspace = args[args.length - 2]; + String columnfamily = args[args.length - 1]; + DatabaseDescriptor.loadSchemas(); + + CFMetaData metadata = Schema.instance.getCFMetaData(keyspace, columnfamily); + if (metadata == null) + throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s", + keyspace, + columnfamily)); + + Keyspace.openWithoutSSTables(keyspace); + Directories directories = Directories.create(keyspace, columnfamily); + Set sstables = new HashSet<>(); + + for (Map.Entry> sstable : directories.sstableLister().skipTemporary(true).list().entrySet()) + { + if (sstable.getKey() != null) + { + try + { + SSTableReader reader = SSTableReader.open(sstable.getKey()); + sstables.add(reader); + } + catch (Throwable t) + { + out.println("Couldn't open sstable: " + sstable.getKey().filenameFor(Component.DATA)); + Throwables.propagate(t); + } + } + } + if (sstables.isEmpty()) + { + out.println("No sstables for " + keyspace + "." + columnfamily); + System.exit(1); + } + + int gcBefore = (int)(System.currentTimeMillis()/1000) - metadata.getGcGraceSeconds(); + Multimap blockers = checkForExpiredSSTableBlockers(sstables, gcBefore); + for (SSTableReader blocker : blockers.keySet()) + { + out.println(String.format("%s blocks %d expired sstables from getting dropped: %s%n", + formatForExpiryTracing(Collections.singleton(blocker)), + blockers.get(blocker).size(), + formatForExpiryTracing(blockers.get(blocker)))); + } + + System.exit(0); + } + + public static Multimap checkForExpiredSSTableBlockers(Iterable sstables, int gcBefore) + { + Multimap blockers = ArrayListMultimap.create(); + for (SSTableReader sstable : sstables) + { + if (sstable.getSSTableMetadata().maxLocalDeletionTime < gcBefore) + { + for (SSTableReader potentialBlocker : sstables) + { + if (!potentialBlocker.equals(sstable) && + potentialBlocker.getMinTimestamp() <= sstable.getMaxTimestamp() && + potentialBlocker.getSSTableMetadata().maxLocalDeletionTime > gcBefore) + blockers.put(potentialBlocker, sstable); + } + } + } + return blockers; + } + + private static String formatForExpiryTracing(Iterable sstables) + { + StringBuilder sb = new StringBuilder(); + + for (SSTableReader sstable : sstables) + sb.append(String.format("[%s (minTS = %d, maxTS = %d, maxLDT = %d)]", sstable, sstable.getMinTimestamp(), sstable.getMaxTimestamp(), sstable.getSSTableMetadata().maxLocalDeletionTime)).append(", "); + + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/028e7cb5/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java index 3fad0ec..5a83c76 100644 --- a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java @@ -26,6 +26,7 @@ import java.util.Collections; import java.util.Set; import java.util.concurrent.ExecutionException; +import com.google.common.collect.Multimap; import com.google.common.collect.Sets; import org.junit.Test; import org.junit.runner.RunWith; @@ -41,6 +42,7 @@ import org.apache.cassandra.db.RowMutation; import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; import org.apache.cassandra.io.sstable.SSTableReader; import org.apache.cassandra.io.sstable.SSTableScanner; +import org.apache.cassandra.tools.SSTableExpiredBlockers; import org.apache.cassandra.utils.ByteBufferUtil; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -197,4 +199,31 @@ public class TTLExpiryTest extends SchemaLoader cfs.clearUnsafe(); } + @Test + public void testCheckForExpiredSSTableBlockers() throws InterruptedException + { + String KEYSPACE1 = "Keyspace1"; + ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1"); + cfs.truncateBlocking(); + cfs.disableAutoCompaction(); + cfs.metadata.gcGraceSeconds(0); + + RowMutation rm = new RowMutation(KEYSPACE1, Util.dk("test").key); + rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis()); + rm.applyUnsafe(); + cfs.forceBlockingFlush(); + SSTableReader blockingSSTable = cfs.getSSTables().iterator().next(); + for (int i = 0; i < 10; i++) + { + rm = new RowMutation(KEYSPACE1, Util.dk("test").key); + rm.delete("Standard1", System.currentTimeMillis()); + rm.applyUnsafe(); + cfs.forceBlockingFlush(); + } + Multimap blockers = SSTableExpiredBlockers.checkForExpiredSSTableBlockers(cfs.getSSTables(), (int) (System.currentTimeMillis() / 1000) + 100); + assertEquals(1, blockers.keySet().size()); + assertTrue(blockers.keySet().contains(blockingSSTable)); + assertEquals(10, blockers.get(blockingSSTable).size()); + } + } http://git-wip-us.apache.org/repos/asf/cassandra/blob/028e7cb5/tools/bin/sstableexpiredblockers ---------------------------------------------------------------------- diff --git a/tools/bin/sstableexpiredblockers b/tools/bin/sstableexpiredblockers new file mode 100755 index 0000000..58cefce --- /dev/null +++ b/tools/bin/sstableexpiredblockers @@ -0,0 +1,49 @@ +#!/bin/sh + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [ "x$CASSANDRA_INCLUDE" = "x" ]; then + for include in "`dirname $0`/cassandra.in.sh" \ + "$HOME/.cassandra.in.sh" \ + /usr/share/cassandra/cassandra.in.sh \ + /usr/local/share/cassandra/cassandra.in.sh \ + /opt/cassandra/cassandra.in.sh; do + if [ -r $include ]; then + . $include + break + fi + done +elif [ -r $CASSANDRA_INCLUDE ]; then + . $CASSANDRA_INCLUDE +fi + + +# Use JAVA_HOME if set, otherwise look for java in PATH +if [ -x $JAVA_HOME/bin/java ]; then + JAVA=$JAVA_HOME/bin/java +else + JAVA=`which java` +fi + +if [ -z "$CLASSPATH" ]; then + echo "You must set the CLASSPATH var" >&2 + exit 1 +fi + +"$JAVA" -cp "$CLASSPATH" -Dstorage-config=$CASSANDRA_CONF \ + -Dlog4j.configuration=log4j-tools.properties \ + org.apache.cassandra.tools.SSTableExpiredBlockers "$@"