cassandra-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From marc...@apache.org
Subject [2/3] cassandra git commit: Merge branch 'cassandra-2.0' into cassandra-2.1
Date Tue, 11 Aug 2015 08:26:08 GMT
Merge branch 'cassandra-2.0' into cassandra-2.1

Conflicts:
	CHANGES.txt
	test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/012b987e
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/012b987e
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/012b987e

Branch: refs/heads/cassandra-2.2
Commit: 012b987e267bdfb39b44c8afd4f4ccbb48a1ddf4
Parents: 76e8eb8 028e7cb
Author: Marcus Eriksson <marcuse@apache.org>
Authored: Tue Aug 11 09:46:33 2015 +0200
Committer: Marcus Eriksson <marcuse@apache.org>
Committed: Tue Aug 11 10:11:15 2015 +0200

----------------------------------------------------------------------
 CHANGES.txt                                     |   1 +
 .../cassandra/tools/SSTableExpiredBlockers.java | 136 +++++++++++++++++++
 .../cassandra/db/compaction/TTLExpiryTest.java  |  29 ++++
 tools/bin/sstableexpiredblockers                |  54 ++++++++
 tools/bin/sstableexpiredblockers.bat            |  23 ++++
 5 files changed, 243 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cassandra/blob/012b987e/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 462de44,7d84538..f7fb63c
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,26 -1,5 +1,27 @@@
 -2.0.17
 +2.1.9
 + * Add new JMX methods to change local compaction strategy (CASSANDRA-9965)
 + * Write hints for paxos commits (CASSANDRA-7342)
 + * (cqlsh) Fix timestamps before 1970 on Windows, always
 +   use UTC for timestamp display (CASSANDRA-10000)
 + * (cqlsh) Avoid overwriting new config file with old config
 +   when both exist (CASSANDRA-9777)
 + * Release snapshot selfRef when doing snapshot repair (CASSANDRA-9998)
 + * Cannot replace token does not exist - DN node removed as Fat Client (CASSANDRA-9871)
 + * Fix handling of enable/disable autocompaction (CASSANDRA-9899)
 + * Commit log segment recycling is disabled by default (CASSANDRA-9896)
 + * Add consistency level to tracing ouput (CASSANDRA-9827)
 + * Fix MarshalException when upgrading superColumn family (CASSANDRA-9582)
 + * Fix broken logging for "empty" flushes in Memtable (CASSANDRA-9837)
 + * Handle corrupt files on startup (CASSANDRA-9686)
 + * Fix clientutil jar and tests (CASSANDRA-9760)
 + * (cqlsh) Allow the SSL protocol version to be specified through the
 +   config file or environment variables (CASSANDRA-9544)
 + * Remove repair snapshot leftover on startup (CASSANDRA-7357)
 + * Use random nodes for batch log when only 2 racks (CASSANDRA-8735)
 + * Ensure atomicity inside thrift and stream session (CASSANDRA-7757)
 + * Fix nodetool info error when the node is not joined (CASSANDRA-9031)
 +Merged from 2.0:
+  * Add tool to find why expired sstables are not getting dropped (CASSANDRA-10015)
   * Remove erroneous pending HH tasks from tpstats/jmx (CASSANDRA-9129)
   * Don't cast expected bf size to an int (CASSANDRA-9959)
   * Log when messages are dropped due to cross_node_timeout (CASSANDRA-9793)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/012b987e/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
index 0000000,b5fa779..b4f7063
mode 000000,100644..100644
--- a/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
+++ b/src/java/org/apache/cassandra/tools/SSTableExpiredBlockers.java
@@@ -1,0 -1,135 +1,136 @@@
+ /*
+  * Licensed to the Apache Software Foundation (ASF) under one
+  * or more contributor license agreements.  See the NOTICE file
+  * distributed with this work for additional information
+  * regarding copyright ownership.  The ASF licenses this file
+  * to you under the Apache License, Version 2.0 (the
+  * "License"); you may not use this file except in compliance
+  * with the License.  You may obtain a copy of the License at
+  *
+  *     http://www.apache.org/licenses/LICENSE-2.0
+  *
+  * Unless required by applicable law or agreed to in writing, software
+  * distributed under the License is distributed on an "AS IS" BASIS,
+  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  * See the License for the specific language governing permissions and
+  * limitations under the License.
+  */
+ package org.apache.cassandra.tools;
+ 
+ import java.io.IOException;
+ import java.io.PrintStream;
+ import java.util.Collections;
+ import java.util.HashSet;
+ import java.util.Map;
+ import java.util.Set;
+ 
+ import com.google.common.base.Throwables;
+ import com.google.common.collect.ArrayListMultimap;
+ import com.google.common.collect.Multimap;
+ 
+ import org.apache.cassandra.config.CFMetaData;
+ import org.apache.cassandra.config.DatabaseDescriptor;
+ import org.apache.cassandra.config.Schema;
++import org.apache.cassandra.db.ColumnFamilyStore;
+ import org.apache.cassandra.db.Directories;
+ import org.apache.cassandra.db.Keyspace;
+ import org.apache.cassandra.io.sstable.Component;
+ import org.apache.cassandra.io.sstable.Descriptor;
+ import org.apache.cassandra.io.sstable.SSTableReader;
+ 
+ /**
+  * During compaction we can drop entire sstables if they only contain expired tombstones
and if it is guaranteed
+  * to not cover anything in other sstables. An expired sstable can be blocked from getting
dropped if its newest
+  * timestamp is newer than the oldest data in another sstable.
+  *
+  * This class outputs all sstables that are blocking other sstables from getting dropped
so that a user can
+  * figure out why certain sstables are still on disk.
+  */
+ public class SSTableExpiredBlockers
+ {
+     public static void main(String[] args) throws IOException
+     {
+         PrintStream out = System.out;
+         if (args.length < 2)
+         {
+             out.println("Usage: sstableexpiredblockers <keyspace> <table>");
+             System.exit(1);
+         }
+         String keyspace = args[args.length - 2];
+         String columnfamily = args[args.length - 1];
+         DatabaseDescriptor.loadSchemas();
+ 
+         CFMetaData metadata = Schema.instance.getCFMetaData(keyspace, columnfamily);
+         if (metadata == null)
+             throw new IllegalArgumentException(String.format("Unknown keyspace/table %s.%s",
+                     keyspace,
+                     columnfamily));
+ 
 -        Keyspace.openWithoutSSTables(keyspace);
 -        Directories directories = Directories.create(keyspace, columnfamily);
++        Keyspace ks = Keyspace.openWithoutSSTables(keyspace);
++        ColumnFamilyStore cfs = ks.getColumnFamilyStore(columnfamily);
++        Directories.SSTableLister lister = cfs.directories.sstableLister().skipTemporary(true);
+         Set<SSTableReader> sstables = new HashSet<>();
 -
 -        for (Map.Entry<Descriptor, Set<Component>> sstable : directories.sstableLister().skipTemporary(true).list().entrySet())
++        for (Map.Entry<Descriptor, Set<Component>> sstable : lister.list().entrySet())
+         {
+             if (sstable.getKey() != null)
+             {
+                 try
+                 {
+                     SSTableReader reader = SSTableReader.open(sstable.getKey());
+                     sstables.add(reader);
+                 }
+                 catch (Throwable t)
+                 {
+                     out.println("Couldn't open sstable: " + sstable.getKey().filenameFor(Component.DATA));
+                     Throwables.propagate(t);
+                 }
+             }
+         }
+         if (sstables.isEmpty())
+         {
+             out.println("No sstables for " + keyspace + "." + columnfamily);
+             System.exit(1);
+         }
+ 
+         int gcBefore = (int)(System.currentTimeMillis()/1000) - metadata.getGcGraceSeconds();
+         Multimap<SSTableReader, SSTableReader> blockers = checkForExpiredSSTableBlockers(sstables,
gcBefore);
+         for (SSTableReader blocker : blockers.keySet())
+         {
+             out.println(String.format("%s blocks %d expired sstables from getting dropped:
%s%n",
+                                     formatForExpiryTracing(Collections.singleton(blocker)),
+                                     blockers.get(blocker).size(),
+                                     formatForExpiryTracing(blockers.get(blocker))));
+         }
+ 
+         System.exit(0);
+     }
+ 
+     public static Multimap<SSTableReader, SSTableReader> checkForExpiredSSTableBlockers(Iterable<SSTableReader>
sstables, int gcBefore)
+     {
+         Multimap<SSTableReader, SSTableReader> blockers = ArrayListMultimap.create();
+         for (SSTableReader sstable : sstables)
+         {
+             if (sstable.getSSTableMetadata().maxLocalDeletionTime < gcBefore)
+             {
+                 for (SSTableReader potentialBlocker : sstables)
+                 {
+                     if (!potentialBlocker.equals(sstable) &&
+                         potentialBlocker.getMinTimestamp() <= sstable.getMaxTimestamp()
&&
+                         potentialBlocker.getSSTableMetadata().maxLocalDeletionTime >
gcBefore)
+                         blockers.put(potentialBlocker, sstable);
+                 }
+             }
+         }
+         return blockers;
+     }
+ 
+     private static String formatForExpiryTracing(Iterable<SSTableReader> sstables)
+     {
+         StringBuilder sb = new StringBuilder();
+ 
+         for (SSTableReader sstable : sstables)
+             sb.append(String.format("[%s (minTS = %d, maxTS = %d, maxLDT = %d)]", sstable,
sstable.getMinTimestamp(), sstable.getMaxTimestamp(), sstable.getSSTableMetadata().maxLocalDeletionTime)).append(",
");
+ 
+         return sb.toString();
+     }
+ }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/012b987e/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
----------------------------------------------------------------------
diff --cc test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
index 678601b,5a83c76..a5b376e
--- a/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
+++ b/test/unit/org/apache/cassandra/db/compaction/TTLExpiryTest.java
@@@ -20,6 -20,13 +20,7 @@@ package org.apache.cassandra.db.compact
   * 
   */
  
 -
 -
 -import java.util.Collections;
 -import java.util.Set;
 -import java.util.concurrent.ExecutionException;
 -
+ import com.google.common.collect.Multimap;
  import com.google.common.collect.Sets;
  import org.junit.Test;
  import org.junit.runner.RunWith;
@@@ -27,15 -34,16 +28,16 @@@
  import org.apache.cassandra.OrderedJUnit4ClassRunner;
  import org.apache.cassandra.SchemaLoader;
  import org.apache.cassandra.Util;
 -import org.apache.cassandra.db.ColumnFamilyStore;
 -import org.apache.cassandra.db.DataRange;
 -import org.apache.cassandra.db.DecoratedKey;
 -import org.apache.cassandra.db.Keyspace;
 -import org.apache.cassandra.db.RowMutation;
 +import org.apache.cassandra.db.*;
  import org.apache.cassandra.db.columniterator.OnDiskAtomIterator;
 +import org.apache.cassandra.io.sstable.ISSTableScanner;
  import org.apache.cassandra.io.sstable.SSTableReader;
 -import org.apache.cassandra.io.sstable.SSTableScanner;
+ import org.apache.cassandra.tools.SSTableExpiredBlockers;
  import org.apache.cassandra.utils.ByteBufferUtil;
 +
 +import java.util.Collections;
 +import java.util.Set;
 +
  import static org.junit.Assert.assertEquals;
  import static org.junit.Assert.assertTrue;
  
@@@ -189,4 -152,78 +191,31 @@@ public class TTLExpiryTest extends Sche
              assertEquals(noTTLKey, iter.getKey());
          }
      }
+ 
+     @Test
 -    public void testAggressiveFullyExpired()
 -    {
 -        String KEYSPACE1 = "Keyspace1";
 -        ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
 -        cfs.disableAutoCompaction();
 -        cfs.metadata.gcGraceSeconds(0);
 -
 -        DecoratedKey ttlKey = Util.dk("ttl");
 -        RowMutation rm = new RowMutation("Keyspace1", ttlKey.key);
 -        rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER,
1, 1);
 -        rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER,
3, 1);
 -        rm.applyUnsafe();
 -        cfs.forceBlockingFlush();
 -
 -        rm = new RowMutation(KEYSPACE1, ttlKey.key);
 -        rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER,
2, 1);
 -        rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER,
5, 1);
 -        rm.applyUnsafe();
 -        cfs.forceBlockingFlush();
 -
 -        rm = new RowMutation(KEYSPACE1, ttlKey.key);
 -        rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER,
4, 1);
 -        rm.add("Standard1", ByteBufferUtil.bytes("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER,
7, 1);
 -        rm.applyUnsafe();
 -        cfs.forceBlockingFlush();
 -
 -        rm = new RowMutation(KEYSPACE1, ttlKey.key);
 -        rm.add("Standard1", ByteBufferUtil.bytes("shadow"), ByteBufferUtil.EMPTY_BYTE_BUFFER,
6, 3);
 -        rm.add("Standard1", ByteBufferUtil.bytes("col2"), ByteBufferUtil.EMPTY_BYTE_BUFFER,
8, 1);
 -        rm.applyUnsafe();
 -        cfs.forceBlockingFlush();
 -
 -        Set<SSTableReader> sstables = Sets.newHashSet(cfs.getSSTables());
 -        int now = (int)(System.currentTimeMillis() / 1000);
 -        int gcBefore = now + 2;
 -        Set<SSTableReader> expired = CompactionController.getFullyExpiredSSTables(
 -                cfs,
 -                sstables,
 -                Collections.EMPTY_SET,
 -                gcBefore);
 -        assertEquals(2, expired.size());
 -
 -        cfs.clearUnsafe();
 -    }
 -
 -    @Test
+     public void testCheckForExpiredSSTableBlockers() throws InterruptedException
+     {
+         String KEYSPACE1 = "Keyspace1";
+         ColumnFamilyStore cfs = Keyspace.open("Keyspace1").getColumnFamilyStore("Standard1");
+         cfs.truncateBlocking();
+         cfs.disableAutoCompaction();
+         cfs.metadata.gcGraceSeconds(0);
+ 
 -        RowMutation rm = new RowMutation(KEYSPACE1, Util.dk("test").key);
 -        rm.add("Standard1", ByteBufferUtil.bytes("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER,
System.currentTimeMillis());
++        Mutation rm = new Mutation(KEYSPACE1, Util.dk("test").getKey());
++        rm.add("Standard1", Util.cellname("col1"), ByteBufferUtil.EMPTY_BYTE_BUFFER, System.currentTimeMillis());
+         rm.applyUnsafe();
+         cfs.forceBlockingFlush();
+         SSTableReader blockingSSTable = cfs.getSSTables().iterator().next();
+         for (int i = 0; i < 10; i++)
+         {
 -            rm = new RowMutation(KEYSPACE1, Util.dk("test").key);
++            rm = new Mutation(KEYSPACE1, Util.dk("test").getKey());
+             rm.delete("Standard1", System.currentTimeMillis());
+             rm.applyUnsafe();
+             cfs.forceBlockingFlush();
+         }
+         Multimap<SSTableReader, SSTableReader> blockers = SSTableExpiredBlockers.checkForExpiredSSTableBlockers(cfs.getSSTables(),
(int) (System.currentTimeMillis() / 1000) + 100);
+         assertEquals(1, blockers.keySet().size());
+         assertTrue(blockers.keySet().contains(blockingSSTable));
+         assertEquals(10, blockers.get(blockingSSTable).size());
+     }
 -
  }

http://git-wip-us.apache.org/repos/asf/cassandra/blob/012b987e/tools/bin/sstableexpiredblockers
----------------------------------------------------------------------
diff --cc tools/bin/sstableexpiredblockers
index 0000000,58cefce..0027208
mode 000000,100755..100755
--- a/tools/bin/sstableexpiredblockers
+++ b/tools/bin/sstableexpiredblockers
@@@ -1,0 -1,49 +1,54 @@@
+ #!/bin/sh
+ 
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements.  See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership.  The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License.  You may obtain a copy of the License at
+ #
+ #     http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ 
+ if [ "x$CASSANDRA_INCLUDE" = "x" ]; then
 -    for include in "`dirname $0`/cassandra.in.sh" \
 -                   "$HOME/.cassandra.in.sh" \
 -                   /usr/share/cassandra/cassandra.in.sh \
++    for include in /usr/share/cassandra/cassandra.in.sh \
+                    /usr/local/share/cassandra/cassandra.in.sh \
 -                   /opt/cassandra/cassandra.in.sh; do
 -        if [ -r $include ]; then
 -            . $include
++                   /opt/cassandra/cassandra.in.sh \
++                   ~/.cassandra.in.sh \
++                   "`dirname "$0"`/cassandra.in.sh"; do
++        if [ -r "$include" ]; then
++            . "$include"
+             break
+         fi
+     done
 -elif [ -r $CASSANDRA_INCLUDE ]; then
 -    . $CASSANDRA_INCLUDE
++elif [ -r "$CASSANDRA_INCLUDE" ]; then
++    . "$CASSANDRA_INCLUDE"
+ fi
+ 
 -
+ # Use JAVA_HOME if set, otherwise look for java in PATH
 -if [ -x $JAVA_HOME/bin/java ]; then
 -    JAVA=$JAVA_HOME/bin/java
++if [ -x "$JAVA_HOME/bin/java" ]; then
++    JAVA="$JAVA_HOME/bin/java"
+ else
 -    JAVA=`which java`
++    JAVA="`which java`"
+ fi
+ 
+ if [ -z "$CLASSPATH" ]; then
+     echo "You must set the CLASSPATH var" >&2
+     exit 1
+ fi
+ 
 -"$JAVA" -cp "$CLASSPATH"  -Dstorage-config=$CASSANDRA_CONF \
 -        -Dlog4j.configuration=log4j-tools.properties \
++if [ "x$MAX_HEAP_SIZE" = "x" ]; then
++    MAX_HEAP_SIZE="256M"
++fi
++
++"$JAVA" $JAVA_AGENT -ea -cp "$CLASSPATH" -Xmx$MAX_HEAP_SIZE \
++        -Dcassandra.storagedir="$cassandra_storagedir" \
++        -Dlogback.configurationFile=logback-tools.xml \
+         org.apache.cassandra.tools.SSTableExpiredBlockers "$@"
++

http://git-wip-us.apache.org/repos/asf/cassandra/blob/012b987e/tools/bin/sstableexpiredblockers.bat
----------------------------------------------------------------------
diff --cc tools/bin/sstableexpiredblockers.bat
index 0000000,0000000..7af1105
new file mode 100644
--- /dev/null
+++ b/tools/bin/sstableexpiredblockers.bat
@@@ -1,0 -1,0 +1,23 @@@
++@REM  Licensed to the Apache Software Foundation (ASF) under one or more
++@REM  contributor license agreements.  See the NOTICE file distributed with
++@REM  this work for additional information regarding copyright ownership.
++@REM  The ASF licenses this file to You under the Apache License, Version 2.0
++@REM  (the "License"); you may not use this file except in compliance with
++@REM  the License.  You may obtain a copy of the License at
++@REM
++@REM      http://www.apache.org/licenses/LICENSE-2.0
++@REM
++@REM  Unless required by applicable law or agreed to in writing, software
++@REM  distributed under the License is distributed on an "AS IS" BASIS,
++@REM  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++@REM  See the License for the specific language governing permissions and
++@REM  limitations under the License.
++
++@echo off
++
++if "%OS%" == "Windows_NT" setlocal
++
++pushd "%~dp0"
++call cassandra.in.bat
++
++"%JAVA_HOME%\bin\java" -cp %CLASSPATH% org.apache.cassandra.tools.SSTableExpiredBlockers
%*


Mime
View raw message